open-consul/agent/consul/server_test.go

// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0

package consul

import (
	"context"
	"crypto/x509"
	"fmt"
	"net"
	"os"
	"reflect"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/armon/go-metrics"
	"github.com/google/tcpproxy"
	"github.com/hashicorp/go-hclog"
	"github.com/hashicorp/go-uuid"
	"github.com/hashicorp/memberlist"
	"github.com/hashicorp/raft"
	"github.com/stretchr/testify/mock"
	"github.com/stretchr/testify/require"
	"golang.org/x/time/rate"
	"google.golang.org/grpc"

	"github.com/hashicorp/consul/agent/hcp"

	"github.com/hashicorp/consul-net-rpc/net/rpc"

	"github.com/hashicorp/consul/agent/connect"
	"github.com/hashicorp/consul/agent/consul/multilimiter"
	rpcRate "github.com/hashicorp/consul/agent/consul/rate"
	external "github.com/hashicorp/consul/agent/grpc-external"
	grpcmiddleware "github.com/hashicorp/consul/agent/grpc-middleware"
	"github.com/hashicorp/consul/agent/metadata"
	"github.com/hashicorp/consul/agent/rpc/middleware"
	"github.com/hashicorp/consul/agent/structs"
	"github.com/hashicorp/consul/agent/token"
	"github.com/hashicorp/consul/ipaddr"
	"github.com/hashicorp/consul/sdk/freeport"
	"github.com/hashicorp/consul/sdk/testutil"
	"github.com/hashicorp/consul/sdk/testutil/retry"
	"github.com/hashicorp/consul/testrpc"
	"github.com/hashicorp/consul/tlsutil"
	"github.com/hashicorp/consul/types"
)

const (
	TestDefaultInitialManagementToken = "d9f05e83-a7ae-47ce-839e-c0d53a68c00a"
)

// testTLSCertificates Generates a TLS CA and server key/cert and returns them
// in PEM encoded form.
func testTLSCertificates(serverName string) (cert string, key string, cacert string, err error) {
	signer, _, err := tlsutil.GeneratePrivateKey()
	if err != nil {
		return "", "", "", err
	}

	ca, _, err := tlsutil.GenerateCA(tlsutil.CAOpts{Signer: signer})
	if err != nil {
		return "", "", "", err
	}

	cert, privateKey, err := tlsutil.GenerateCert(tlsutil.CertOpts{
		Signer:      signer,
		CA:          ca,
		Name:        "Test Cert Name",
		Days:        365,
		DNSNames:    []string{serverName},
		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
	})
	if err != nil {
		return "", "", "", err
	}

	return cert, privateKey, ca, nil
}

func testServerACLConfig(c *Config) {
	c.PrimaryDatacenter = "dc1"
	c.ACLsEnabled = true
	c.ACLInitialManagementToken = TestDefaultInitialManagementToken
	c.ACLResolverSettings.ACLDefaultPolicy = "deny"
}

func configureTLS(config *Config) {
	config.TLSConfig.InternalRPC.CAFile = "../../test/ca/root.cer"
	config.TLSConfig.InternalRPC.CertFile = "../../test/key/ourdomain.cer"
	config.TLSConfig.InternalRPC.KeyFile = "../../test/key/ourdomain.key"
}

var id int64

func uniqueNodeName(name string) string {
	name = strings.ReplaceAll(name, "/", "_")
	return fmt.Sprintf("%s-node-%d", name, atomic.AddInt64(&id, 1))
}

// This will find the leader of a list of servers and verify that leader establishment has completed
func waitForLeaderEstablishment(t *testing.T, servers ...*Server) {
	t.Helper()
	retry.Run(t, func(r *retry.R) {
		hasLeader := false
		for _, srv := range servers {
			if srv.IsLeader() {
				hasLeader = true
				require.True(r, srv.isReadyForConsistentReads(), "Leader %s hasn't finished establishing leadership yet", srv.config.NodeName)
			}
		}
		require.True(r, hasLeader, "Cluster has not elected a leader yet")
	})
}

func testServerConfig(t *testing.T) (string, *Config) {
	dir := testutil.TempDir(t, "consul")
	config := DefaultConfig()

	ports := freeport.GetN(t, 4) // {server, serf_lan, serf_wan, grpc}
	config.NodeName = uniqueNodeName(t.Name())
	config.Bootstrap = true
	config.Datacenter = "dc1"
	config.PrimaryDatacenter = "dc1"
	config.DataDir = dir

	// bind the rpc server to a random port. config.RPCAdvertise will be
	// set to the listen address unless it was set in the configuration.
	// In that case get the address from srv.Listener.Addr().
	config.RPCAddr = &net.TCPAddr{IP: []byte{127, 0, 0, 1}, Port: ports[0]}

	nodeID, err := uuid.GenerateUUID()
	if err != nil {
		t.Fatal(err)
	}
	config.NodeID = types.NodeID(nodeID)

	// set the memberlist bind port to 0 to bind to a random port.
	// memberlist will update the value of BindPort after bind
	// to the actual value.
	config.SerfLANConfig.MemberlistConfig.BindAddr = "127.0.0.1"
	config.SerfLANConfig.MemberlistConfig.BindPort = ports[1]
	config.SerfLANConfig.MemberlistConfig.AdvertisePort = ports[1]
	config.SerfLANConfig.MemberlistConfig.SuspicionMult = 2
	config.SerfLANConfig.MemberlistConfig.ProbeTimeout = 50 * time.Millisecond
	config.SerfLANConfig.MemberlistConfig.ProbeInterval = 100 * time.Millisecond
	config.SerfLANConfig.MemberlistConfig.GossipInterval = 100 * time.Millisecond
	config.SerfLANConfig.MemberlistConfig.DeadNodeReclaimTime = 100 * time.Millisecond

	config.SerfWANConfig.MemberlistConfig.BindAddr = "127.0.0.1"
	config.SerfWANConfig.MemberlistConfig.BindPort = ports[2]
	config.SerfWANConfig.MemberlistConfig.AdvertisePort = ports[2]
	config.SerfWANConfig.MemberlistConfig.SuspicionMult = 2
	config.SerfWANConfig.MemberlistConfig.ProbeTimeout = 50 * time.Millisecond
	config.SerfWANConfig.MemberlistConfig.ProbeInterval = 100 * time.Millisecond
	config.SerfWANConfig.MemberlistConfig.GossipInterval = 100 * time.Millisecond
	config.SerfWANConfig.MemberlistConfig.DeadNodeReclaimTime = 100 * time.Millisecond

	config.RaftConfig.LeaderLeaseTimeout = 100 * time.Millisecond
	config.RaftConfig.HeartbeatTimeout = 200 * time.Millisecond
	config.RaftConfig.ElectionTimeout = 200 * time.Millisecond

	config.ReconcileInterval = 300 * time.Millisecond

	config.AutopilotConfig.ServerStabilizationTime = 100 * time.Millisecond
	config.ServerHealthInterval = 50 * time.Millisecond
	config.AutopilotInterval = 100 * time.Millisecond

	config.CoordinateUpdatePeriod = 100 * time.Millisecond
	config.LeaveDrainTime = 1 * time.Millisecond

	// TODO (slackpad) - We should be able to run all tests w/o this, but it
	// looks like several depend on it.
	config.RPCHoldTimeout = 10 * time.Second

	config.GRPCPort = ports[3]

	config.ConnectEnabled = true
	config.CAConfig = &structs.CAConfiguration{
		ClusterID: connect.TestClusterID,
		Provider:  structs.ConsulCAProvider,
		Config: map[string]interface{}{
			"PrivateKey":          "",
			"RootCert":            "",
			"LeafCertTTL":         "72h",
			"IntermediateCertTTL": "288h",
		},
	}
	config.PeeringEnabled = true
	return dir, config
}

// Deprecated: use testServerWithConfig instead. It does the same thing and more.
func testServer(t *testing.T) (string, *Server) {
	return testServerWithConfig(t)
}

// Deprecated: use testServerWithConfig
func testServerDC(t *testing.T, dc string) (string, *Server) {
	return testServerWithConfig(t, func(c *Config) {
		c.Datacenter = dc
		c.Bootstrap = true
	})
}

// Deprecated: use testServerWithConfig
func testServerDCBootstrap(t *testing.T, dc string, bootstrap bool) (string, *Server) {
	return testServerWithConfig(t, func(c *Config) {
		c.Datacenter = dc
		c.PrimaryDatacenter = dc
		c.Bootstrap = bootstrap
	})
}

// Deprecated: use testServerWithConfig
func testServerDCExpect(t *testing.T, dc string, expect int) (string, *Server) {
	return testServerWithConfig(t, func(c *Config) {
		c.Datacenter = dc
		c.Bootstrap = false
		c.BootstrapExpect = expect
	})
}

func testServerWithConfig(t *testing.T, configOpts ...func(*Config)) (string, *Server) {
	var dir string
	var srv *Server

	var config *Config
	var deps Deps
	// Retry added to avoid cases where bind addr is already in use
	retry.RunWith(retry.ThreeTimes(), t, func(r *retry.R) {
		dir, config = testServerConfig(t)
		for _, fn := range configOpts {
			fn(config)
		}

		// Apply config to copied fields because many tests only set the old
		// values.
		config.ACLResolverSettings.ACLsEnabled = config.ACLsEnabled
		config.ACLResolverSettings.NodeName = config.NodeName
		config.ACLResolverSettings.Datacenter = config.Datacenter
		config.ACLResolverSettings.EnterpriseMeta = *config.AgentEnterpriseMeta()

		var err error
		deps = newDefaultDeps(t, config)
		srv, err = newServerWithDeps(t, config, deps)
		if err != nil {
			r.Fatalf("err: %v", err)
		}
	})
	t.Cleanup(func() { srv.Shutdown() })

	for _, grpcPort := range []int{srv.config.GRPCPort, srv.config.GRPCTLSPort} {
		if grpcPort == 0 {
			continue
		}

		// Normally the gRPC server listener is created at the agent level and
		// passed down into the Server creation.
		ln, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", grpcPort))
		require.NoError(t, err)

		protocol := grpcmiddleware.ProtocolPlaintext
		if grpcPort == srv.config.GRPCTLSPort || deps.TLSConfigurator.GRPCServerUseTLS() {
			protocol = grpcmiddleware.ProtocolTLS
			// Set the internally managed server certificate. The cert manager is hooked to the Agent, so we need to bypass that here.
			if srv.config.PeeringEnabled && srv.config.ConnectEnabled {
				key, _ := srv.config.CAConfig.Config["PrivateKey"].(string)
				cert, _ := srv.config.CAConfig.Config["RootCert"].(string)
				if key != "" && cert != "" {
					ca := &structs.CARoot{
						SigningKey: key,
						RootCert:   cert,
					}
					require.NoError(t, deps.TLSConfigurator.UpdateAutoTLSCert(connect.TestServerLeaf(t, srv.config.Datacenter, ca)))
					deps.TLSConfigurator.UpdateAutoTLSPeeringServerName(connect.PeeringServerSAN("dc1", connect.TestTrustDomain))
				}
			}

		}
		ln = grpcmiddleware.LabelledListener{Listener: ln, Protocol: protocol}

		go func() {
			_ = srv.externalGRPCServer.Serve(ln)
		}()
		t.Cleanup(srv.externalGRPCServer.Stop)
	}

	return dir, srv
}

// cb is a function that can alter the test servers configuration prior to the server starting.
func testACLServerWithConfig(t *testing.T, cb func(*Config), initReplicationToken bool) (string, *Server, rpc.ClientCodec) {
	opts := []func(*Config){testServerACLConfig}
	if cb != nil {
		opts = append(opts, cb)
	}
	dir, srv := testServerWithConfig(t, opts...)

	if initReplicationToken {
		// setup some tokens here so we get less warnings in the logs
		srv.tokens.UpdateReplicationToken(TestDefaultInitialManagementToken, token.TokenSourceConfig)
	}

	codec := rpcClient(t, srv)
	return dir, srv, codec
}

func testGRPCIntegrationServer(t *testing.T, cb func(*Config)) (*Server, *grpc.ClientConn, rpc.ClientCodec) {
	_, srv, codec := testACLServerWithConfig(t, cb, false)

	grpcAddr := fmt.Sprintf("127.0.0.1:%d", srv.config.GRPCPort)
	//nolint:staticcheck
	conn, err := grpc.Dial(grpcAddr, grpc.WithInsecure())
	require.NoError(t, err)

	t.Cleanup(func() { _ = conn.Close() })

	return srv, conn, codec
}

func newServer(t *testing.T, c *Config) (*Server, error) {
	return newServerWithDeps(t, c, newDefaultDeps(t, c))
}

func newServerWithDeps(t *testing.T, c *Config, deps Deps) (*Server, error) {
	// chain server up notification
	oldNotify := c.NotifyListen
	up := make(chan struct{})
	c.NotifyListen = func() {
		close(up)
		if oldNotify != nil {
			oldNotify()
		}
	}
	grpcServer := external.NewServer(deps.Logger.Named("grpc.external"), nil, deps.TLSConfigurator, rpcRate.NullRequestLimitsHandler())
	srv, err := NewServer(c, deps, grpcServer, nil, deps.Logger)
	if err != nil {
		return nil, err
	}
	t.Cleanup(func() { srv.Shutdown() })

	// wait until after listen
	<-up

	// get the real address
	//
	// the server already sets the RPCAdvertise address
	// if it wasn't configured since it needs it for
	// some initialization
	//
	// todo(fs): setting RPCAddr should probably be guarded
	// todo(fs): but for now it is a shortcut to avoid fixing
	// todo(fs): tests which depend on that value. They should
	// todo(fs): just get the listener address instead.
	c.RPCAddr = srv.Listener.Addr().(*net.TCPAddr)
	return srv, nil
}

func TestServer_StartStop(t *testing.T) {
	t.Parallel()
	// Start up a server and then stop it.
	_, s1 := testServer(t)
	if err := s1.Shutdown(); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Shut down again, which should be idempotent.
	if err := s1.Shutdown(); err != nil {
		t.Fatalf("err: %v", err)
	}
}

func TestServer_fixupACLDatacenter(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	t.Parallel()

	_, s1 := testServerWithConfig(t, func(c *Config) {
		c.Datacenter = "aye"
		c.PrimaryDatacenter = "aye"
		c.ACLsEnabled = true
	})
	defer s1.Shutdown()

	_, s2 := testServerWithConfig(t, func(c *Config) {
		c.Datacenter = "bee"
		c.PrimaryDatacenter = "aye"
		c.ACLsEnabled = true
	})
	defer s2.Shutdown()

	// Try to join
	joinWAN(t, s2, s1)
	retry.Run(t, func(r *retry.R) {
		if got, want := len(s1.WANMembers()), 2; got != want {
			r.Fatalf("got %d s1 WAN members want %d", got, want)
		}
		if got, want := len(s2.WANMembers()), 2; got != want {
			r.Fatalf("got %d s2 WAN members want %d", got, want)
		}
	})

	testrpc.WaitForLeader(t, s1.RPC, "aye")
	testrpc.WaitForLeader(t, s2.RPC, "bee")

	require.Equal(t, "aye", s1.config.Datacenter)
	require.Equal(t, "aye", s1.config.PrimaryDatacenter)
	require.Equal(t, "aye", s1.config.PrimaryDatacenter)

	require.Equal(t, "bee", s2.config.Datacenter)
	require.Equal(t, "aye", s2.config.PrimaryDatacenter)
	require.Equal(t, "aye", s2.config.PrimaryDatacenter)
}

func TestServer_JoinLAN(t *testing.T) {
	t.Parallel()
	dir1, s1 := testServer(t)
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	dir2, s2 := testServer(t)
	defer os.RemoveAll(dir2)
	defer s2.Shutdown()

	// Try to join
	joinLAN(t, s2, s1)
	retry.Run(t, func(r *retry.R) {
		if got, want := len(s1.LANMembersInAgentPartition()), 2; got != want {
			r.Fatalf("got %d s1 LAN members want %d", got, want)
		}
		if got, want := len(s2.LANMembersInAgentPartition()), 2; got != want {
			r.Fatalf("got %d s2 LAN members want %d", got, want)
		}
	})
}

// TestServer_JoinLAN_SerfAllowedCIDRs test that IPs might be blocked with
// Serf.
//
// To run properly, this test requires to be able to bind and have access on
// 127.0.1.1 which is the case for most Linux machines and Windows, so Unit
// test will run in the CI.
//
// To run it on Mac OS, please run this command first, otherwise the test will
// be skipped: `sudo ifconfig lo0 alias 127.0.1.1 up`
func TestServer_JoinLAN_SerfAllowedCIDRs(t *testing.T) {
	t.Parallel()

	const targetAddr = "127.0.1.1"

	skipIfCannotBindToIP(t, targetAddr)

	dir1, s1 := testServerWithConfig(t, func(c *Config) {
		c.BootstrapExpect = 1
		lan, err := memberlist.ParseCIDRs([]string{"127.0.0.1/32"})
		require.NoError(t, err)
		c.SerfLANConfig.MemberlistConfig.CIDRsAllowed = lan
		wan, err := memberlist.ParseCIDRs([]string{"127.0.0.0/24", "::1/128"})
		require.NoError(t, err)
		c.SerfWANConfig.MemberlistConfig.CIDRsAllowed = wan
	})
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	dir2, a2 := testClientWithConfig(t, func(c *Config) {
		c.SerfLANConfig.MemberlistConfig.BindAddr = targetAddr
	})
	defer os.RemoveAll(dir2)
	defer a2.Shutdown()

	dir3, rs3 := testServerWithConfig(t, func(c *Config) {
		c.BootstrapExpect = 1
		c.Datacenter = "dc2"
	})
	defer os.RemoveAll(dir3)
	defer rs3.Shutdown()

	leaderAddr := joinAddrLAN(s1)
	if _, err := a2.JoinLAN([]string{leaderAddr}, nil); err != nil {
		t.Fatalf("Expected no error, had: %#v", err)
	}
	// Try to join
	joinWAN(t, rs3, s1)
	retry.Run(t, func(r *retry.R) {
		if got, want := len(s1.LANMembersInAgentPartition()), 1; got != want {
			// LAN is blocked, should be 1 only
			r.Fatalf("got %d s1 LAN members want %d", got, want)
		}
		if got, want := len(a2.LANMembersInAgentPartition()), 2; got != want {
			// LAN is blocked a2 can see s1, but not s1
			r.Fatalf("got %d a2 LAN members want %d", got, want)
		}
		if got, want := len(s1.WANMembers()), 2; got != want {
			r.Fatalf("got %d s1 WAN members want %d", got, want)
		}
		if got, want := len(rs3.WANMembers()), 2; got != want {
			r.Fatalf("got %d rs3 WAN members want %d", got, want)
		}
	})
}

// TestServer_JoinWAN_SerfAllowedCIDRs test that IPs might be
// blocked with Serf.
//
// To run properly, this test requires to be able to bind and have access on
// 127.0.1.1 which is the case for most Linux machines and Windows, so Unit
// test will run in the CI.
//
// To run it on Mac OS, please run this command first, otherwise the test will
// be skipped: `sudo ifconfig lo0 alias 127.0.1.1 up`
func TestServer_JoinWAN_SerfAllowedCIDRs(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	const targetAddr = "127.0.1.1"

	skipIfCannotBindToIP(t, targetAddr)

	wanCIDRs, err := memberlist.ParseCIDRs([]string{"127.0.0.1/32"})
	require.NoError(t, err)

	dir1, s1 := testServerWithConfig(t, func(c *Config) {
		c.Bootstrap = true
		c.BootstrapExpect = 1
		c.Datacenter = "dc1"
		c.SerfWANConfig.MemberlistConfig.CIDRsAllowed = wanCIDRs
	})
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	waitForLeaderEstablishment(t, s1)
	testrpc.WaitForLeader(t, s1.RPC, "dc1")

	dir2, s2 := testServerWithConfig(t, func(c *Config) {
		c.Bootstrap = true
		c.BootstrapExpect = 1
		c.PrimaryDatacenter = "dc1"
		c.Datacenter = "dc2"
		c.SerfWANConfig.MemberlistConfig.BindAddr = targetAddr
	})
	defer os.RemoveAll(dir2)
	defer s2.Shutdown()

	waitForLeaderEstablishment(t, s2)
	testrpc.WaitForLeader(t, s2.RPC, "dc2")

	// Joining should be fine
	joinWANWithNoMembershipChecks(t, s2, s1)

	// But membership is blocked if you go and take a peek on the server.
	t.Run("LAN membership should only show each other", func(t *testing.T) {
		require.Len(t, s1.LANMembersInAgentPartition(), 1)
		require.Len(t, s2.LANMembersInAgentPartition(), 1)
	})
	t.Run("WAN membership in the primary should not show the secondary", func(t *testing.T) {
		require.Len(t, s1.WANMembers(), 1)
	})
	t.Run("WAN membership in the secondary can show the primary", func(t *testing.T) {
		require.Len(t, s2.WANMembers(), 2)
	})
}

func skipIfCannotBindToIP(t *testing.T, ip string) {
	l, err := net.Listen("tcp", net.JoinHostPort(ip, "0"))
	if err != nil {
		t.Skipf("Cannot bind on %s, to run on Mac OS: `sudo ifconfig lo0 alias %s up`", ip, ip)
	}
	l.Close()
}

func TestServer_LANReap(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	t.Parallel()

	configureServer := func(c *Config) {
		c.SerfFloodInterval = 100 * time.Millisecond
		c.SerfLANConfig.ReconnectTimeout = 250 * time.Millisecond
		c.SerfLANConfig.TombstoneTimeout = 250 * time.Millisecond
		c.SerfLANConfig.ReapInterval = 300 * time.Millisecond
	}

	dir1, s1 := testServerWithConfig(t, func(c *Config) {
		c.Datacenter = "dc1"
		c.Bootstrap = true
		configureServer(c)
	})
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	dir2, s2 := testServerWithConfig(t, func(c *Config) {
		c.Datacenter = "dc1"
		c.Bootstrap = false
		configureServer(c)
	})
	defer os.RemoveAll(dir2)

	dir3, s3 := testServerWithConfig(t, func(c *Config) {
		c.Datacenter = "dc1"
		c.Bootstrap = false
		configureServer(c)
	})
	defer os.RemoveAll(dir3)
	defer s3.Shutdown()

	// Try to join
	joinLAN(t, s2, s1)
	joinLAN(t, s3, s1)

	testrpc.WaitForLeader(t, s1.RPC, "dc1")
	testrpc.WaitForLeader(t, s2.RPC, "dc1")
	testrpc.WaitForLeader(t, s3.RPC, "dc1")

	retry.Run(t, func(r *retry.R) {
		require.Len(r, s1.LANMembersInAgentPartition(), 3)
		require.Len(r, s2.LANMembersInAgentPartition(), 3)
		require.Len(r, s3.LANMembersInAgentPartition(), 3)
	})

	// Check the router has both
	retry.Run(t, func(r *retry.R) {
		require.Len(r, s1.serverLookup.Servers(), 3)
		require.Len(r, s2.serverLookup.Servers(), 3)
		require.Len(r, s3.serverLookup.Servers(), 3)
	})

	// shutdown the second dc
	s2.Shutdown()

	retry.Run(t, func(r *retry.R) {
		require.Len(r, s1.LANMembersInAgentPartition(), 2)
		servers := s1.serverLookup.Servers()
		require.Len(r, servers, 2)
		// require.Equal(r, s1.config.NodeName, servers[0].Name)
	})
}

func TestServer_JoinWAN(t *testing.T) {
	t.Parallel()
	dir1, s1 := testServer(t)
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	dir2, s2 := testServerDC(t, "dc2")
	defer os.RemoveAll(dir2)
	defer s2.Shutdown()

	// Try to join
	joinWAN(t, s2, s1)
	retry.Run(t, func(r *retry.R) {
		if got, want := len(s1.WANMembers()), 2; got != want {
			r.Fatalf("got %d s1 WAN members want %d", got, want)
		}
		if got, want := len(s2.WANMembers()), 2; got != want {
			r.Fatalf("got %d s2 WAN members want %d", got, want)
		}
	})

	// Check the router has both
	retry.Run(t, func(r *retry.R) {
		if got, want := len(s1.router.GetDatacenters()), 2; got != want {
			r.Fatalf("got %d routes want %d", got, want)
		}
		if got, want := len(s2.router.GetDatacenters()), 2; got != want {
			r.Fatalf("got %d datacenters want %d", got, want)
		}
	})
}

func TestServer_WANReap(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	t.Parallel()
	dir1, s1 := testServerWithConfig(t, func(c *Config) {
		c.Datacenter = "dc1"
		c.Bootstrap = true
		c.SerfFloodInterval = 100 * time.Millisecond
		c.SerfWANConfig.ReconnectTimeout = 250 * time.Millisecond
		c.SerfWANConfig.TombstoneTimeout = 250 * time.Millisecond
		c.SerfWANConfig.ReapInterval = 500 * time.Millisecond
	})
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	dir2, s2 := testServerDC(t, "dc2")
	defer os.RemoveAll(dir2)

	// Try to join
	joinWAN(t, s2, s1)
	retry.Run(t, func(r *retry.R) {
		require.Len(r, s1.WANMembers(), 2)
		require.Len(r, s2.WANMembers(), 2)
	})

	// Check the router has both
	retry.Run(t, func(r *retry.R) {
		require.Len(r, s1.router.GetDatacenters(), 2)
		require.Len(r, s2.router.GetDatacenters(), 2)
	})

	// shutdown the second dc
	s2.Shutdown()

	retry.Run(t, func(r *retry.R) {
		require.Len(r, s1.WANMembers(), 1)
		datacenters := s1.router.GetDatacenters()
		require.Len(r, datacenters, 1)
		require.Equal(r, "dc1", datacenters[0])
	})

}

func TestServer_JoinWAN_Flood(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	t.Parallel()
	// Set up two servers in a WAN.
	dir1, s1 := testServerDCBootstrap(t, "dc1", true)
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	dir2, s2 := testServerDCBootstrap(t, "dc2", true)
	defer os.RemoveAll(dir2)
	defer s2.Shutdown()

	joinWAN(t, s2, s1)

	for _, s := range []*Server{s1, s2} {
		retry.Run(t, func(r *retry.R) {
			if got, want := len(s.WANMembers()), 2; got != want {
				r.Fatalf("got %d WAN members want %d", got, want)
			}
		})
	}

	dir3, s3 := testServerDCBootstrap(t, "dc1", false)
	defer os.RemoveAll(dir3)
	defer s3.Shutdown()

	// Do just a LAN join for the new server and make sure it
	// shows up in the WAN.
	joinLAN(t, s3, s1)

	for _, s := range []*Server{s1, s2, s3} {
		retry.Run(t, func(r *retry.R) {
			if got, want := len(s.WANMembers()), 3; got != want {
				r.Fatalf("got %d WAN members for %s want %d", got, s.config.NodeName, want)
			}
		})
	}
}

// This is a mirror of a similar test in agent/agent_test.go
func TestServer_JoinWAN_viaMeshGateway(t *testing.T) {
	// if this test is failing because of expired certificates
	// use the procedure in test/CA-GENERATION.md
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	t.Parallel()

	port := freeport.GetOne(t)
	gwAddr := ipaddr.FormatAddressPort("127.0.0.1", port)

	dir1, s1 := testServerWithConfig(t, func(c *Config) {
		c.TLSConfig.Domain = "consul"
		c.NodeName = "bob"
		c.Datacenter = "dc1"
		c.PrimaryDatacenter = "dc1"
		c.Bootstrap = true
		// tls
		c.TLSConfig.InternalRPC.CAFile = "../../test/hostname/CertAuth.crt"
		c.TLSConfig.InternalRPC.CertFile = "../../test/hostname/Bob.crt"
		c.TLSConfig.InternalRPC.KeyFile = "../../test/hostname/Bob.key"
		c.TLSConfig.InternalRPC.VerifyIncoming = true
		c.TLSConfig.InternalRPC.VerifyOutgoing = true
		c.TLSConfig.InternalRPC.VerifyServerHostname = true
		// wanfed
		c.ConnectMeshGatewayWANFederationEnabled = true
	})
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	dir2, s2 := testServerWithConfig(t, func(c *Config) {
		c.TLSConfig.Domain = "consul"
		c.NodeName = "betty"
		c.Datacenter = "dc2"
		c.PrimaryDatacenter = "dc1"
		c.Bootstrap = true
		// tls
		c.TLSConfig.InternalRPC.CAFile = "../../test/hostname/CertAuth.crt"
		c.TLSConfig.InternalRPC.CertFile = "../../test/hostname/Betty.crt"
		c.TLSConfig.InternalRPC.KeyFile = "../../test/hostname/Betty.key"
		c.TLSConfig.InternalRPC.VerifyIncoming = true
		c.TLSConfig.InternalRPC.VerifyOutgoing = true
		c.TLSConfig.InternalRPC.VerifyServerHostname = true
		// wanfed
		c.ConnectMeshGatewayWANFederationEnabled = true
	})
	defer os.RemoveAll(dir2)
	defer s2.Shutdown()

	dir3, s3 := testServerWithConfig(t, func(c *Config) {
		c.TLSConfig.Domain = "consul"
		c.NodeName = "bonnie"
		c.Datacenter = "dc3"
		c.PrimaryDatacenter = "dc1"
		c.Bootstrap = true
		// tls
		c.TLSConfig.InternalRPC.CAFile = "../../test/hostname/CertAuth.crt"
		c.TLSConfig.InternalRPC.CertFile = "../../test/hostname/Bonnie.crt"
		c.TLSConfig.InternalRPC.KeyFile = "../../test/hostname/Bonnie.key"
		c.TLSConfig.InternalRPC.VerifyIncoming = true
		c.TLSConfig.InternalRPC.VerifyOutgoing = true
		c.TLSConfig.InternalRPC.VerifyServerHostname = true
		// wanfed
		c.ConnectMeshGatewayWANFederationEnabled = true
	})
	defer os.RemoveAll(dir3)
	defer s3.Shutdown()

	// We'll use the same gateway for all datacenters since it doesn't care.
	var p tcpproxy.Proxy
	p.AddSNIRoute(gwAddr, "bob.server.dc1.consul", tcpproxy.To(s1.config.RPCAddr.String()))
	p.AddSNIRoute(gwAddr, "betty.server.dc2.consul", tcpproxy.To(s2.config.RPCAddr.String()))
	p.AddSNIRoute(gwAddr, "bonnie.server.dc3.consul", tcpproxy.To(s3.config.RPCAddr.String()))
	p.AddStopACMESearch(gwAddr)
	require.NoError(t, p.Start())
	defer func() {
		p.Close()
		p.Wait()
	}()

	t.Logf("routing %s => %s", "bob.server.dc1.consul", s1.config.RPCAddr.String())
	t.Logf("routing %s => %s", "betty.server.dc2.consul", s2.config.RPCAddr.String())
	t.Logf("routing %s => %s", "bonnie.server.dc3.consul", s3.config.RPCAddr.String())

	// Register this into the catalog in dc1.
	{
		arg := structs.RegisterRequest{
			Datacenter: "dc1",
			Node:       "bob",
			Address:    "127.0.0.1",
			Service: &structs.NodeService{
				Kind:    structs.ServiceKindMeshGateway,
				ID:      "mesh-gateway",
				Service: "mesh-gateway",
				Meta:    map[string]string{structs.MetaWANFederationKey: "1"},
				Port:    port,
			},
		}

		var out struct{}
		require.NoError(t, s1.RPC(context.Background(), "Catalog.Register", &arg, &out))
	}

	// Wait for it to make it into the gateway locator.
	retry.Run(t, func(r *retry.R) {
		require.NotEmpty(r, s1.gatewayLocator.PickGateway("dc1"))
	})

	// Seed the secondaries with the address of the primary and wait for that to
	// be in their locators.
	s2.RefreshPrimaryGatewayFallbackAddresses([]string{gwAddr})
	retry.Run(t, func(r *retry.R) {
		require.NotEmpty(r, s2.gatewayLocator.PickGateway("dc1"))
	})
	s3.RefreshPrimaryGatewayFallbackAddresses([]string{gwAddr})
	retry.Run(t, func(r *retry.R) {
		require.NotEmpty(r, s3.gatewayLocator.PickGateway("dc1"))
	})

	// Try to join from secondary to primary. We can't use joinWAN() because we
	// are simulating proper bootstrapping and if ACLs were on we would have to
	// delay gateway registration in the secondary until after one directional
	// join. So this way we explicitly join secondary-to-primary as a standalone
	// operation and follow it up later with a full join.
	_, err := s2.JoinWAN([]string{joinAddrWAN(s1)})
	require.NoError(t, err)
	retry.Run(t, func(r *retry.R) {
		if got, want := len(s2.WANMembers()), 2; got != want {
			r.Fatalf("got %d s2 WAN members want %d", got, want)
		}
	})
	_, err = s3.JoinWAN([]string{joinAddrWAN(s1)})
	require.NoError(t, err)
	retry.Run(t, func(r *retry.R) {
		if got, want := len(s3.WANMembers()), 3; got != want {
			r.Fatalf("got %d s3 WAN members want %d", got, want)
		}
	})

	// Now we can register this into the catalog in dc2 and dc3.
	{
		arg := structs.RegisterRequest{
			Datacenter: "dc2",
			Node:       "betty",
			Address:    "127.0.0.1",
			Service: &structs.NodeService{
				Kind:    structs.ServiceKindMeshGateway,
				ID:      "mesh-gateway",
				Service: "mesh-gateway",
				Meta:    map[string]string{structs.MetaWANFederationKey: "1"},
				Port:    port,
			},
		}

		var out struct{}
		require.NoError(t, s2.RPC(context.Background(), "Catalog.Register", &arg, &out))
	}
	{
		arg := structs.RegisterRequest{
			Datacenter: "dc3",
			Node:       "bonnie",
			Address:    "127.0.0.1",
			Service: &structs.NodeService{
				Kind:    structs.ServiceKindMeshGateway,
				ID:      "mesh-gateway",
				Service: "mesh-gateway",
				Meta:    map[string]string{structs.MetaWANFederationKey: "1"},
				Port:    port,
			},
		}

		var out struct{}
		require.NoError(t, s3.RPC(context.Background(), "Catalog.Register", &arg, &out))
	}

	// Wait for it to make it into the gateway locator in dc2 and then for
	// AE to carry it back to the primary
	retry.Run(t, func(r *retry.R) {
		require.NotEmpty(r, s3.gatewayLocator.PickGateway("dc2"))
		require.NotEmpty(r, s2.gatewayLocator.PickGateway("dc2"))
		require.NotEmpty(r, s1.gatewayLocator.PickGateway("dc2"))

		require.NotEmpty(r, s3.gatewayLocator.PickGateway("dc3"))
		require.NotEmpty(r, s2.gatewayLocator.PickGateway("dc3"))
		require.NotEmpty(r, s1.gatewayLocator.PickGateway("dc3"))
	})

	// Try to join again using the standard verification method now that
	// all of the plumbing is in place.
	joinWAN(t, s2, s1)
	retry.Run(t, func(r *retry.R) {
		if got, want := len(s1.WANMembers()), 3; got != want {
			r.Fatalf("got %d s1 WAN members want %d", got, want)
		}
		if got, want := len(s2.WANMembers()), 3; got != want {
			r.Fatalf("got %d s2 WAN members want %d", got, want)
		}
	})

	// Check the router has all of them
	retry.Run(t, func(r *retry.R) {
		if got, want := len(s1.router.GetDatacenters()), 3; got != want {
			r.Fatalf("got %d routes want %d", got, want)
		}
		if got, want := len(s2.router.GetDatacenters()), 3; got != want {
			r.Fatalf("got %d datacenters want %d", got, want)
		}
		if got, want := len(s3.router.GetDatacenters()), 3; got != want {
			r.Fatalf("got %d datacenters want %d", got, want)
		}
	})

	// Ensure we can do some trivial RPC in all directions.
	servers := map[string]*Server{"dc1": s1, "dc2": s2, "dc3": s3}
	names := map[string]string{"dc1": "bob", "dc2": "betty", "dc3": "bonnie"}
	for _, srcDC := range []string{"dc1", "dc2", "dc3"} {
		srv := servers[srcDC]
		for _, dstDC := range []string{"dc1", "dc2", "dc3"} {
			if srcDC == dstDC {
				continue
			}
			t.Run(srcDC+" to "+dstDC, func(t *testing.T) {
				arg := structs.DCSpecificRequest{
					Datacenter: dstDC,
				}
				var out structs.IndexedNodes
				require.NoError(t, srv.RPC(context.Background(), "Catalog.ListNodes", &arg, &out))
				require.Len(t, out.Nodes, 1)
				node := out.Nodes[0]
				require.Equal(t, dstDC, node.Datacenter)
				require.Equal(t, names[dstDC], node.Node)
			})
		}
	}
}

func TestServer_JoinSeparateLanAndWanAddresses(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	t.Parallel()
	dir1, s1 := testServerWithConfig(t, func(c *Config) {
		c.NodeName = t.Name() + "-s1"
		c.Datacenter = "dc1"
		c.Bootstrap = true
		c.SerfFloodInterval = 100 * time.Millisecond
	})
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	s2Name := t.Name() + "-s2"
	dir2, s2 := testServerWithConfig(t, func(c *Config) {
		c.NodeName = s2Name
		c.Datacenter = "dc2"
		c.Bootstrap = false
		// This wan address will be expected to be seen on s1
		c.SerfWANConfig.MemberlistConfig.AdvertiseAddr = "127.0.0.2"
		// This lan address will be expected to be seen on s3
		c.SerfLANConfig.MemberlistConfig.AdvertiseAddr = "127.0.0.3"
		c.SerfFloodInterval = 100 * time.Millisecond
	})

	defer os.RemoveAll(dir2)
	defer s2.Shutdown()

	dir3, s3 := testServerWithConfig(t, func(c *Config) {
		c.NodeName = t.Name() + "-s3"
		c.Datacenter = "dc2"
		c.Bootstrap = true
		c.SerfFloodInterval = 100 * time.Millisecond
	})
	defer os.RemoveAll(dir3)
	defer s3.Shutdown()

	// Join s2 to s1 on wan
	joinWAN(t, s2, s1)

	// Join s3 to s2 on lan
	joinLAN(t, s3, s2)

	// We rely on flood joining to fill across the LAN, so we expect s3 to
	// show up on the WAN as well, even though it's not explicitly joined.
	retry.Run(t, func(r *retry.R) {
		if got, want := len(s1.WANMembers()), 3; got != want {
			r.Fatalf("got %d s1 WAN members want %d", got, want)
		}
		if got, want := len(s2.WANMembers()), 3; got != want {
			r.Fatalf("got %d s2 WAN members want %d", got, want)
		}
		if got, want := len(s2.LANMembersInAgentPartition()), 2; got != want {
			r.Fatalf("got %d s2 LAN members want %d", got, want)
		}
		if got, want := len(s3.LANMembersInAgentPartition()), 2; got != want {
			r.Fatalf("got %d s3 LAN members want %d", got, want)
		}
	})

	// Check the router has both
	retry.Run(t, func(r *retry.R) {
		if len(s1.router.GetDatacenters()) != 2 {
			r.Fatalf("remote consul missing")
		}
		if len(s2.router.GetDatacenters()) != 2 {
			r.Fatalf("remote consul missing")
		}
		if len(s2.serverLookup.Servers()) != 2 {
			r.Fatalf("local consul fellow s3 for s2 missing")
		}
	})

	// Get and check the wan address of s2 from s1
	var s2WanAddr string
	for _, member := range s1.WANMembers() {
		if member.Name == s2Name+".dc2" {
			s2WanAddr = member.Addr.String()
		}
	}
	if s2WanAddr != "127.0.0.2" {
		t.Fatalf("s1 sees s2 on a wrong address: %s, expecting: %s", s2WanAddr, "127.0.0.2")
	}

	// Get and check the lan address of s2 from s3
	var s2LanAddr string
	for _, lanmember := range s3.LANMembersInAgentPartition() {
		if lanmember.Name == s2Name {
			s2LanAddr = lanmember.Addr.String()
		}
	}
	if s2LanAddr != "127.0.0.3" {
		t.Fatalf("s3 sees s2 on a wrong address: %s, expecting: %s", s2LanAddr, "127.0.0.3")
	}
}

func TestServer_LeaveLeader(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	t.Parallel()
	dir1, s1 := testServer(t)
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	dir2, s2 := testServerDCBootstrap(t, "dc1", false)
	defer os.RemoveAll(dir2)
	defer s2.Shutdown()

	dir3, s3 := testServerDCBootstrap(t, "dc1", false)
	defer os.RemoveAll(dir3)
	defer s3.Shutdown()

	testrpc.WaitForLeader(t, s1.RPC, "dc1")
	joinLAN(t, s2, s1)
	joinLAN(t, s3, s1)
	retry.Run(t, func(r *retry.R) {
		r.Check(wantPeers(s1, 3))
		r.Check(wantPeers(s2, 3))
		r.Check(wantPeers(s3, 3))
	})
	// Issue a leave to the leader
	var leader *Server
	switch {
	case s1.IsLeader():
		leader = s1
	case s2.IsLeader():
		leader = s2
	case s3.IsLeader():
		leader = s3
	default:
		t.Fatal("no leader")
	}
	if err := leader.Leave(); err != nil {
		t.Fatal("leave failed: ", err)
	}

	// Should lose a peer
	retry.Run(t, func(r *retry.R) {
		r.Check(wantPeers(s1, 2))
		r.Check(wantPeers(s2, 2))
		r.Check(wantPeers(s3, 2))
	})
}

func TestServer_Leave(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	t.Parallel()
	dir1, s1 := testServer(t)
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	// Second server not in bootstrap mode
	dir2, s2 := testServerDCBootstrap(t, "dc1", false)
	defer os.RemoveAll(dir2)
	defer s2.Shutdown()

	// Try to join
	joinLAN(t, s2, s1)

	testrpc.WaitForLeader(t, s1.RPC, "dc1")
	testrpc.WaitForLeader(t, s2.RPC, "dc1")

	// Issue a leave to the non-leader
	var nonleader *Server
	switch {
	case s1.IsLeader():
		nonleader = s2
	case s2.IsLeader():
		nonleader = s1
	default:
		t.Fatal("no leader")
	}
	if err := nonleader.Leave(); err != nil {
		t.Fatal("leave failed: ", err)
	}

	// Should lose a peer
	retry.Run(t, func(r *retry.R) {
		r.Check(wantPeers(s1, 1))
		r.Check(wantPeers(s2, 1))
	})
}

func TestServer_RPC(t *testing.T) {
	t.Parallel()
	dir1, s1 := testServer(t)
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	var out struct{}
	if err := s1.RPC(context.Background(), "Status.Ping", struct{}{}, &out); err != nil {
		t.Fatalf("err: %v", err)
	}
}

// TestServer_RPC_MetricsIntercept_Off proves that we can turn off net/rpc interceptors all together.
func TestServer_RPC_MetricsIntercept_Off(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	storage := &sync.Map{} // string -> float32
	keyMakingFunc := func(key []string, labels []metrics.Label) string {
		allKey := strings.Join(key, "+")

		for _, label := range labels {
			if label.Name == "method" {
				allKey = allKey + "+" + label.Value
			}
		}

		return allKey
	}

	simpleRecorderFunc := func(key []string, val float32, labels []metrics.Label) {
		storage.Store(keyMakingFunc(key, labels), val)
	}

	t.Run("test no net/rpc interceptor metric with nil func", func(t *testing.T) {
		_, conf := testServerConfig(t)
		deps := newDefaultDeps(t, conf)

		// "disable" metrics net/rpc interceptor
		deps.GetNetRPCInterceptorFunc = nil
		// "hijack" the rpc recorder for asserts;
		// note that there will be "internal" net/rpc calls made
		// that will still show up; those don't go thru the net/rpc interceptor;
		// see consul.agent.rpc.middleware.RPCTypeInternal for context
		deps.NewRequestRecorderFunc = func(logger hclog.Logger, isLeader func() bool, localDC string) *middleware.RequestRecorder {
			// for the purposes of this test, we don't need isLeader or localDC
			return &middleware.RequestRecorder{
				Logger:       hclog.NewInterceptLogger(&hclog.LoggerOptions{}),
				RecorderFunc: simpleRecorderFunc,
			}
		}

		s1, err := NewServer(conf, deps, grpc.NewServer(), nil, deps.Logger)
		if err != nil {
			t.Fatalf("err: %v", err)
		}
		t.Cleanup(func() { s1.Shutdown() })

		var out struct{}
		if err := s1.RPC(context.Background(), "Status.Ping", struct{}{}, &out); err != nil {
			t.Fatalf("err: %v", err)
		}

		key := keyMakingFunc(middleware.OneTwelveRPCSummary[0].Name, []metrics.Label{{Name: "method", Value: "Status.Ping"}})

		if _, ok := storage.Load(key); ok {
			t.Fatalf("Did not expect to find key %s in the metrics log, ", key)
		}
	})

	t.Run("test no net/rpc interceptor metric with func that gives nil", func(t *testing.T) {
		_, conf := testServerConfig(t)
		deps := newDefaultDeps(t, conf)

		// "hijack" the rpc recorder for asserts;
		// note that there will be "internal" net/rpc calls made
		// that will still show up; those don't go thru the net/rpc interceptor;
		// see consul.agent.rpc.middleware.RPCTypeInternal for context
		deps.NewRequestRecorderFunc = func(logger hclog.Logger, isLeader func() bool, localDC string) *middleware.RequestRecorder {
			// for the purposes of this test, we don't need isLeader or localDC
			return &middleware.RequestRecorder{
				Logger:       hclog.NewInterceptLogger(&hclog.LoggerOptions{}),
				RecorderFunc: simpleRecorderFunc,
			}
		}

		deps.GetNetRPCInterceptorFunc = func(recorder *middleware.RequestRecorder) rpc.ServerServiceCallInterceptor {
			return nil
		}

		s2, err := NewServer(conf, deps, grpc.NewServer(), nil, deps.Logger)
		if err != nil {
			t.Fatalf("err: %v", err)
		}
		t.Cleanup(func() { s2.Shutdown() })
		if err != nil {
			t.Fatalf("err: %v", err)
		}

		var out struct{}
		if err := s2.RPC(context.Background(), "Status.Ping", struct{}{}, &out); err != nil {
			t.Fatalf("err: %v", err)
		}

		key := keyMakingFunc(middleware.OneTwelveRPCSummary[0].Name, []metrics.Label{{Name: "method", Value: "Status.Ping"}})

		if _, ok := storage.Load(key); ok {
			t.Fatalf("Did not expect to find key %s in the metrics log, ", key)
		}
	})
}

// TestServer_RPC_RequestRecorder proves that we cannot make a server without a valid RequestRecorder provider func
// or a non nil RequestRecorder.
func TestServer_RPC_RequestRecorder(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	t.Run("test nil func provider", func(t *testing.T) {
		_, conf := testServerConfig(t)
		deps := newDefaultDeps(t, conf)
		deps.NewRequestRecorderFunc = nil

		s1, err := NewServer(conf, deps, grpc.NewServer(), nil, deps.Logger)

		require.Error(t, err, "need err when provider func is nil")
		require.Equal(t, err.Error(), "cannot initialize server without an RPC request recorder provider")

		t.Cleanup(func() {
			if s1 != nil {
				s1.Shutdown()
			}
		})
	})

	t.Run("test nil RequestRecorder", func(t *testing.T) {
		_, conf := testServerConfig(t)
		deps := newDefaultDeps(t, conf)
		deps.NewRequestRecorderFunc = func(logger hclog.Logger, isLeader func() bool, localDC string) *middleware.RequestRecorder {
			return nil
		}

		s2, err := NewServer(conf, deps, grpc.NewServer(), nil, deps.Logger)

		require.Error(t, err, "need err when RequestRecorder is nil")
		require.Equal(t, err.Error(), "cannot initialize server with a nil RPC request recorder")

		t.Cleanup(func() {
			if s2 != nil {
				s2.Shutdown()
			}
		})
	})
}

// TestServer_RPC_MetricsIntercept mocks a request recorder and asserts that RPC calls are observed.
func TestServer_RPC_MetricsIntercept(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	_, conf := testServerConfig(t)
	deps := newDefaultDeps(t, conf)

	// The method used to record metric observations here is similar to that used in
	// interceptors_test.go.
	storage := &sync.Map{} // string -> float32
	keyMakingFunc := func(key []string, labels []metrics.Label) string {
		allKey := strings.Join(key, "+")

		for _, label := range labels {
			allKey = allKey + "+" + label.Value
		}

		return allKey
	}

	simpleRecorderFunc := func(key []string, val float32, labels []metrics.Label) {
		storage.Store(keyMakingFunc(key, labels), val)
	}
	deps.NewRequestRecorderFunc = func(logger hclog.Logger, isLeader func() bool, localDC string) *middleware.RequestRecorder {
		// for the purposes of this test, we don't need isLeader or localDC
		return &middleware.RequestRecorder{
			Logger:       hclog.NewInterceptLogger(&hclog.LoggerOptions{}),
			RecorderFunc: simpleRecorderFunc,
		}
	}

	deps.GetNetRPCInterceptorFunc = func(recorder *middleware.RequestRecorder) rpc.ServerServiceCallInterceptor {
		return func(reqServiceMethod string, argv, replyv reflect.Value, handler func() error) {
			reqStart := time.Now()

			err := handler()

			recorder.Record(reqServiceMethod, "test", reqStart, argv.Interface(), err != nil)
		}
	}

	s, err := newServerWithDeps(t, conf, deps)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	defer s.Shutdown()
	testrpc.WaitForTestAgent(t, s.RPC, "dc1")

	// asserts
	t.Run("test happy path for metrics interceptor", func(t *testing.T) {
		var out struct{}
		if err := s.RPC(context.Background(), "Status.Ping", struct{}{}, &out); err != nil {
			t.Fatalf("err: %v", err)
		}

		expectedLabels := []metrics.Label{
			{Name: "method", Value: "Status.Ping"},
			{Name: "errored", Value: "false"},
			{Name: "request_type", Value: "read"},
			{Name: "rpc_type", Value: "test"},
			{Name: "server_role", Value: "unreported"},
		}

		key := keyMakingFunc(middleware.OneTwelveRPCSummary[0].Name, expectedLabels)

		if _, ok := storage.Load(key); !ok {
			// the compound key will look like: "rpc+server+call+Status.Ping+false+read+test+unreported"
			t.Fatalf("Did not find key %s in the metrics log, ", key)
		}
	})
}

func TestServer_JoinLAN_TLS(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	t.Parallel()
	_, conf1 := testServerConfig(t)
	conf1.TLSConfig.InternalRPC.VerifyIncoming = true
	conf1.TLSConfig.InternalRPC.VerifyOutgoing = true
	configureTLS(conf1)
	s1, err := newServer(t, conf1)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	defer s1.Shutdown()
	testrpc.WaitForTestAgent(t, s1.RPC, "dc1")

	_, conf2 := testServerConfig(t)
	conf2.Bootstrap = false
	conf2.TLSConfig.InternalRPC.VerifyIncoming = true
	conf2.TLSConfig.InternalRPC.VerifyOutgoing = true
	configureTLS(conf2)
	s2, err := newServer(t, conf2)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	defer s2.Shutdown()

	// Try to join
	joinLAN(t, s2, s1)
	testrpc.WaitForTestAgent(t, s2.RPC, "dc1")

	// Verify Raft has established a peer
	retry.Run(t, func(r *retry.R) {
		r.Check(wantRaft([]*Server{s1, s2}))
	})
}

func TestServer_Expect(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	// All test servers should be in expect=3 mode, except for the 3rd one,
	// but one with expect=0 can cause a bootstrap to occur from the other
	// servers as currently implemented.
	dir1, s1 := testServerDCExpect(t, "dc1", 3)
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	dir2, s2 := testServerDCExpect(t, "dc1", 3)
	defer os.RemoveAll(dir2)
	defer s2.Shutdown()

	dir3, s3 := testServerDCExpect(t, "dc1", 0)
	defer os.RemoveAll(dir3)
	defer s3.Shutdown()

	dir4, s4 := testServerDCExpect(t, "dc1", 3)
	defer os.RemoveAll(dir4)
	defer s4.Shutdown()

	// Join the first two servers.
	joinLAN(t, s2, s1)

	// Should have no peers yet since the bootstrap didn't occur.
	retry.Run(t, func(r *retry.R) {
		r.Check(wantPeers(s1, 0))
		r.Check(wantPeers(s2, 0))
	})

	// Join the third node.
	joinLAN(t, s3, s1)

	// Now we have three servers so we should bootstrap.
	retry.Run(t, func(r *retry.R) {
		r.Check(wantPeers(s1, 3))
		r.Check(wantPeers(s2, 3))
		r.Check(wantPeers(s3, 3))
	})

	// Join the fourth node.
	joinLAN(t, s4, s1)

	// Wait for the new server to see itself added to the cluster.
	retry.Run(t, func(r *retry.R) {
		r.Check(wantRaft([]*Server{s1, s2, s3, s4}))
	})
}

// Should not trigger bootstrap and new election when s3 joins, since cluster exists
func TestServer_AvoidReBootstrap(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	dir1, s1 := testServerDCExpect(t, "dc1", 2)
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	dir2, s2 := testServerDCExpect(t, "dc1", 0)
	defer os.RemoveAll(dir2)
	defer s2.Shutdown()

	dir3, s3 := testServerDCExpect(t, "dc1", 2)
	defer os.RemoveAll(dir3)
	defer s3.Shutdown()

	// Join the first two servers
	joinLAN(t, s2, s1)

	// Make sure a leader is elected, grab the current term and then add in
	// the third server.
	testrpc.WaitForLeader(t, s1.RPC, "dc1")
	termBefore := s1.raft.Stats()["last_log_term"]
	joinLAN(t, s3, s1)

	// Wait for the new server to see itself added to the cluster.
	retry.Run(t, func(r *retry.R) {
		r.Check(wantRaft([]*Server{s1, s2, s3}))
	})

	// Make sure there's still a leader and that the term didn't change,
	// so we know an election didn't occur.
	testrpc.WaitForLeader(t, s1.RPC, "dc1")
	termAfter := s1.raft.Stats()["last_log_term"]
	if termAfter != termBefore {
		t.Fatalf("looks like an election took place")
	}
}

func TestServer_Expect_NonVoters(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	t.Parallel()
	dir1, s1 := testServerWithConfig(t, func(c *Config) {
		c.Bootstrap = false
		c.BootstrapExpect = 2
		c.ReadReplica = true
	})
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	dir2, s2 := testServerDCExpect(t, "dc1", 2)
	defer os.RemoveAll(dir2)
	defer s2.Shutdown()

	dir3, s3 := testServerDCExpect(t, "dc1", 2)
	defer os.RemoveAll(dir3)
	defer s3.Shutdown()

	// Join the first two servers.
	joinLAN(t, s2, s1)

	// Should have no peers yet since the bootstrap didn't occur.
	retry.Run(t, func(r *retry.R) {
		r.Check(wantPeers(s1, 0))
		r.Check(wantPeers(s2, 0))
	})

	// Join the third node.
	joinLAN(t, s3, s1)

	// Now we have three servers so we should bootstrap.
	retry.Run(t, func(r *retry.R) {
		r.Check(wantPeers(s1, 2))
		r.Check(wantPeers(s2, 2))
		r.Check(wantPeers(s3, 2))
	})

	// Make sure a leader is elected
	testrpc.WaitForLeader(t, s1.RPC, "dc1")
	retry.Run(t, func(r *retry.R) {
		r.Check(wantRaft([]*Server{s1, s2, s3}))
	})
}

func TestServer_BadExpect(t *testing.T) {
	t.Parallel()
	// this one is in expect=3 mode
	dir1, s1 := testServerDCExpect(t, "dc1", 3)
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	// this one is in expect=2 mode
	dir2, s2 := testServerDCExpect(t, "dc1", 2)
	defer os.RemoveAll(dir2)
	defer s2.Shutdown()

	// and this one is in expect=3 mode
	dir3, s3 := testServerDCExpect(t, "dc1", 3)
	defer os.RemoveAll(dir3)
	defer s3.Shutdown()

	// Try to join
	joinLAN(t, s2, s1)

	// should have no peers yet
	retry.Run(t, func(r *retry.R) {
		r.Check(wantPeers(s1, 0))
		r.Check(wantPeers(s2, 0))
	})

	// join the third node
	joinLAN(t, s3, s1)

	// should still have no peers (because s2 is in expect=2 mode)
	retry.Run(t, func(r *retry.R) {
		r.Check(wantPeers(s1, 0))
		r.Check(wantPeers(s2, 0))
		r.Check(wantPeers(s3, 0))
	})
}

type fakeGlobalResp struct{}

func (r *fakeGlobalResp) Add(interface{}) {
}

func (r *fakeGlobalResp) New() interface{} {
	return struct{}{}
}

func TestServer_keyringRPCs(t *testing.T) {
	t.Parallel()
	dir1, s1 := testServerDC(t, "dc1")
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()
	retry.Run(t, func(r *retry.R) {
		if len(s1.router.GetDatacenters()) != 1 {
			r.Fatal(nil)
		}
	})

	// Check that an error from a remote DC is returned
	_, err := s1.keyringRPCs("Bad.Method", nil, []string{s1.config.Datacenter})
	if err == nil {
		t.Fatalf("should have errored")
	}
	if !strings.Contains(err.Error(), "Bad.Method") {
		t.Fatalf("unexpected error: %s", err)
	}
}

func testVerifyRPC(s1, s2 *Server, t *testing.T) (bool, error) {
	joinLAN(t, s1, s2)
	retry.Run(t, func(r *retry.R) {
		r.Check(wantRaft([]*Server{s1, s2}))
	})

	// Have s2 make an RPC call to s1
	var leader *metadata.Server
	for _, server := range s2.serverLookup.Servers() {
		if server.Name == s1.config.NodeName {
			leader = server
		}
	}
	if leader == nil {
		t.Fatal("no leader")
	}
	return s2.connPool.Ping(leader.Datacenter, leader.ShortName, leader.Addr)
}

func TestServer_TLSToNoTLS(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	t.Parallel()
	// Set up a server with no TLS configured
	dir1, s1 := testServer(t)
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	testrpc.WaitForLeader(t, s1.RPC, "dc1")

	// Add a second server with TLS configured
	dir2, s2 := testServerWithConfig(t, func(c *Config) {
		c.Bootstrap = false
		c.TLSConfig.InternalRPC.CAFile = "../../test/client_certs/rootca.crt"
		c.TLSConfig.InternalRPC.CertFile = "../../test/client_certs/server.crt"
		c.TLSConfig.InternalRPC.KeyFile = "../../test/client_certs/server.key"
	})
	defer os.RemoveAll(dir2)
	defer s2.Shutdown()

	success, err := testVerifyRPC(s1, s2, t)
	if err != nil {
		t.Fatal(err)
	}
	if !success {
		t.Fatalf("bad: %v", success)
	}
}

func TestServer_TLSForceOutgoingToNoTLS(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	t.Parallel()
	// Set up a server with no TLS configured
	dir1, s1 := testServer(t)
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	testrpc.WaitForLeader(t, s1.RPC, "dc1")

	// Add a second server with TLS and VerifyOutgoing set
	dir2, s2 := testServerWithConfig(t, func(c *Config) {
		c.Bootstrap = false
		c.TLSConfig.InternalRPC.CAFile = "../../test/client_certs/rootca.crt"
		c.TLSConfig.InternalRPC.CertFile = "../../test/client_certs/server.crt"
		c.TLSConfig.InternalRPC.KeyFile = "../../test/client_certs/server.key"
		c.TLSConfig.InternalRPC.VerifyOutgoing = true
	})
	defer os.RemoveAll(dir2)
	defer s2.Shutdown()

	_, err := testVerifyRPC(s1, s2, t)
	if err == nil || !strings.Contains(err.Error(), "remote error: tls") {
		t.Fatalf("should fail")
	}
}

func TestServer_TLSToFullVerify(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	t.Parallel()
	// Set up a server with TLS and VerifyIncoming set
	dir1, s1 := testServerWithConfig(t, func(c *Config) {
		c.TLSConfig.InternalRPC.CAFile = "../../test/client_certs/rootca.crt"
		c.TLSConfig.InternalRPC.CertFile = "../../test/client_certs/server.crt"
		c.TLSConfig.InternalRPC.KeyFile = "../../test/client_certs/server.key"
		c.TLSConfig.InternalRPC.VerifyOutgoing = true
	})
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	testrpc.WaitForLeader(t, s1.RPC, "dc1")

	// Add a second server with TLS configured
	dir2, s2 := testServerWithConfig(t, func(c *Config) {
		c.Bootstrap = false
		c.TLSConfig.InternalRPC.CAFile = "../../test/client_certs/rootca.crt"
		c.TLSConfig.InternalRPC.CertFile = "../../test/client_certs/server.crt"
		c.TLSConfig.InternalRPC.KeyFile = "../../test/client_certs/server.key"
	})
	defer os.RemoveAll(dir2)
	defer s2.Shutdown()

	success, err := testVerifyRPC(s1, s2, t)
	if err != nil {
		t.Fatal(err)
	}
	if !success {
		t.Fatalf("bad: %v", success)
	}
}

func TestServer_RevokeLeadershipIdempotent(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	t.Parallel()
	dir1, s1 := testServer(t)
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	testrpc.WaitForLeader(t, s1.RPC, "dc1")

	s1.revokeLeadership()
	s1.revokeLeadership()
}

func TestServer_ReloadConfig(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}
	t.Parallel()

	entryInit := &structs.ProxyConfigEntry{
		Kind: structs.ProxyDefaults,
		Name: structs.ProxyConfigGlobal,
		Config: map[string]interface{}{
			// these are made a []uint8 and a int64 to allow the Equals test to pass
			// otherwise it will fail complaining about data types
			"foo": "bar",
			"bar": int64(1),
		},
	}

	dir1, s := testServerWithConfig(t, func(c *Config) {
		c.Build = "1.5.0"
		c.RPCRateLimit = 500
		c.RPCMaxBurst = 5000
		c.RequestLimitsMode = "permissive"
		c.RequestLimitsReadRate = 500
		c.RequestLimitsWriteRate = 500
		c.RPCClientTimeout = 60 * time.Second
		// Set one raft param to be non-default in the initial config, others are
		// default.
		c.RaftConfig.TrailingLogs = 1234
	})
	defer os.RemoveAll(dir1)
	defer s.Shutdown()

	testrpc.WaitForTestAgent(t, s.RPC, "dc1")

	limiter := s.rpcLimiter.Load().(*rate.Limiter)
	require.Equal(t, rate.Limit(500), limiter.Limit())
	require.Equal(t, 5000, limiter.Burst())

	require.Equal(t, 60*time.Second, s.connPool.RPCClientTimeout())

	rc := ReloadableConfig{
		RequestLimits: &RequestLimits{
			Mode:      rpcRate.ModeEnforcing,
			ReadRate:  1000,
			WriteRate: 1100,
		},
		RPCClientTimeout:     2 * time.Minute,
		RPCRateLimit:         1000,
		RPCMaxBurst:          10000,
		ConfigEntryBootstrap: []structs.ConfigEntry{entryInit},
		// Reset the custom one to default be removing it from config file (it will
		// be a zero value here).
		RaftTrailingLogs: 0,

		// Set a different Raft param to something custom now
		RaftSnapshotThreshold: 4321,

		// Leave other raft fields default
	}

	mockHandler := rpcRate.NewMockRequestLimitsHandler(t)
	mockHandler.On("UpdateConfig", mock.Anything).Return(func(cfg rpcRate.HandlerConfig) {})

	s.incomingRPCLimiter = mockHandler
	require.NoError(t, s.ReloadConfig(rc))

	_, entry, err := s.fsm.State().ConfigEntry(nil, structs.ProxyDefaults, structs.ProxyConfigGlobal, structs.DefaultEnterpriseMetaInDefaultPartition())
	require.NoError(t, err)
	require.NotNil(t, entry)
	global, ok := entry.(*structs.ProxyConfigEntry)
	require.True(t, ok)
	require.Equal(t, entryInit.Kind, global.Kind)
	require.Equal(t, entryInit.Name, global.Name)
	require.Equal(t, entryInit.Config, global.Config)

	// Check rate limiter got updated
	limiter = s.rpcLimiter.Load().(*rate.Limiter)
	require.Equal(t, rate.Limit(1000), limiter.Limit())
	require.Equal(t, 10000, limiter.Burst())

	// Check the incoming RPC rate limiter got updated
	mockHandler.AssertCalled(t, "UpdateConfig", rpcRate.HandlerConfig{
		GlobalLimitConfig: rpcRate.GlobalLimitConfig{
			Mode: rc.RequestLimits.Mode,
			ReadWriteConfig: rpcRate.ReadWriteConfig{
				ReadConfig: multilimiter.LimiterConfig{
					Rate:  rc.RequestLimits.ReadRate,
					Burst: int(rc.RequestLimits.ReadRate) * requestLimitsBurstMultiplier,
				},
				WriteConfig: multilimiter.LimiterConfig{
					Rate:  rc.RequestLimits.WriteRate,
					Burst: int(rc.RequestLimits.WriteRate) * requestLimitsBurstMultiplier,
				},
			},
		},
	})

	// Check RPC client timeout got updated
	require.Equal(t, 2*time.Minute, s.connPool.RPCClientTimeout())

	// Check raft config
	defaults := DefaultConfig()
	got := s.raft.ReloadableConfig()
	require.Equal(t, uint64(4321), got.SnapshotThreshold,
		"should have be reloaded to new value")
	require.Equal(t, defaults.RaftConfig.SnapshotInterval, got.SnapshotInterval,
		"should have remained the default interval")
	require.Equal(t, defaults.RaftConfig.TrailingLogs, got.TrailingLogs,
		"should have reloaded to default trailing_logs")

	// Now check that update each of those raft fields separately works correctly
	// too.
}

func TestServer_computeRaftReloadableConfig(t *testing.T) {

	defaults := DefaultConfig().RaftConfig

	cases := []struct {
		name string
		rc   ReloadableConfig
		want raft.ReloadableConfig
	}{
		{
			// This case is the common path - reload is called with a ReloadableConfig
			// populated from the RuntimeConfig which has zero values for the fields.
			// On startup we selectively pick non-zero runtime config fields to
			// override defaults so we need to do the same.
			name: "Still defaults",
			rc:   ReloadableConfig{},
			want: raft.ReloadableConfig{
				SnapshotThreshold: defaults.SnapshotThreshold,
				SnapshotInterval:  defaults.SnapshotInterval,
				TrailingLogs:      defaults.TrailingLogs,
				ElectionTimeout:   defaults.ElectionTimeout,
				HeartbeatTimeout:  defaults.HeartbeatTimeout,
			},
		},
		{
			name: "Threshold set",
			rc: ReloadableConfig{
				RaftSnapshotThreshold: 123456,
			},
			want: raft.ReloadableConfig{
				SnapshotThreshold: 123456,
				SnapshotInterval:  defaults.SnapshotInterval,
				TrailingLogs:      defaults.TrailingLogs,
				ElectionTimeout:   defaults.ElectionTimeout,
				HeartbeatTimeout:  defaults.HeartbeatTimeout,
			},
		},
		{
			name: "interval set",
			rc: ReloadableConfig{
				RaftSnapshotInterval: 13 * time.Minute,
			},
			want: raft.ReloadableConfig{
				SnapshotThreshold: defaults.SnapshotThreshold,
				SnapshotInterval:  13 * time.Minute,
				TrailingLogs:      defaults.TrailingLogs,
				ElectionTimeout:   defaults.ElectionTimeout,
				HeartbeatTimeout:  defaults.HeartbeatTimeout,
			},
		},
		{
			name: "trailing logs set",
			rc: ReloadableConfig{
				RaftTrailingLogs: 78910,
			},
			want: raft.ReloadableConfig{
				SnapshotThreshold: defaults.SnapshotThreshold,
				SnapshotInterval:  defaults.SnapshotInterval,
				TrailingLogs:      78910,
				ElectionTimeout:   defaults.ElectionTimeout,
				HeartbeatTimeout:  defaults.HeartbeatTimeout,
			},
		},
		{
			name: "all set",
			rc: ReloadableConfig{
				RaftSnapshotThreshold: 123456,
				RaftSnapshotInterval:  13 * time.Minute,
				RaftTrailingLogs:      78910,
				ElectionTimeout:       300 * time.Millisecond,
				HeartbeatTimeout:      400 * time.Millisecond,
			},
			want: raft.ReloadableConfig{
				SnapshotThreshold: 123456,
				SnapshotInterval:  13 * time.Minute,
				TrailingLogs:      78910,
				ElectionTimeout:   300 * time.Millisecond,
				HeartbeatTimeout:  400 * time.Millisecond,
			},
		},
	}

	for _, tc := range cases {
		tc := tc
		t.Run(tc.name, func(t *testing.T) {
			got := computeRaftReloadableConfig(tc.rc)
			require.Equal(t, tc.want, got)
		})
	}
}

func TestServer_RPC_RateLimit(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	t.Parallel()
	_, conf1 := testServerConfig(t)
	conf1.RPCRateLimit = 2
	conf1.RPCMaxBurst = 2
	s1, err := newServer(t, conf1)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	defer s1.Shutdown()
	testrpc.WaitForLeader(t, s1.RPC, "dc1")

	retry.Run(t, func(r *retry.R) {
		var out struct{}
		if err := s1.RPC(context.Background(), "Status.Ping", struct{}{}, &out); err != structs.ErrRPCRateExceeded {
			r.Fatalf("err: %v", err)
		}
	})
}

// TestServer_Peering_LeadershipCheck tests that a peering service can receive the leader address
// through the LeaderAddress IRL.
func TestServer_Peering_LeadershipCheck(t *testing.T) {
	if testing.Short() {
		t.Skip("too slow for testing.Short")
	}

	t.Parallel()

	// given two servers: s1 (leader), s2 (follower)
	_, conf1 := testServerConfig(t)
	s1, err := newServer(t, conf1)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	defer s1.Shutdown()

	_, conf2 := testServerConfig(t)
	conf2.Bootstrap = false
	s2, err := newServer(t, conf2)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	defer s2.Shutdown()

	// Try to join
	joinLAN(t, s2, s1)

	// Verify Raft has established a peer
	retry.Run(t, func(r *retry.R) {
		r.Check(wantRaft([]*Server{s1, s2}))
	})

	testrpc.WaitForLeader(t, s1.RPC, "dc1")
	testrpc.WaitForLeader(t, s2.RPC, "dc1")
	waitForLeaderEstablishment(t, s1)

	// the actual tests
	// when leadership has been established s2 should have the address of s1
	// in the peering service
	peeringLeaderAddr := s2.peeringBackend.GetLeaderAddress()

	require.Equal(t, s1.config.RPCAddr.String(), peeringLeaderAddr)
	// test corollary by transitivity to future-proof against any setup bugs
	require.NotEqual(t, s2.config.RPCAddr.String(), peeringLeaderAddr)
}

func TestServer_hcpManager(t *testing.T) {
	_, conf1 := testServerConfig(t)
	conf1.BootstrapExpect = 1
	conf1.RPCAdvertise = &net.TCPAddr{IP: []byte{127, 0, 0, 2}, Port: conf1.RPCAddr.Port}
	hcp1 := hcp.NewMockClient(t)
	hcp1.EXPECT().PushServerStatus(mock.Anything, mock.MatchedBy(func(status *hcp.ServerStatus) bool {
		return status.ID == string(conf1.NodeID)
	})).Run(func(ctx context.Context, status *hcp.ServerStatus) {
		require.Equal(t, status.LanAddress, "127.0.0.2")
	}).Call.Return(nil)

	deps1 := newDefaultDeps(t, conf1)
	deps1.HCP.Client = hcp1
	s1, err := newServerWithDeps(t, conf1, deps1)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	defer s1.Shutdown()
	require.NotNil(t, s1.hcpManager)
	waitForLeaderEstablishment(t, s1)
	hcp1.AssertExpectations(t)

}