open-consul/agent/auto-config/auto_config_test.go

package autoconf

import (
	"context"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/mock"
	"github.com/stretchr/testify/require"

	"github.com/hashicorp/consul/agent/cache"
	cachetype "github.com/hashicorp/consul/agent/cache-types"
	"github.com/hashicorp/consul/agent/config"
	"github.com/hashicorp/consul/agent/connect"
	"github.com/hashicorp/consul/agent/metadata"
	"github.com/hashicorp/consul/agent/structs"
	"github.com/hashicorp/consul/agent/token"
	"github.com/hashicorp/consul/lib/retry"
	"github.com/hashicorp/consul/proto/pbautoconf"
	"github.com/hashicorp/consul/proto/pbconfig"
	"github.com/hashicorp/consul/sdk/testutil"
	testretry "github.com/hashicorp/consul/sdk/testutil/retry"
)

type configLoader struct {
	opts config.BuilderOpts
}

func (c *configLoader) Load(source config.Source) (*config.RuntimeConfig, []string, error) {
	return config.Load(c.opts, source)
}

func (c *configLoader) addConfigHCL(cfg string) {
	c.opts.HCL = append(c.opts.HCL, cfg)
}

func requireChanNotReady(t *testing.T, ch <-chan struct{}) {
	select {
	case <-ch:
		require.Fail(t, "chan is ready when it shouldn't be")
	default:
		return
	}
}

func requireChanReady(t *testing.T, ch <-chan struct{}) {
	select {
	case <-ch:
		return
	default:
		require.Fail(t, "chan is not ready when it should be")
	}
}

func waitForChan(timer *time.Timer, ch <-chan struct{}) bool {
	select {
	case <-timer.C:
		return false
	case <-ch:
		return true
	}
}

func waitForChans(timeout time.Duration, chans ...<-chan struct{}) bool {
	timer := time.NewTimer(timeout)
	defer timer.Stop()

	for _, ch := range chans {
		if !waitForChan(timer, ch) {
			return false
		}
	}
	return true
}

func TestNew(t *testing.T) {
	type testCase struct {
		modify   func(*Config)
		err      string
		validate func(t *testing.T, ac *AutoConfig)
	}

	cases := map[string]testCase{
		"no-direct-rpc": {
			modify: func(c *Config) {
				c.DirectRPC = nil
			},
			err: "must provide a direct RPC delegate",
		},
		"no-config-loader": {
			modify: func(c *Config) {
				c.Loader = nil
			},
			err: "must provide a config loader",
		},
		"no-cache": {
			modify: func(c *Config) {
				c.Cache = nil
			},
			err: "must provide a cache",
		},
		"no-tls-configurator": {
			modify: func(c *Config) {
				c.TLSConfigurator = nil
			},
			err: "must provide a TLS configurator",
		},
		"no-tokens": {
			modify: func(c *Config) {
				c.Tokens = nil
			},
			err: "must provide a token store",
		},
		"ok": {
			validate: func(t *testing.T, ac *AutoConfig) {
				t.Helper()
				require.NotNil(t, ac.logger)
				require.NotNil(t, ac.acConfig.Waiter)
				require.Equal(t, time.Minute, ac.acConfig.FallbackRetry)
				require.Equal(t, 10*time.Second, ac.acConfig.FallbackLeeway)
			},
		},
	}

	for name, tcase := range cases {
		t.Run(name, func(t *testing.T) {
			cfg := Config{
				Loader: func(source config.Source) (cfg *config.RuntimeConfig, warnings []string, err error) {
					return nil, nil, nil
				},
				DirectRPC:       newMockDirectRPC(t),
				Tokens:          newMockTokenStore(t),
				Cache:           newMockCache(t),
				TLSConfigurator: newMockTLSConfigurator(t),
				ServerProvider:  newMockServerProvider(t),
			}

			if tcase.modify != nil {
				tcase.modify(&cfg)
			}

			ac, err := New(cfg)
			if tcase.err != "" {
				testutil.RequireErrorContains(t, err, tcase.err)
			} else {
				require.NoError(t, err)
				require.NotNil(t, ac)
				if tcase.validate != nil {
					tcase.validate(t, ac)
				}
			}
		})
	}
}

func TestReadConfig(t *testing.T) {
	// just testing that some auto config source gets injected
	ac := AutoConfig{
		autoConfigSource: config.LiteralSource{
			Name:   autoConfigFileName,
			Config: config.Config{NodeName: stringPointer("hobbiton")},
		},
		logger: testutil.Logger(t),
		acConfig: Config{
			Loader: func(source config.Source) (*config.RuntimeConfig, []string, error) {
				cfg, _, err := source.Parse()
				if err != nil {
					return nil, nil, err
				}
				return &config.RuntimeConfig{
					DevMode:  true,
					NodeName: *cfg.NodeName,
				}, nil, nil
			},
		},
	}

	cfg, err := ac.ReadConfig()
	require.NoError(t, err)
	require.NotNil(t, cfg)
	require.Equal(t, "hobbiton", cfg.NodeName)
	require.True(t, cfg.DevMode)
	require.Same(t, ac.config, cfg)
}

func setupRuntimeConfig(t *testing.T) *configLoader {
	t.Helper()

	dataDir := testutil.TempDir(t, "auto-config")

	opts := config.BuilderOpts{
		Config: config.Config{
			DataDir:    &dataDir,
			Datacenter: stringPointer("dc1"),
			NodeName:   stringPointer("autoconf"),
			BindAddr:   stringPointer("127.0.0.1"),
		},
	}
	return &configLoader{opts: opts}
}

func TestInitialConfiguration_disabled(t *testing.T) {
	loader := setupRuntimeConfig(t)
	loader.addConfigHCL(`
		primary_datacenter = "primary"
		auto_config = {
			enabled = false
		}
	`)

	conf := newMockedConfig(t).Config
	conf.Loader = loader.Load

	ac, err := New(conf)
	require.NoError(t, err)
	require.NotNil(t, ac)

	cfg, err := ac.InitialConfiguration(context.Background())
	require.NoError(t, err)
	require.NotNil(t, cfg)
	require.Equal(t, "primary", cfg.PrimaryDatacenter)
	require.NoFileExists(t, filepath.Join(*loader.opts.Config.DataDir, autoConfigFileName))
}

func TestInitialConfiguration_cancelled(t *testing.T) {
	mcfg := newMockedConfig(t)

	loader := setupRuntimeConfig(t)
	loader.addConfigHCL(`
		primary_datacenter = "primary"
		auto_config = {
			enabled = true
			intro_token = "blarg"
			server_addresses = ["127.0.0.1:8300"]
		}
		verify_outgoing = true
	`)
	mcfg.Config.Loader = loader.Load

	expectedRequest := pbautoconf.AutoConfigRequest{
		Datacenter: "dc1",
		Node:       "autoconf",
		JWT:        "blarg",
	}

	mcfg.directRPC.On("RPC", "dc1", "autoconf", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 8300}, "AutoConfig.InitialConfiguration", &expectedRequest, mock.Anything).Return(fmt.Errorf("injected error")).Times(0)
	mcfg.serverProvider.On("FindLANServer").Return(nil).Times(0)

	ac, err := New(mcfg.Config)
	require.NoError(t, err)
	require.NotNil(t, ac)

	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(100*time.Millisecond))
	defer cancelFn()

	cfg, err := ac.InitialConfiguration(ctx)
	testutil.RequireErrorContains(t, err, context.DeadlineExceeded.Error())
	require.Nil(t, cfg)
}

func TestInitialConfiguration_restored(t *testing.T) {
	mcfg := newMockedConfig(t)

	loader := setupRuntimeConfig(t)
	loader.addConfigHCL(`
		auto_config = {
			enabled = true
			intro_token ="blarg"
			server_addresses = ["127.0.0.1:8300"]
		}
		verify_outgoing = true
	`)

	mcfg.Config.Loader = loader.Load

	indexedRoots, cert, extraCACerts := mcfg.setupInitialTLS(t, "autoconf", "dc1", "secret")

	// persist an auto config response to the data dir where it is expected
	persistedFile := filepath.Join(*loader.opts.Config.DataDir, autoConfigFileName)
	response := &pbautoconf.AutoConfigResponse{
		Config: &pbconfig.Config{
			PrimaryDatacenter: "primary",
			TLS: &pbconfig.TLS{
				VerifyServerHostname: true,
			},
			ACL: &pbconfig.ACL{
				Tokens: &pbconfig.ACLTokens{
					Agent: "secret",
				},
			},
		},
		CARoots:             mustTranslateCARootsToProtobuf(t, indexedRoots),
		Certificate:         mustTranslateIssuedCertToProtobuf(t, cert),
		ExtraCACertificates: extraCACerts,
	}
	data, err := pbMarshaler.MarshalToString(response)
	require.NoError(t, err)
	require.NoError(t, ioutil.WriteFile(persistedFile, []byte(data), 0600))

	// recording the initial configuration even when restoring is going to update
	// the agent token in the token store
	mcfg.tokens.On("UpdateAgentToken", "secret", token.TokenSourceConfig).Return(true).Once()

	// prepopulation is going to grab the token to populate the correct cache key
	mcfg.tokens.On("AgentToken").Return("secret").Times(0)

	ac, err := New(mcfg.Config)
	require.NoError(t, err)
	require.NotNil(t, ac)

	cfg, err := ac.InitialConfiguration(context.Background())
	require.NoError(t, err, data)
	require.NotNil(t, cfg)
	require.Equal(t, "primary", cfg.PrimaryDatacenter)
}

func TestInitialConfiguration_success(t *testing.T) {
	mcfg := newMockedConfig(t)
	loader := setupRuntimeConfig(t)
	loader.addConfigHCL(`
		auto_config = {
			enabled = true
			intro_token ="blarg"
			server_addresses = ["127.0.0.1:8300"]
		}
		verify_outgoing = true
	`)
	mcfg.Config.Loader = loader.Load

	indexedRoots, cert, extraCerts := mcfg.setupInitialTLS(t, "autoconf", "dc1", "secret")

	// this gets called when InitialConfiguration is invoked to record the token from the
	// auto-config response
	mcfg.tokens.On("UpdateAgentToken", "secret", token.TokenSourceConfig).Return(true).Once()

	// prepopulation is going to grab the token to populate the correct cache key
	mcfg.tokens.On("AgentToken").Return("secret").Times(0)

	// no server provider
	mcfg.serverProvider.On("FindLANServer").Return(nil).Times(0)

	populateResponse := func(args mock.Arguments) {
		resp, ok := args.Get(5).(*pbautoconf.AutoConfigResponse)
		require.True(t, ok)
		resp.Config = &pbconfig.Config{
			PrimaryDatacenter: "primary",
			TLS: &pbconfig.TLS{
				VerifyServerHostname: true,
			},
			ACL: &pbconfig.ACL{
				Tokens: &pbconfig.ACLTokens{
					Agent: "secret",
				},
			},
		}

		resp.CARoots = mustTranslateCARootsToProtobuf(t, indexedRoots)
		resp.Certificate = mustTranslateIssuedCertToProtobuf(t, cert)
		resp.ExtraCACertificates = extraCerts
	}

	expectedRequest := pbautoconf.AutoConfigRequest{
		Datacenter: "dc1",
		Node:       "autoconf",
		JWT:        "blarg",
	}

	mcfg.directRPC.On(
		"RPC",
		"dc1",
		"autoconf",
		&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 8300},
		"AutoConfig.InitialConfiguration",
		&expectedRequest,
		&pbautoconf.AutoConfigResponse{}).Return(nil).Run(populateResponse)

	ac, err := New(mcfg.Config)
	require.NoError(t, err)
	require.NotNil(t, ac)

	cfg, err := ac.InitialConfiguration(context.Background())
	require.NoError(t, err)
	require.NotNil(t, cfg)
	require.Equal(t, "primary", cfg.PrimaryDatacenter)

	// the file was written to.
	persistedFile := filepath.Join(*loader.opts.Config.DataDir, autoConfigFileName)
	require.FileExists(t, persistedFile)
}

func TestInitialConfiguration_retries(t *testing.T) {
	mcfg := newMockedConfig(t)
	loader := setupRuntimeConfig(t)
	loader.addConfigHCL(`
		auto_config = {
			enabled = true
			intro_token ="blarg"
			server_addresses = [
				"198.18.0.1:8300",
				"198.18.0.2:8398",
				"198.18.0.3:8399",
				"127.0.0.1:1234"
			]
		}
		verify_outgoing = true
	`)
	mcfg.Config.Loader = loader.Load

	// reduce the retry wait times to make this test run faster
	mcfg.Config.Waiter = &retry.Waiter{MinFailures: 2, MaxWait: time.Millisecond}

	indexedRoots, cert, extraCerts := mcfg.setupInitialTLS(t, "autoconf", "dc1", "secret")

	// this gets called when InitialConfiguration is invoked to record the token from the
	// auto-config response
	mcfg.tokens.On("UpdateAgentToken", "secret", token.TokenSourceConfig).Return(true).Once()

	// prepopulation is going to grab the token to populate the correct cache key
	mcfg.tokens.On("AgentToken").Return("secret").Times(0)

	// no server provider
	mcfg.serverProvider.On("FindLANServer").Return(nil).Times(0)

	populateResponse := func(args mock.Arguments) {
		resp, ok := args.Get(5).(*pbautoconf.AutoConfigResponse)
		require.True(t, ok)
		resp.Config = &pbconfig.Config{
			PrimaryDatacenter: "primary",
			TLS: &pbconfig.TLS{
				VerifyServerHostname: true,
			},
			ACL: &pbconfig.ACL{
				Tokens: &pbconfig.ACLTokens{
					Agent: "secret",
				},
			},
		}

		resp.CARoots = mustTranslateCARootsToProtobuf(t, indexedRoots)
		resp.Certificate = mustTranslateIssuedCertToProtobuf(t, cert)
		resp.ExtraCACertificates = extraCerts
	}

	expectedRequest := pbautoconf.AutoConfigRequest{
		Datacenter: "dc1",
		Node:       "autoconf",
		JWT:        "blarg",
	}

	// basically the 198.18.0.* addresses should fail indefinitely. the first time through the
	// outer loop we inject a failure for the DNS resolution of localhost to 127.0.0.1. Then
	// the second time through the outer loop we allow the localhost one to work.
	mcfg.directRPC.On(
		"RPC",
		"dc1",
		"autoconf",
		&net.TCPAddr{IP: net.IPv4(198, 18, 0, 1), Port: 8300},
		"AutoConfig.InitialConfiguration",
		&expectedRequest,
		&pbautoconf.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Times(0)
	mcfg.directRPC.On(
		"RPC",
		"dc1",
		"autoconf",
		&net.TCPAddr{IP: net.IPv4(198, 18, 0, 2), Port: 8398},
		"AutoConfig.InitialConfiguration",
		&expectedRequest,
		&pbautoconf.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Times(0)
	mcfg.directRPC.On(
		"RPC",
		"dc1",
		"autoconf",
		&net.TCPAddr{IP: net.IPv4(198, 18, 0, 3), Port: 8399},
		"AutoConfig.InitialConfiguration",
		&expectedRequest,
		&pbautoconf.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Times(0)
	mcfg.directRPC.On(
		"RPC",
		"dc1",
		"autoconf",
		&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 1234},
		"AutoConfig.InitialConfiguration",
		&expectedRequest,
		&pbautoconf.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Once()
	mcfg.directRPC.On(
		"RPC",
		"dc1",
		"autoconf",
		&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 1234},
		"AutoConfig.InitialConfiguration",
		&expectedRequest,
		&pbautoconf.AutoConfigResponse{}).Return(nil).Run(populateResponse).Once()

	ac, err := New(mcfg.Config)
	require.NoError(t, err)
	require.NotNil(t, ac)

	cfg, err := ac.InitialConfiguration(context.Background())
	require.NoError(t, err)
	require.NotNil(t, cfg)
	require.Equal(t, "primary", cfg.PrimaryDatacenter)

	// the file was written to.
	persistedFile := filepath.Join(*loader.opts.Config.DataDir, autoConfigFileName)
	require.FileExists(t, persistedFile)
}

func TestGoRoutineManagement(t *testing.T) {
	mcfg := newMockedConfig(t)
	loader := setupRuntimeConfig(t)
	loader.addConfigHCL(`
		auto_config = {
			enabled = true
			intro_token ="blarg"
			server_addresses = ["127.0.0.1:8300"]
		}
		verify_outgoing = true
	`)
	mcfg.Config.Loader = loader.Load

	// prepopulation is going to grab the token to populate the correct cache key
	mcfg.tokens.On("AgentToken").Return("secret").Times(0)

	ac, err := New(mcfg.Config)
	require.NoError(t, err)

	// priming the config so some other requests will work properly that need to
	// read from the configuration. We are going to avoid doing InitialConfiguration
	// for this test as we only are really concerned with the go routine management
	_, err = ac.ReadConfig()
	require.NoError(t, err)

	var rootsCtx context.Context
	var leafCtx context.Context
	var ctxLock sync.Mutex

	rootsReq := ac.caRootsRequest()
	mcfg.cache.On("Notify",
		mock.Anything,
		cachetype.ConnectCARootName,
		&rootsReq,
		rootsWatchID,
		mock.Anything,
	).Return(nil).Times(2).Run(func(args mock.Arguments) {
		ctxLock.Lock()
		rootsCtx = args.Get(0).(context.Context)
		ctxLock.Unlock()
	})

	leafReq := ac.leafCertRequest()
	mcfg.cache.On("Notify",
		mock.Anything,
		cachetype.ConnectCALeafName,
		&leafReq,
		leafWatchID,
		mock.Anything,
	).Return(nil).Times(2).Run(func(args mock.Arguments) {
		ctxLock.Lock()
		leafCtx = args.Get(0).(context.Context)
		ctxLock.Unlock()
	})

	// we will start/stop things twice
	mcfg.tokens.On("Notify", token.TokenKindAgent).Return(token.Notifier{}).Times(2)
	mcfg.tokens.On("StopNotify", token.Notifier{}).Times(2)

	mcfg.tlsCfg.On("AutoEncryptCertNotAfter").Return(time.Now().Add(10 * time.Minute)).Times(0)

	// ensure that auto-config isn't running
	require.False(t, ac.IsRunning())

	// ensure that nothing bad happens and that it reports as stopped
	require.False(t, ac.Stop())

	// ensure that the Done chan also reports that things are not running
	// in other words the chan is immediately selectable
	requireChanReady(t, ac.Done())

	// start auto-config
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	require.NoError(t, ac.Start(ctx))

	waitForContexts := func() bool {
		ctxLock.Lock()
		defer ctxLock.Unlock()
		return !(rootsCtx == nil || leafCtx == nil)
	}

	// wait for the cache notifications to get started
	require.Eventually(t, waitForContexts, 100*time.Millisecond, 10*time.Millisecond)

	// hold onto the Done chan to test for the go routine exiting
	done := ac.Done()

	// ensure we report as running
	require.True(t, ac.IsRunning())

	// ensure the done chan is not selectable yet
	requireChanNotReady(t, done)

	// ensure we error if we attempt to start again
	err = ac.Start(ctx)
	testutil.RequireErrorContains(t, err, "AutoConfig is already running")

	// now stop things - it should return true indicating that it was running
	// when we attempted to stop it.
	require.True(t, ac.Stop())

	// ensure that the go routine shuts down - it will close the done chan. Also it should cancel
	// the cache watches by cancelling the context it passed into the Notify call.
	require.True(t, waitForChans(100*time.Millisecond, done, leafCtx.Done(), rootsCtx.Done()), "AutoConfig didn't shut down")
	require.False(t, ac.IsRunning())

	// restart it
	require.NoError(t, ac.Start(ctx))

	// get the new Done chan
	done = ac.Done()

	// ensure that context cancellation causes us to stop as well
	cancel()
	require.True(t, waitForChans(100*time.Millisecond, done))
}

type testAutoConfig struct {
	mcfg          *mockedConfig
	ac            *AutoConfig
	tokenUpdates  chan struct{}
	originalToken string

	initialRoots *structs.IndexedCARoots
	initialCert  *structs.IssuedCert
	extraCerts   []string
}

func startedAutoConfig(t *testing.T, autoEncrypt bool) testAutoConfig {
	t.Helper()
	mcfg := newMockedConfig(t)
	loader := setupRuntimeConfig(t)
	if !autoEncrypt {
		loader.addConfigHCL(`
			auto_config = {
				enabled = true
				intro_token ="blarg"
				server_addresses = ["127.0.0.1:8300"]
			}
			verify_outgoing = true
		`)
	} else {
		loader.addConfigHCL(`
			auto_encrypt {
				tls = true
			}
			verify_outgoing = true
		`)
	}
	mcfg.Config.Loader = loader.Load
	mcfg.Config.FallbackLeeway = time.Nanosecond

	originalToken := "a5deaa25-11ca-48bf-a979-4c3a7aa4b9a9"

	if !autoEncrypt {
		// this gets called when InitialConfiguration is invoked to record the token from the
		// auto-config response
		mcfg.tokens.On("UpdateAgentToken", originalToken, token.TokenSourceConfig).Return(true).Once()
	}

	// we expect this to be retrieved twice: first during cache prepopulation
	// and then again when setting up the cache watch for the leaf cert.
	// However one of those expectations is setup in the expectInitialTLS
	// method so we only need one more here
	mcfg.tokens.On("AgentToken").Return(originalToken).Once()

	if autoEncrypt {
		// when using AutoEncrypt we also have to grab the token once more
		// when setting up the initial RPC as the ACL token is what is used
		// to authorize the request.
		mcfg.tokens.On("AgentToken").Return(originalToken).Once()
	}

	// this is called once during Start to initialze the token watches
	tokenUpdateCh := make(chan struct{})
	tokenNotifier := token.Notifier{
		Ch: tokenUpdateCh,
	}
	mcfg.tokens.On("Notify", token.TokenKindAgent).Once().Return(tokenNotifier)
	mcfg.tokens.On("StopNotify", tokenNotifier).Once()

	// expect the roots watch on the cache
	mcfg.cache.On("Notify",
		mock.Anything,
		cachetype.ConnectCARootName,
		&structs.DCSpecificRequest{Datacenter: "dc1"},
		rootsWatchID,
		mock.Anything,
	).Return(nil).Once()

	mcfg.cache.On("Notify",
		mock.Anything,
		cachetype.ConnectCALeafName,
		&cachetype.ConnectCALeafRequest{
			Datacenter: "dc1",
			Agent:      "autoconf",
			Token:      originalToken,
			DNSSAN:     defaultDNSSANs,
			IPSAN:      defaultIPSANs,
		},
		leafWatchID,
		mock.Anything,
	).Return(nil).Once()

	// override the server provider - most of the other tests set it up so that this
	// always returns no server (simulating a state where we haven't joined gossip).
	// this seems like a good place to ensure this other way of finding server information
	// works
	mcfg.serverProvider.On("FindLANServer").Once().Return(&metadata.Server{
		Addr: &net.TCPAddr{IP: net.IPv4(198, 18, 0, 1), Port: 8300},
	})

	indexedRoots, cert, extraCerts := mcfg.setupInitialTLS(t, "autoconf", "dc1", originalToken)

	mcfg.tlsCfg.On("AutoEncryptCertNotAfter").Return(cert.ValidBefore).Once()

	populateResponse := func(args mock.Arguments) {
		method := args.String(3)

		switch method {
		case "AutoConfig.InitialConfiguration":
			resp, ok := args.Get(5).(*pbautoconf.AutoConfigResponse)
			require.True(t, ok)
			resp.Config = &pbconfig.Config{
				PrimaryDatacenter: "primary",
				TLS: &pbconfig.TLS{
					VerifyServerHostname: true,
				},
				ACL: &pbconfig.ACL{
					Tokens: &pbconfig.ACLTokens{
						Agent: originalToken,
					},
				},
			}

			resp.CARoots = mustTranslateCARootsToProtobuf(t, indexedRoots)
			resp.Certificate = mustTranslateIssuedCertToProtobuf(t, cert)
			resp.ExtraCACertificates = extraCerts
		case "AutoEncrypt.Sign":
			resp, ok := args.Get(5).(*structs.SignedResponse)
			require.True(t, ok)
			*resp = structs.SignedResponse{
				VerifyServerHostname: true,
				ConnectCARoots:       *indexedRoots,
				IssuedCert:           *cert,
				ManualCARoots:        extraCerts,
			}
		}
	}

	if !autoEncrypt {
		expectedRequest := pbautoconf.AutoConfigRequest{
			Datacenter: "dc1",
			Node:       "autoconf",
			JWT:        "blarg",
		}

		mcfg.directRPC.On(
			"RPC",
			"dc1",
			"autoconf",
			&net.TCPAddr{IP: net.IPv4(198, 18, 0, 1), Port: 8300},
			"AutoConfig.InitialConfiguration",
			&expectedRequest,
			&pbautoconf.AutoConfigResponse{}).Return(nil).Run(populateResponse).Once()
	} else {
		expectedRequest := structs.CASignRequest{
			WriteRequest: structs.WriteRequest{Token: originalToken},
			Datacenter:   "dc1",
			// TODO (autoconf) Maybe in the future we should populate a CSR
			// and do some manual parsing/verification of the contents. The
			// bits not having to do with the signing key such as the requested
			// SANs and CN. For now though the mockDirectRPC type will empty
			// the CSR so we have to pass in an empty string to the expectation.
			CSR: "",
		}

		mcfg.directRPC.On(
			"RPC",
			"dc1",
			"autoconf", // reusing the same name to prevent needing more configurability
			&net.TCPAddr{IP: net.IPv4(198, 18, 0, 1), Port: 8300},
			"AutoEncrypt.Sign",
			&expectedRequest,
			&structs.SignedResponse{}).Return(nil).Run(populateResponse)
	}

	ac, err := New(mcfg.Config)
	require.NoError(t, err)
	require.NotNil(t, ac)

	cfg, err := ac.InitialConfiguration(context.Background())
	require.NoError(t, err)
	require.NotNil(t, cfg)
	if !autoEncrypt {
		// auto-encrypt doesn't modify the config but rather sets the value
		// in the TLS configurator
		require.True(t, cfg.VerifyServerHostname)
	}

	ctx, cancel := context.WithCancel(context.Background())
	require.NoError(t, ac.Start(ctx))
	t.Cleanup(func() {
		done := ac.Done()
		cancel()
		timer := time.NewTimer(1 * time.Second)
		defer timer.Stop()
		select {
		case <-done:
			// do nothing
		case <-timer.C:
			t.Fatalf("AutoConfig wasn't stopped within 1 second after test completion")
		}
	})

	return testAutoConfig{
		mcfg:          mcfg,
		ac:            ac,
		tokenUpdates:  tokenUpdateCh,
		originalToken: originalToken,
		initialRoots:  indexedRoots,
		initialCert:   cert,
		extraCerts:    extraCerts,
	}
}

// this test ensures that the cache watches are restarted with
// the updated token after receiving a token update
func TestTokenUpdate(t *testing.T) {
	testAC := startedAutoConfig(t, false)

	newToken := "1a4cc445-86ed-46b4-a355-bbf5a11dddb0"

	rootsCtx, rootsCancel := context.WithCancel(context.Background())
	testAC.mcfg.cache.On("Notify",
		mock.Anything,
		cachetype.ConnectCARootName,
		&structs.DCSpecificRequest{Datacenter: testAC.ac.config.Datacenter},
		rootsWatchID,
		mock.Anything,
	).Return(nil).Once().Run(func(args mock.Arguments) {
		rootsCancel()
	})

	leafCtx, leafCancel := context.WithCancel(context.Background())
	testAC.mcfg.cache.On("Notify",
		mock.Anything,
		cachetype.ConnectCALeafName,
		&cachetype.ConnectCALeafRequest{
			Datacenter: "dc1",
			Agent:      "autoconf",
			Token:      newToken,
			DNSSAN:     defaultDNSSANs,
			IPSAN:      defaultIPSANs,
		},
		leafWatchID,
		mock.Anything,
	).Return(nil).Once().Run(func(args mock.Arguments) {
		leafCancel()
	})

	// this will be retrieved once when resetting the leaf cert watch
	testAC.mcfg.tokens.On("AgentToken").Return(newToken).Once()

	// send the notification about the token update
	testAC.tokenUpdates <- struct{}{}

	// wait for the leaf cert watches
	require.True(t, waitForChans(100*time.Millisecond, leafCtx.Done(), rootsCtx.Done()), "New cache watches were not started within 100ms")
}

func TestRootsUpdate(t *testing.T) {
	testAC := startedAutoConfig(t, false)

	secondCA := connect.TestCA(t, testAC.initialRoots.Roots[0])
	secondRoots := structs.IndexedCARoots{
		ActiveRootID: secondCA.ID,
		TrustDomain:  connect.TestClusterID,
		Roots: []*structs.CARoot{
			secondCA,
			testAC.initialRoots.Roots[0],
		},
		QueryMeta: structs.QueryMeta{
			Index: 99,
		},
	}

	updatedCtx, cancel := context.WithCancel(context.Background())
	testAC.mcfg.tlsCfg.On("UpdateAutoTLS",
		testAC.extraCerts,
		[]string{secondCA.RootCert, testAC.initialRoots.Roots[0].RootCert},
		testAC.initialCert.CertPEM,
		"redacted",
		true,
	).Return(nil).Once().Run(func(args mock.Arguments) {
		cancel()
	})

	// when a cache event comes in we end up recalculating the fallback timer which requires this call
	testAC.mcfg.tlsCfg.On("AutoEncryptCertNotAfter").Return(time.Now().Add(10 * time.Minute)).Once()

	req := structs.DCSpecificRequest{Datacenter: "dc1"}
	require.True(t, testAC.mcfg.cache.sendNotification(context.Background(), req.CacheInfo().Key, cache.UpdateEvent{
		CorrelationID: rootsWatchID,
		Result:        &secondRoots,
		Meta: cache.ResultMeta{
			Index: secondRoots.Index,
		},
	}))

	require.True(t, waitForChans(100*time.Millisecond, updatedCtx.Done()), "TLS certificates were not updated within the alotted time")

	// persisting these to disk happens right after the chan we are waiting for will have fired above
	// however there is no deterministic way to know once its been written outside of maybe a filesystem
	// event notifier. That seems a little heavy handed just for this and especially to do in any sort
	// of cross platform way.
	testretry.Run(t, func(r *testretry.R) {
		resp, err := testAC.ac.readPersistedAutoConfig()
		require.NoError(r, err)
		require.Equal(r, secondRoots.ActiveRootID, resp.CARoots.GetActiveRootID())
	})
}

func TestCertUpdate(t *testing.T) {
	testAC := startedAutoConfig(t, false)
	secondCert := newLeaf(t, "autoconf", "dc1", testAC.initialRoots.Roots[0], 99, 10*time.Minute)

	updatedCtx, cancel := context.WithCancel(context.Background())
	testAC.mcfg.tlsCfg.On("UpdateAutoTLS",
		testAC.extraCerts,
		[]string{testAC.initialRoots.Roots[0].RootCert},
		secondCert.CertPEM,
		"redacted",
		true,
	).Return(nil).Once().Run(func(args mock.Arguments) {
		cancel()
	})

	// when a cache event comes in we end up recalculating the fallback timer which requires this call
	testAC.mcfg.tlsCfg.On("AutoEncryptCertNotAfter").Return(secondCert.ValidBefore).Once()

	req := cachetype.ConnectCALeafRequest{
		Datacenter: "dc1",
		Agent:      "autoconf",
		Token:      testAC.originalToken,
		DNSSAN:     defaultDNSSANs,
		IPSAN:      defaultIPSANs,
	}
	require.True(t, testAC.mcfg.cache.sendNotification(context.Background(), req.CacheInfo().Key, cache.UpdateEvent{
		CorrelationID: leafWatchID,
		Result:        secondCert,
		Meta: cache.ResultMeta{
			Index: secondCert.ModifyIndex,
		},
	}))

	require.True(t, waitForChans(100*time.Millisecond, updatedCtx.Done()), "TLS certificates were not updated within the alotted time")

	// persisting these to disk happens after all the things we would wait for in assertCertUpdated
	// will have fired. There is no deterministic way to know once its been written so we wrap
	// this in a retry.
	testretry.Run(t, func(r *testretry.R) {
		resp, err := testAC.ac.readPersistedAutoConfig()
		require.NoError(r, err)

		// ensure the roots got persisted to disk
		require.Equal(r, secondCert.CertPEM, resp.Certificate.GetCertPEM())
	})
}

func TestFallback(t *testing.T) {
	testAC := startedAutoConfig(t, false)

	// at this point everything is operating normally and we are just
	// waiting for events. We are going to send a new cert that is basically
	// already expired and then allow the fallback routine to kick in.
	secondCert := newLeaf(t, "autoconf", "dc1", testAC.initialRoots.Roots[0], 100, time.Nanosecond)
	secondCA := connect.TestCA(t, testAC.initialRoots.Roots[0])
	secondRoots := structs.IndexedCARoots{
		ActiveRootID: secondCA.ID,
		TrustDomain:  connect.TestClusterID,
		Roots: []*structs.CARoot{
			secondCA,
			testAC.initialRoots.Roots[0],
		},
		QueryMeta: structs.QueryMeta{
			Index: 101,
		},
	}
	thirdCert := newLeaf(t, "autoconf", "dc1", secondCA, 102, 10*time.Minute)

	// setup the expectation for when the certs got updated initially
	updatedCtx, updateCancel := context.WithCancel(context.Background())
	testAC.mcfg.tlsCfg.On("UpdateAutoTLS",
		testAC.extraCerts,
		[]string{testAC.initialRoots.Roots[0].RootCert},
		secondCert.CertPEM,
		"redacted",
		true,
	).Return(nil).Once().Run(func(args mock.Arguments) {
		updateCancel()
	})

	// when a cache event comes in we end up recalculating the fallback timer which requires this call
	testAC.mcfg.tlsCfg.On("AutoEncryptCertNotAfter").Return(secondCert.ValidBefore).Once()
	testAC.mcfg.tlsCfg.On("AutoEncryptCertExpired").Return(true).Once()

	fallbackCtx, fallbackCancel := context.WithCancel(context.Background())

	// also testing here that we can change server IPs for ongoing operations
	testAC.mcfg.serverProvider.On("FindLANServer").Once().Return(&metadata.Server{
		Addr: &net.TCPAddr{IP: net.IPv4(198, 18, 23, 2), Port: 8300},
	})

	// after sending the notification for the cert update another InitialConfiguration RPC
	// will be made to pull down the latest configuration. So we need to set up the response
	// for the second RPC
	populateResponse := func(args mock.Arguments) {
		resp, ok := args.Get(5).(*pbautoconf.AutoConfigResponse)
		require.True(t, ok)
		resp.Config = &pbconfig.Config{
			PrimaryDatacenter: "primary",
			TLS: &pbconfig.TLS{
				VerifyServerHostname: true,
			},
			ACL: &pbconfig.ACL{
				Tokens: &pbconfig.ACLTokens{
					Agent: testAC.originalToken,
				},
			},
		}

		resp.CARoots = mustTranslateCARootsToProtobuf(t, &secondRoots)
		resp.Certificate = mustTranslateIssuedCertToProtobuf(t, thirdCert)
		resp.ExtraCACertificates = testAC.extraCerts

		fallbackCancel()
	}

	expectedRequest := pbautoconf.AutoConfigRequest{
		Datacenter: "dc1",
		Node:       "autoconf",
		JWT:        "blarg",
	}

	testAC.mcfg.directRPC.On(
		"RPC",
		"dc1",
		"autoconf",
		&net.TCPAddr{IP: net.IPv4(198, 18, 23, 2), Port: 8300},
		"AutoConfig.InitialConfiguration",
		&expectedRequest,
		&pbautoconf.AutoConfigResponse{}).Return(nil).Run(populateResponse).Once()

	// this gets called when InitialConfiguration is invoked to record the token from the
	// auto-config response which is how the Fallback for auto-config works
	testAC.mcfg.tokens.On("UpdateAgentToken", testAC.originalToken, token.TokenSourceConfig).Return(true).Once()

	testAC.mcfg.expectInitialTLS(t, "autoconf", "dc1", testAC.originalToken, secondCA, &secondRoots, thirdCert, testAC.extraCerts)

	// after the second RPC we now will use the new certs validity period in the next run loop iteration
	testAC.mcfg.tlsCfg.On("AutoEncryptCertNotAfter").Return(time.Now().Add(10 * time.Minute)).Once()

	// now that all the mocks are set up we can trigger the whole thing by sending the second expired cert
	// as a cache update event.
	req := cachetype.ConnectCALeafRequest{
		Datacenter: "dc1",
		Agent:      "autoconf",
		Token:      testAC.originalToken,
		DNSSAN:     defaultDNSSANs,
		IPSAN:      defaultIPSANs,
	}
	require.True(t, testAC.mcfg.cache.sendNotification(context.Background(), req.CacheInfo().Key, cache.UpdateEvent{
		CorrelationID: leafWatchID,
		Result:        secondCert,
		Meta: cache.ResultMeta{
			Index: secondCert.ModifyIndex,
		},
	}))

	// wait for the TLS certificates to get updated
	require.True(t, waitForChans(100*time.Millisecond, updatedCtx.Done()), "TLS certificates were not updated within the alotted time")

	// now wait for the fallback routine to be invoked
	require.True(t, waitForChans(100*time.Millisecond, fallbackCtx.Done()), "fallback routines did not get invoked within the alotted time")

	// persisting these to disk happens after the RPC we waited on above will have fired
	// There is no deterministic way to know once its been written so we wrap this in a retry.
	testretry.Run(t, func(r *testretry.R) {
		resp, err := testAC.ac.readPersistedAutoConfig()
		require.NoError(r, err)

		// ensure the roots got persisted to disk
		require.Equal(r, thirdCert.CertPEM, resp.Certificate.GetCertPEM())
		require.Equal(r, secondRoots.ActiveRootID, resp.CARoots.GetActiveRootID())
	})
}

func TestIntroToken(t *testing.T) {
	tokenFile := testutil.TempFile(t, "intro-token")
	t.Cleanup(func() { os.Remove(tokenFile.Name()) })

	tokenFileEmpty := testutil.TempFile(t, "intro-token-empty")
	t.Cleanup(func() { os.Remove(tokenFileEmpty.Name()) })

	tokenFromFile := "8ae34d3a-8adf-446a-b236-69874597cb5b"
	tokenFromConfig := "3ad9b572-ea42-4e47-9cd0-53a398a98abf"
	require.NoError(t, ioutil.WriteFile(tokenFile.Name(), []byte(tokenFromFile), 0600))

	type testCase struct {
		config *config.RuntimeConfig
		err    string
		token  string
	}

	cases := map[string]testCase{
		"config": {
			config: &config.RuntimeConfig{
				AutoConfig: config.AutoConfig{
					IntroToken:     tokenFromConfig,
					IntroTokenFile: tokenFile.Name(),
				},
			},
			token: tokenFromConfig,
		},
		"file": {
			config: &config.RuntimeConfig{
				AutoConfig: config.AutoConfig{
					IntroTokenFile: tokenFile.Name(),
				},
			},
			token: tokenFromFile,
		},
		"file-empty": {
			config: &config.RuntimeConfig{
				AutoConfig: config.AutoConfig{
					IntroTokenFile: tokenFileEmpty.Name(),
				},
			},
			err: "intro_token_file did not contain any token",
		},
	}

	for name, tcase := range cases {
		t.Run(name, func(t *testing.T) {
			ac := AutoConfig{
				config: tcase.config,
			}

			token, err := ac.introToken()
			if tcase.err != "" {
				testutil.RequireErrorContains(t, err, tcase.err)
			} else {
				require.NoError(t, err)
				require.Equal(t, tcase.token, token)
			}
		})
	}

}