open-consul/agent/consul/operator_backend_test.go

package consul

import (
	"context"
	"testing"
	"time"

	"github.com/hashicorp/consul/acl"
	external "github.com/hashicorp/consul/agent/grpc-external"
	"github.com/hashicorp/consul/agent/structs"
	"github.com/hashicorp/consul/proto/private/pboperator"
	"github.com/hashicorp/consul/sdk/testutil/retry"
	"google.golang.org/grpc/credentials/insecure"

	"github.com/stretchr/testify/require"
	gogrpc "google.golang.org/grpc"

	"github.com/hashicorp/consul/sdk/testutil"
	"github.com/hashicorp/consul/testrpc"
)

func TestOperatorBackend_TransferLeader(t *testing.T) {
	t.Parallel()

	conf := testClusterConfig{
		Datacenter: "dc1",
		Servers:    3,
		ServerConf: func(config *Config) {
			config.RaftConfig.HeartbeatTimeout = 2 * time.Second
			config.RaftConfig.ElectionTimeout = 2 * time.Second
			config.RaftConfig.LeaderLeaseTimeout = 1 * time.Second
		},
	}

	nodes := newTestCluster(t, &conf)
	s1 := nodes.Servers[0]
	// Make sure a leader is elected
	testrpc.WaitForLeader(t, s1.RPC, "dc1")

	// Make a write call to server2 and make sure it gets forwarded to server1
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	t.Cleanup(cancel)

	// Dial server2 directly
	conn, err := gogrpc.DialContext(ctx, s1.config.RPCAddr.String(),
		gogrpc.WithContextDialer(newServerDialer(s1.config.RPCAddr.String())),
		gogrpc.WithTransportCredentials(insecure.NewCredentials()),
		gogrpc.WithBlock())
	require.NoError(t, err)
	t.Cleanup(func() { conn.Close() })

	operatorClient := pboperator.NewOperatorServiceClient(conn)

	testutil.RunStep(t, "transfer leader", func(t *testing.T) {
		beforeLeader, _ := s1.raft.LeaderWithID()
		require.NotEmpty(t, beforeLeader)
		// Do the grpc Write call to server2
		req := pboperator.TransferLeaderRequest{
			ID: "",
		}
		reply, err := operatorClient.TransferLeader(ctx, &req)
		require.NoError(t, err)
		require.True(t, reply.Success)
		testrpc.WaitForLeader(t, s1.RPC, "dc1")
		retry.Run(t, func(r *retry.R) {
			time.Sleep(1 * time.Second)
			afterLeader, _ := s1.raft.LeaderWithID()
			require.NotEmpty(r, afterLeader)
			require.NotEqual(r, afterLeader, beforeLeader)
		})

	})
}

func TestOperatorBackend_TransferLeaderWithACL(t *testing.T) {
	t.Parallel()

	conf := testClusterConfig{
		Datacenter: "dc1",
		Servers:    3,
		ServerConf: func(config *Config) {
			config.RaftConfig.HeartbeatTimeout = 2 * time.Second
			config.RaftConfig.ElectionTimeout = 2 * time.Second
			config.RaftConfig.LeaderLeaseTimeout = 1 * time.Second
			config.ACLsEnabled = true
			config.ACLInitialManagementToken = "root"
			config.ACLResolverSettings.ACLDefaultPolicy = "deny"
		},
	}

	nodes := newTestCluster(t, &conf)
	s1 := nodes.Servers[0]
	// Make sure a leader is elected
	testrpc.WaitForLeader(t, s1.RPC, "dc1")
	// Make a write call to server2 and make sure it gets forwarded to server1
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	t.Cleanup(cancel)

	// Dial server2 directly
	conn, err := gogrpc.DialContext(ctx, s1.config.RPCAddr.String(),
		gogrpc.WithContextDialer(newServerDialer(s1.config.RPCAddr.String())),
		gogrpc.WithTransportCredentials(insecure.NewCredentials()),
		gogrpc.WithBlock())
	require.NoError(t, err)
	t.Cleanup(func() { conn.Close() })

	operatorClient := pboperator.NewOperatorServiceClient(conn)

	codec := rpcClient(t, s1)
	rules := `operator = "write"`
	tokenWrite := createTokenWithPolicyNameFull(t, codec, "the-policy-write", rules, "root")
	rules = `operator = "read"`
	tokenRead := createToken(t, codec, rules)
	require.NoError(t, err)

	testutil.RunStep(t, "transfer leader no token", func(t *testing.T) {
		beforeLeader, _ := s1.raft.LeaderWithID()
		require.NotEmpty(t, beforeLeader)
		// Do the grpc Write call to server2
		req := pboperator.TransferLeaderRequest{
			ID: "",
		}
		reply, err := operatorClient.TransferLeader(ctx, &req)
		require.True(t, acl.IsErrPermissionDenied(err))
		require.Nil(t, reply)
		time.Sleep(1 * time.Second)
		testrpc.WaitForLeader(t, s1.RPC, "dc1")
		retry.RunWith(&retry.Timer{Wait: time.Second, Timeout: 3 * time.Second}, t, func(r *retry.R) {
			afterLeader, _ := s1.raft.LeaderWithID()
			require.NotEmpty(r, afterLeader)
			if afterLeader != beforeLeader {
				r.Fatalf("leader should have changed %s == %s", afterLeader, beforeLeader)
			}
		})

	})

	testutil.RunStep(t, "transfer leader operator read token", func(t *testing.T) {

		beforeLeader, _ := s1.raft.LeaderWithID()
		require.NotEmpty(t, beforeLeader)
		// Do the grpc Write call to server2
		req := pboperator.TransferLeaderRequest{
			ID: "",
		}

		ctxToken, err := external.ContextWithQueryOptions(ctx, structs.QueryOptions{Token: tokenRead})
		require.NoError(t, err)

		reply, err := operatorClient.TransferLeader(ctxToken, &req)
		require.True(t, acl.IsErrPermissionDenied(err))
		require.Nil(t, reply)
		testrpc.WaitForLeader(t, s1.RPC, "dc1")
		retry.RunWith(&retry.Timer{Wait: time.Second, Timeout: 3 * time.Second}, t, func(r *retry.R) {
			afterLeader, _ := s1.raft.LeaderWithID()
			require.NotEmpty(r, afterLeader)
			if afterLeader != beforeLeader {
				r.Fatalf("leader should have changed %s == %s", afterLeader, beforeLeader)
			}
		})
	})

	testutil.RunStep(t, "transfer leader operator write token", func(t *testing.T) {

		beforeLeader, _ := s1.raft.LeaderWithID()
		require.NotEmpty(t, beforeLeader)
		// Do the grpc Write call to server2
		req := pboperator.TransferLeaderRequest{
			ID: "",
		}
		ctxToken, err := external.ContextWithQueryOptions(ctx, structs.QueryOptions{Token: tokenWrite.SecretID})
		require.NoError(t, err)
		reply, err := operatorClient.TransferLeader(ctxToken, &req)
		require.NoError(t, err)
		require.True(t, reply.Success)
		time.Sleep(1 * time.Second)
		testrpc.WaitForLeader(t, s1.RPC, "dc1")
		retry.RunWith(&retry.Timer{Wait: 2 * time.Second, Timeout: 6 * time.Second}, t, func(r *retry.R) {
			afterLeader, _ := s1.raft.LeaderWithID()
			require.NotEmpty(r, afterLeader)
			if afterLeader == beforeLeader {
				r.Fatalf("leader should have changed %s == %s", afterLeader, beforeLeader)
			}
		})
	})
}