open-nomad/e2e/auth/auth_test.go

// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0

package auth

import (
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/hashicorp/nomad/api"
	"github.com/hashicorp/nomad/e2e/e2eutil"
	"github.com/hashicorp/nomad/helper/uuid"
	"github.com/shoenig/test/must"
)

var validPolicySpec = `
namespace "%s" {
  policy = "read"
  variables {
    path "test/*" {
      capabilities = [ "write", "destroy" ]
	}
  }
}

node {
  policy = "write"
}
`

// TestAuth verifies that we're correctly enforcing ACLs with different
// combinations of tokens, policies, API types, and topologies.
func TestAuth(t *testing.T) {

	// Wait until we have a usable cluster before running the tests.
	nomadClient := e2eutil.NomadClient(t)
	e2eutil.WaitForLeader(t, nomadClient)
	e2eutil.WaitForNodesReady(t, nomadClient, 1)

	nodes, _, err := nomadClient.Nodes().List(nil)
	must.NoError(t, err, must.Sprint("expected no error from root client"))
	must.Greater(t, 0, len(nodes))
	node, _, err := nomadClient.Nodes().Info(nodes[0].ID, nil)

	ns := uuid.Generate()
	validPolicyName := uuid.Generate()
	invalidPolicyName := uuid.Generate()

	setupAuthTest(t, nomadClient, ns, validPolicyName, invalidPolicyName)

	// Test cases that exercise requests directly to the server
	t.Run("AnonServerRequests", testAnonServerRequests(node, ns))
	t.Run("BogusServerRequests", testBogusServerRequests(nomadClient, node, ns))
	t.Run("InvalidPermissionsServerRequests",
		testInvalidPermissionsServerRequests(nomadClient, node, ns, invalidPolicyName))
	t.Run("ValidPermissionsServerRequests",
		testValidPermissionsServerRequests(nomadClient, node, ns, validPolicyName))

	// Test cases that exercise requests forwarded from the client
	t.Run("AnonClientRequests", testAnonClientRequests(node, ns))
	t.Run("BogusClientRequests", testBogusClientRequests(nomadClient, node, ns))
	t.Run("InvalidPermissionsClientRequests",
		testInvalidPermissionsClientRequests(nomadClient, node, ns, invalidPolicyName))
	t.Run("ValidPermissionsClientRequests",
		testValidPermissionsClientRequests(nomadClient, node, ns, validPolicyName))
}

func testAnonServerRequests(node *api.Node, ns string) func(t *testing.T) {
	return func(t *testing.T) {
		nomadClient := e2eutil.NomadClient(t)
		nomadClient.SetSecretID("")

		testReadNamespaceAPI(t, nomadClient, ns, "", true)
		testNodeAPI(t, nomadClient, node.ID, "", true)
		testVariablesAPI(t, nomadClient, ns, "", true, true)
	}
}

func testBogusServerRequests(nomadClient *api.Client,
	node *api.Node, ns string) func(t *testing.T) {
	return func(t *testing.T) {
		authToken := uuid.Generate()

		testReadNamespaceAPI(t, nomadClient, ns, authToken, true)
		testNodeAPI(t, nomadClient, node.ID, authToken, true)
		testVariablesAPI(t, nomadClient, ns, authToken, true, true)
	}
}

func testInvalidPermissionsServerRequests(nomadClient *api.Client,
	node *api.Node, ns, policyName string) func(t *testing.T) {
	return func(t *testing.T) {
		token, _, err := nomadClient.ACLTokens().Create(&api.ACLToken{
			Name:          policyName,
			Type:          "client",
			Policies:      []string{policyName},
			ExpirationTTL: time.Minute,
		}, nil)
		must.NoError(t, err)
		authToken := token.SecretID

		testReadNamespaceAPI(t, nomadClient, ns, authToken, true)
		testNodeAPI(t, nomadClient, node.ID, authToken, true)
		testVariablesAPI(t, nomadClient, ns, authToken, true, true)
	}
}

func testValidPermissionsServerRequests(nomadClient *api.Client,
	node *api.Node, ns, policyName string) func(t *testing.T) {
	return func(t *testing.T) {
		token, _, err := nomadClient.ACLTokens().Create(&api.ACLToken{
			Name:          policyName,
			Type:          "client",
			Policies:      []string{policyName},
			ExpirationTTL: time.Minute,
		}, nil)
		must.NoError(t, err)
		authToken := token.SecretID

		testReadNamespaceAPI(t, nomadClient, ns, authToken, false)
		testNodeAPI(t, nomadClient, node.ID, authToken, false)
		testVariablesAPI(t, nomadClient, ns, authToken, false, true)
	}
}

func testAnonClientRequests(node *api.Node, ns string) func(t *testing.T) {
	return func(t *testing.T) {
		config := api.DefaultConfig()
		config.Address = addressForNode(node)
		nomadClient, err := api.NewClient(config)
		nomadClient.SetSecretID("")
		must.NoError(t, err)

		testReadNamespaceAPI(t, nomadClient, ns, "", true)
		testNodeAPI(t, nomadClient, node.ID, "", true)
		testVariablesAPI(t, nomadClient, ns, "", true, true)
	}
}

func testBogusClientRequests(rootClient *api.Client,
	node *api.Node, ns string) func(t *testing.T) {
	return func(t *testing.T) {
		config := api.DefaultConfig()
		config.Address = addressForNode(node)
		nomadClient, err := api.NewClient(config)
		must.NoError(t, err)

		authToken := uuid.Generate()

		testReadNamespaceAPI(t, nomadClient, ns, authToken, true)
		testNodeAPI(t, nomadClient, node.ID, authToken, true)
		testVariablesAPI(t, nomadClient, ns, authToken, true, true)
	}
}

func testInvalidPermissionsClientRequests(rootClient *api.Client,
	node *api.Node, ns, policyName string) func(t *testing.T) {
	return func(t *testing.T) {
		token, _, err := rootClient.ACLTokens().Create(&api.ACLToken{
			Name:          policyName,
			Type:          "client",
			Policies:      []string{policyName},
			ExpirationTTL: time.Minute,
		}, nil)
		must.NoError(t, err)

		config := api.DefaultConfig()
		config.Address = addressForNode(node)
		nomadClient, err := api.NewClient(config)
		must.NoError(t, err)

		authToken := token.SecretID

		testReadNamespaceAPI(t, nomadClient, ns, authToken, true)
		testNodeAPI(t, nomadClient, node.ID, authToken, true)
		testVariablesAPI(t, nomadClient, ns, authToken, true, true)
	}
}

func testValidPermissionsClientRequests(rootClient *api.Client,
	node *api.Node, ns, policyName string) func(t *testing.T) {
	return func(t *testing.T) {
		token, _, err := rootClient.ACLTokens().Create(&api.ACLToken{
			Name:          policyName,
			Type:          "client",
			Policies:      []string{policyName},
			ExpirationTTL: time.Minute,
		}, nil)
		must.NoError(t, err)

		config := api.DefaultConfig()
		config.Address = addressForNode(node)
		nomadClient, err := api.NewClient(config)
		must.NoError(t, err)

		authToken := token.SecretID

		testReadNamespaceAPI(t, nomadClient, ns, authToken, false)
		testNodeAPI(t, nomadClient, node.ID, authToken, false)
		testVariablesAPI(t, nomadClient, ns, authToken, false, true)
	}
}

// testReadNamespaceAPI exercises an API that requires any namespace capability
func testReadNamespaceAPI(t *testing.T, nomadClient *api.Client, ns, authToken string, expectErr bool) {
	t.Helper()
	opts := &api.QueryOptions{AuthToken: authToken}
	_, _, err := nomadClient.Namespaces().Info(ns, opts)
	if expectErr {
		must.Error(t, err, must.Sprint("expected error when reading namespace"))
	} else {
		must.NoError(t, err, must.Sprint("expected no error reading namespace"))
	}
}

// testNodeAPI exercises an API that requires the node:write permission
func testNodeAPI(t *testing.T, nomadClient *api.Client, nodeID, authToken string, expectErr bool) {
	t.Helper()
	opts := &api.WriteOptions{AuthToken: authToken}
	_, _, err := nomadClient.Nodes().ForceEvaluate(nodeID, opts)
	if expectErr {
		must.Error(t, err, must.Sprint("expected error when force-evaluating node"))
	} else {
		must.NoError(t, err, must.Sprint("expected no error force-evaluating node"))
	}
}

// testVariablesAPI exercises an API that requires namespace capabilities for
// variables
func testVariablesAPI(t *testing.T, nomadClient *api.Client, ns, authToken string, expectErrTestPath, expectErrOutsidePath bool) {
	t.Helper()
	opts := &api.WriteOptions{Namespace: ns, AuthToken: authToken}

	_, _, err := nomadClient.Variables().Create(&api.Variable{
		Namespace: ns,
		Path:      "test/" + t.Name(),
		Items:     map[string]string{"foo": t.Name()},
	}, opts)

	if expectErrTestPath {
		must.Error(t, err, must.Sprint("expected error when writing variable"))
	} else {
		must.NoError(t, err, must.Sprint("expected no error writing variable"))
	}
	t.Cleanup(func() {
		_, err := nomadClient.Variables().Delete("test/"+t.Name(), opts)
		if !expectErrTestPath {
			must.NoError(t, err, must.Sprint("expected no error cleaning up variable"))
		}
	})

	_, _, err = nomadClient.Variables().Create(&api.Variable{
		Namespace: ns,
		Path:      "other/" + t.Name(),
		Items:     map[string]string{"foo": t.Name()},
	}, opts)

	if expectErrOutsidePath {
		must.Error(t, err, must.Sprint("expected error when writing variable"))
	} else {
		must.NoError(t, err, must.Sprint("expected no error writing variable"))
	}
	t.Cleanup(func() {
		// no test should ever write this variable, so we don't expect delete to
		// work either but need it for cleanup just in case we did write it
		nomadClient.Variables().Delete("other/"+t.Name(), opts)
	})

}

func setupAuthTest(t *testing.T, nomadClient *api.Client,
	ns, validPolicyName, invalidPolicyName string) {
	t.Helper()

	_, err := nomadClient.Namespaces().Register(&api.Namespace{Name: ns}, nil)
	must.NoError(t, err, must.Sprint("expected no error when registering namespace"))

	t.Cleanup(func() {
		_, err := nomadClient.Namespaces().Delete(ns, nil)
		must.NoError(t, err, must.Sprint("expected no error cleaning up namespace"))
	})

	// Create a valid and useful policy
	_, err = nomadClient.ACLPolicies().Upsert(&api.ACLPolicy{
		Name:  validPolicyName,
		Rules: fmt.Sprintf(validPolicySpec, ns),
	}, nil)
	must.NoError(t, err, must.Sprint("expected no error when registering policy"))

	t.Cleanup(func() {
		_, err := nomadClient.ACLPolicies().Delete(validPolicyName, nil)
		must.NoError(t, err, must.Sprint("expected no error cleaning up ACL policy"))
	})

	// Create a useless policy
	_, err = nomadClient.ACLPolicies().Upsert(&api.ACLPolicy{
		Name:  invalidPolicyName,
		Rules: `plugin { policy = "read" }`,
	}, nil)
	must.NoError(t, err, must.Sprint("expected no error when registering policy"))

	t.Cleanup(func() {
		_, err := nomadClient.ACLPolicies().Delete(invalidPolicyName, nil)
		must.NoError(t, err, must.Sprint("expected no error cleaning up ACL policy"))
	})
}

// addressForNode is a hacky way of getting the address with or without
// mTLS. The test code can't read the api.Client's internals to see if we're in
// mTLS mode, so we assume if the environment is set up for mTLS that we're
// using it. We also need to make sure we're using the AWS public IP address for
// machines running in the nightly E2E environment, and that address isn't the
// advertised address
func addressForNode(node *api.Node) string {
	if publicIP, ok := node.Attributes["unique.platform.aws.public-ipv4"]; ok {
		if v := os.Getenv("NOMAD_CACERT"); v != "" {
			return fmt.Sprintf("https://%s:4646", publicIP)
		} else {
			return fmt.Sprintf("http://%s:4646", publicIP)
		}
	}

	if v := os.Getenv("NOMAD_CACERT"); v != "" {
		return fmt.Sprintf("https://%s", node.HTTPAddr)
	}
	return fmt.Sprintf("http://%s", node.HTTPAddr)
}