open-nomad/command/agent/http_test.go

package agent

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/hashicorp/go-msgpack/codec"
	"github.com/hashicorp/nomad/acl"
	"github.com/hashicorp/nomad/api"
	"github.com/hashicorp/nomad/ci"
	"github.com/hashicorp/nomad/helper/pointer"
	"github.com/hashicorp/nomad/helper/testlog"
	"github.com/hashicorp/nomad/nomad/mock"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/nomad/structs/config"
	"github.com/hashicorp/nomad/testutil"
	"github.com/shoenig/test/must"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// makeHTTPServer returns a test server whose logs will be written to
// the passed writer. If the writer is nil, the logs are written to stderr.
func makeHTTPServer(t testing.TB, cb func(c *Config)) *TestAgent {
	return NewTestAgent(t, t.Name(), cb)
}

func BenchmarkHTTPRequests(b *testing.B) {
	s := makeHTTPServer(b, func(c *Config) {
		c.Client.Enabled = false
	})
	defer s.Shutdown()

	job := mock.Job()
	var allocs []*structs.Allocation
	count := 1000
	for i := 0; i < count; i++ {
		alloc := mock.Alloc()
		alloc.Job = job
		alloc.JobID = job.ID
		alloc.Name = fmt.Sprintf("my-job.web[%d]", i)
		allocs = append(allocs, alloc)
	}

	handler := func(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
		return allocs[:count], nil
	}
	b.ResetTimer()

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			resp := httptest.NewRecorder()
			req, _ := http.NewRequest("GET", "/v1/kv/key", nil)
			s.Server.wrap(handler)(resp, req)
		}
	})
}

func TestMultipleInterfaces(t *testing.T) {
	ci.Parallel(t)

	httpIps := []string{"127.0.0.1", "127.0.0.2"}

	s := makeHTTPServer(t, func(c *Config) {
		c.Addresses.HTTP = strings.Join(httpIps, " ")
		c.ACL.Enabled = true
	})
	defer s.Shutdown()

	httpPort := s.ports[0]
	for _, ip := range httpIps {
		resp, err := http.Get(fmt.Sprintf("http://%s:%d/", ip, httpPort))

		assert.Nil(t, err)
		assert.Equal(t, resp.StatusCode, 200)
	}
}

// TestRootFallthrough tests rootFallthrough handler to
// verify redirect and 404 behavior
func TestRootFallthrough(t *testing.T) {
	ci.Parallel(t)

	cases := []struct {
		desc             string
		path             string
		expectedPath     string
		expectedRawQuery string
		expectedCode     int
	}{
		{
			desc:         "unknown endpoint 404s",
			path:         "/v1/unknown/endpoint",
			expectedCode: 404,
		},
		{
			desc:         "root path redirects to ui",
			path:         "/",
			expectedPath: "/ui/",
			expectedCode: 307,
		},
		{
			desc:             "root path with one-time token redirects to ui",
			path:             "/?ott=whatever",
			expectedPath:     "/ui/",
			expectedRawQuery: "ott=whatever",
			expectedCode:     307,
		},
	}

	s := makeHTTPServer(t, nil)
	defer s.Shutdown()

	// setup a client that doesn't follow redirects
	client := &http.Client{
		CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
			return http.ErrUseLastResponse
		},
	}

	for _, tc := range cases {
		t.Run(tc.desc, func(t *testing.T) {

			reqURL := fmt.Sprintf("http://%s%s", s.Agent.config.AdvertiseAddrs.HTTP, tc.path)

			resp, err := client.Get(reqURL)
			require.NoError(t, err)
			require.Equal(t, tc.expectedCode, resp.StatusCode)

			if tc.expectedPath != "" {
				loc, err := resp.Location()
				require.NoError(t, err)
				require.Equal(t, tc.expectedPath, loc.Path)
				require.Equal(t, tc.expectedRawQuery, loc.RawQuery)
			}
		})
	}
}

func TestSetIndex(t *testing.T) {
	ci.Parallel(t)
	resp := httptest.NewRecorder()
	setIndex(resp, 1000)
	header := resp.Header().Get("X-Nomad-Index")
	if header != "1000" {
		t.Fatalf("Bad: %v", header)
	}
	setIndex(resp, 2000)
	if v := resp.Header()["X-Nomad-Index"]; len(v) != 1 {
		t.Fatalf("bad: %#v", v)
	}
}

func TestSetKnownLeader(t *testing.T) {
	ci.Parallel(t)
	resp := httptest.NewRecorder()
	setKnownLeader(resp, true)
	header := resp.Header().Get("X-Nomad-KnownLeader")
	if header != "true" {
		t.Fatalf("Bad: %v", header)
	}
	resp = httptest.NewRecorder()
	setKnownLeader(resp, false)
	header = resp.Header().Get("X-Nomad-KnownLeader")
	if header != "false" {
		t.Fatalf("Bad: %v", header)
	}
}

func TestSetLastContact(t *testing.T) {
	ci.Parallel(t)
	resp := httptest.NewRecorder()
	setLastContact(resp, 123456*time.Microsecond)
	header := resp.Header().Get("X-Nomad-LastContact")
	if header != "123" {
		t.Fatalf("Bad: %v", header)
	}
}

func TestSetMeta(t *testing.T) {
	ci.Parallel(t)
	meta := structs.QueryMeta{
		Index:       1000,
		KnownLeader: true,
		LastContact: 123456 * time.Microsecond,
	}
	resp := httptest.NewRecorder()
	setMeta(resp, &meta)
	header := resp.Header().Get("X-Nomad-Index")
	if header != "1000" {
		t.Fatalf("Bad: %v", header)
	}
	header = resp.Header().Get("X-Nomad-KnownLeader")
	if header != "true" {
		t.Fatalf("Bad: %v", header)
	}
	header = resp.Header().Get("X-Nomad-LastContact")
	if header != "123" {
		t.Fatalf("Bad: %v", header)
	}
}

func TestSetHeaders(t *testing.T) {
	ci.Parallel(t)
	s := makeHTTPServer(t, nil)
	s.Agent.config.HTTPAPIResponseHeaders = map[string]string{"foo": "bar"}
	defer s.Shutdown()

	resp := httptest.NewRecorder()
	handler := func(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
		return &structs.Job{Name: "foo"}, nil
	}

	req, _ := http.NewRequest("GET", "/v1/kv/key", nil)
	s.Server.wrap(handler)(resp, req)
	header := resp.Header().Get("foo")

	if header != "bar" {
		t.Fatalf("expected header: %v, actual: %v", "bar", header)
	}

}

func TestContentTypeIsJSON(t *testing.T) {
	ci.Parallel(t)
	s := makeHTTPServer(t, nil)
	defer s.Shutdown()

	resp := httptest.NewRecorder()

	handler := func(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
		return &structs.Job{Name: "foo"}, nil
	}

	req, _ := http.NewRequest("GET", "/v1/kv/key", nil)
	s.Server.wrap(handler)(resp, req)

	contentType := resp.Header().Get("Content-Type")

	if contentType != "application/json" {
		t.Fatalf("Content-Type header was not 'application/json'")
	}
}

func TestWrapNonJSON(t *testing.T) {
	ci.Parallel(t)
	s := makeHTTPServer(t, nil)
	defer s.Shutdown()

	resp := httptest.NewRecorder()

	handler := func(resp http.ResponseWriter, req *http.Request) ([]byte, error) {
		return []byte("test response"), nil
	}

	req, _ := http.NewRequest("GET", "/v1/kv/key", nil)
	s.Server.wrapNonJSON(handler)(resp, req)

	respBody, _ := io.ReadAll(resp.Body)
	require.Equal(t, respBody, []byte("test response"))

}

func TestWrapNonJSON_Error(t *testing.T) {
	ci.Parallel(t)
	s := makeHTTPServer(t, nil)
	defer s.Shutdown()

	handlerRPCErr := func(resp http.ResponseWriter, req *http.Request) ([]byte, error) {
		return nil, structs.NewErrRPCCoded(404, "not found")
	}

	handlerCodedErr := func(resp http.ResponseWriter, req *http.Request) ([]byte, error) {
		return nil, CodedError(422, "unprocessable")
	}

	// RPC coded error
	{
		resp := httptest.NewRecorder()
		req, _ := http.NewRequest("GET", "/v1/kv/key", nil)
		s.Server.wrapNonJSON(handlerRPCErr)(resp, req)
		respBody, _ := io.ReadAll(resp.Body)
		require.Equal(t, []byte("not found"), respBody)
		require.Equal(t, 404, resp.Code)
	}

	// CodedError
	{
		resp := httptest.NewRecorder()
		req, _ := http.NewRequest("GET", "/v1/kv/key", nil)
		s.Server.wrapNonJSON(handlerCodedErr)(resp, req)
		respBody, _ := io.ReadAll(resp.Body)
		require.Equal(t, []byte("unprocessable"), respBody)
		require.Equal(t, 422, resp.Code)
	}

}

func TestPrettyPrint(t *testing.T) {
	ci.Parallel(t)
	testPrettyPrint("pretty=1", true, t)
}

func TestPrettyPrintOff(t *testing.T) {
	ci.Parallel(t)
	testPrettyPrint("pretty=0", false, t)
}

func TestPrettyPrintBare(t *testing.T) {
	ci.Parallel(t)
	testPrettyPrint("pretty", true, t)
}

func testPrettyPrint(pretty string, prettyFmt bool, t *testing.T) {
	s := makeHTTPServer(t, nil)
	defer s.Shutdown()

	r := &structs.Job{Name: "foo"}

	resp := httptest.NewRecorder()
	handler := func(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
		return r, nil
	}

	urlStr := "/v1/job/foo?" + pretty
	req, _ := http.NewRequest("GET", urlStr, nil)
	s.Server.wrap(handler)(resp, req)

	var expected bytes.Buffer
	var err error
	if prettyFmt {
		enc := codec.NewEncoder(&expected, structs.JsonHandlePretty)
		err = enc.Encode(r)
		expected.WriteByte('\n')
	} else {
		enc := codec.NewEncoder(&expected, structs.JsonHandleWithExtensions)
		err = enc.Encode(r)
	}
	if err != nil {
		t.Fatalf("failed to encode: %v", err)
	}
	actual, err := io.ReadAll(resp.Body)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !bytes.Equal(expected.Bytes(), actual) {
		t.Fatalf("bad:\nexpected:\t%q\nactual:\t\t%q", expected.String(), string(actual))
	}
}

func TestPermissionDenied(t *testing.T) {
	s := makeHTTPServer(t, func(c *Config) {
		c.ACL.Enabled = true
	})
	defer s.Shutdown()

	{
		resp := httptest.NewRecorder()
		handler := func(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
			return nil, structs.ErrPermissionDenied
		}

		req, _ := http.NewRequest("GET", "/v1/job/foo", nil)
		s.Server.wrap(handler)(resp, req)
		assert.Equal(t, resp.Code, 403)
	}

	// When remote RPC is used the errors have "rpc error: " prependend
	{
		resp := httptest.NewRecorder()
		handler := func(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
			return nil, fmt.Errorf("rpc error: %v", structs.ErrPermissionDenied)
		}

		req, _ := http.NewRequest("GET", "/v1/job/foo", nil)
		s.Server.wrap(handler)(resp, req)
		assert.Equal(t, resp.Code, 403)
	}
}

func TestTokenNotFound(t *testing.T) {
	s := makeHTTPServer(t, func(c *Config) {
		c.ACL.Enabled = true
	})
	defer s.Shutdown()

	resp := httptest.NewRecorder()
	handler := func(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
		return nil, structs.ErrTokenNotFound
	}

	urlStr := "/v1/job/foo"
	req, _ := http.NewRequest("GET", urlStr, nil)
	s.Server.wrap(handler)(resp, req)
	assert.Equal(t, resp.Code, 403)
}

func TestParseWait(t *testing.T) {
	ci.Parallel(t)
	resp := httptest.NewRecorder()
	var b structs.QueryOptions

	req, err := http.NewRequest("GET",
		"/v1/catalog/nodes?wait=60s&index=1000", nil)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	if d := parseWait(resp, req, &b); d {
		t.Fatalf("unexpected done")
	}

	if b.MinQueryIndex != 1000 {
		t.Fatalf("Bad: %v", b)
	}
	if b.MaxQueryTime != 60*time.Second {
		t.Fatalf("Bad: %v", b)
	}
}

func TestParseWait_InvalidTime(t *testing.T) {
	ci.Parallel(t)
	resp := httptest.NewRecorder()
	var b structs.QueryOptions

	req, err := http.NewRequest("GET",
		"/v1/catalog/nodes?wait=60foo&index=1000", nil)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	if d := parseWait(resp, req, &b); !d {
		t.Fatalf("expected done")
	}

	if resp.Code != 400 {
		t.Fatalf("bad code: %v", resp.Code)
	}
}

func TestParseWait_InvalidIndex(t *testing.T) {
	ci.Parallel(t)
	resp := httptest.NewRecorder()
	var b structs.QueryOptions

	req, err := http.NewRequest("GET",
		"/v1/catalog/nodes?wait=60s&index=foo", nil)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	if d := parseWait(resp, req, &b); !d {
		t.Fatalf("expected done")
	}

	if resp.Code != 400 {
		t.Fatalf("bad code: %v", resp.Code)
	}
}

func TestParseConsistency(t *testing.T) {
	ci.Parallel(t)
	var b structs.QueryOptions
	var resp *httptest.ResponseRecorder

	testCases := [2]string{"/v1/catalog/nodes?stale", "/v1/catalog/nodes?stale=true"}
	for _, urlPath := range testCases {
		req, err := http.NewRequest("GET", urlPath, nil)
		must.NoError(t, err)
		resp = httptest.NewRecorder()
		parseConsistency(resp, req, &b)
		must.True(t, b.AllowStale)
	}

	req, err := http.NewRequest("GET", "/v1/catalog/nodes?stale=false", nil)
	must.NoError(t, err)
	resp = httptest.NewRecorder()
	parseConsistency(resp, req, &b)
	must.False(t, b.AllowStale)

	req, err = http.NewRequest("GET", "/v1/catalog/nodes?stale=random", nil)
	must.NoError(t, err)
	resp = httptest.NewRecorder()
	parseConsistency(resp, req, &b)
	must.False(t, b.AllowStale)
	must.EqOp(t, 400, resp.Code)

	b = structs.QueryOptions{}
	req, err = http.NewRequest("GET", "/v1/catalog/nodes?consistent", nil)
	must.NoError(t, err)

	resp = httptest.NewRecorder()
	parseConsistency(resp, req, &b)
	must.False(t, b.AllowStale)
}

func TestParseRegion(t *testing.T) {
	ci.Parallel(t)
	s := makeHTTPServer(t, nil)
	defer s.Shutdown()

	req, err := http.NewRequest("GET",
		"/v1/jobs?region=foo", nil)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	var region string
	s.Server.parseRegion(req, &region)
	if region != "foo" {
		t.Fatalf("bad %s", region)
	}

	region = ""
	req, err = http.NewRequest("GET", "/v1/jobs", nil)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	s.Server.parseRegion(req, &region)
	if region != "global" {
		t.Fatalf("bad %s", region)
	}
}

func TestParseToken(t *testing.T) {
	ci.Parallel(t)
	s := makeHTTPServer(t, nil)
	defer s.Shutdown()

	cases := []struct {
		Name          string
		HeaderKey     string
		HeaderValue   string
		ExpectedToken string
	}{
		{
			Name:          "Parses token from X-Nomad-Token",
			HeaderKey:     "X-Nomad-Token",
			HeaderValue:   " foobar",
			ExpectedToken: "foobar",
		},
		{
			Name:          "Parses token from bearer authentication",
			HeaderKey:     "Authorization",
			HeaderValue:   "Bearer foobar",
			ExpectedToken: "foobar",
		},
		{
			Name:          "Fails to parse token from bad bearer authentication",
			HeaderKey:     "Authorization",
			HeaderValue:   "foobar",
			ExpectedToken: "",
		},
	}

	for i := range cases {
		tc := cases[i]
		t.Run(tc.Name, func(t *testing.T) {
			req, err := http.NewRequest("GET", "/v1/jobs", nil)
			req.Header.Add(tc.HeaderKey, tc.HeaderValue)
			if err != nil {
				t.Fatalf("err: %v", err)
			}

			var token string
			s.Server.parseToken(req, &token)
			if token != tc.ExpectedToken {
				t.Fatalf("bad %s", token)
			}
		})
	}
}

func TestParseBool(t *testing.T) {
	ci.Parallel(t)

	cases := []struct {
		Input    string
		Expected *bool
		Err      bool // true if an error should be expected
	}{
		{
			Input:    "",
			Expected: nil,
		},
		{
			Input:    "true",
			Expected: pointer.Of(true),
		},
		{
			Input:    "false",
			Expected: pointer.Of(false),
		},
		{
			Input: "1234",
			Err:   true,
		},
	}

	for i := range cases {
		tc := cases[i]
		t.Run("Input-"+tc.Input, func(t *testing.T) {
			testURL, err := url.Parse("http://localhost/foo?resources=" + tc.Input)
			require.NoError(t, err)
			req := &http.Request{
				URL: testURL,
			}

			result, err := parseBool(req, "resources")
			if tc.Err {
				require.Error(t, err)
				require.Nil(t, result)
			} else {
				require.NoError(t, err)
				require.Equal(t, tc.Expected, result)
			}
		})
	}
}

func Test_parseInt(t *testing.T) {
	ci.Parallel(t)

	cases := []struct {
		Input    string
		Expected *int
		Err      bool
	}{
		{
			Input:    "",
			Expected: nil,
		},
		{
			Input:    "13",
			Expected: pointer.Of(13),
		},
		{
			Input:    "99",
			Expected: pointer.Of(99),
		},
		{
			Input: "ten",
			Err:   true,
		},
	}

	for i := range cases {
		tc := cases[i]
		t.Run("Input-"+tc.Input, func(t *testing.T) {
			testURL, err := url.Parse("http://localhost/foo?eval_priority=" + tc.Input)
			require.NoError(t, err)
			req := &http.Request{
				URL: testURL,
			}

			result, err := parseInt(req, "eval_priority")
			if tc.Err {
				require.Error(t, err)
				require.Nil(t, result)
			} else {
				require.NoError(t, err)
				require.Equal(t, tc.Expected, result)
			}
		})
	}
}

func TestParsePagination(t *testing.T) {
	ci.Parallel(t)
	s := makeHTTPServer(t, nil)
	defer s.Shutdown()

	cases := []struct {
		Input             string
		ExpectedNextToken string
		ExpectedPerPage   int32
	}{
		{
			Input: "",
		},
		{
			Input:             "next_token=a&per_page=3",
			ExpectedNextToken: "a",
			ExpectedPerPage:   3,
		},
		{
			Input:             "next_token=a&next_token=b",
			ExpectedNextToken: "a",
		},
		{
			Input: "per_page=a",
		},
	}

	for i := range cases {
		tc := cases[i]
		t.Run("Input-"+tc.Input, func(t *testing.T) {

			req, err := http.NewRequest("GET",
				"/v1/volumes/csi/external?"+tc.Input, nil)

			require.NoError(t, err)
			opts := &structs.QueryOptions{}
			parsePagination(req, opts)
			require.Equal(t, tc.ExpectedNextToken, opts.NextToken)
			require.Equal(t, tc.ExpectedPerPage, opts.PerPage)
		})
	}
}

// TestHTTP_VerifyHTTPSClient asserts that a client certificate signed by the
// appropriate CA is required when VerifyHTTPSClient=true.
func TestHTTP_VerifyHTTPSClient(t *testing.T) {
	ci.Parallel(t)
	const (
		cafile  = "../../helper/tlsutil/testdata/ca.pem"
		foocert = "../../helper/tlsutil/testdata/nomad-foo.pem"
		fookey  = "../../helper/tlsutil/testdata/nomad-foo-key.pem"
	)
	s := makeHTTPServer(t, func(c *Config) {
		c.Region = "foo" // match the region on foocert
		c.TLSConfig = &config.TLSConfig{
			EnableHTTP:        true,
			VerifyHTTPSClient: true,
			CAFile:            cafile,
			CertFile:          foocert,
			KeyFile:           fookey,
		}
		c.LogLevel = "off"
	})
	defer s.Shutdown()

	reqURL := fmt.Sprintf("https://%s/v1/agent/self", s.Agent.config.AdvertiseAddrs.HTTP)

	// FAIL: Requests that expect 127.0.0.1 as the name should fail
	resp, err := http.Get(reqURL)
	if err == nil {
		resp.Body.Close()
		t.Fatalf("expected non-nil error but received: %v", resp.StatusCode)
	}
	urlErr, ok := err.(*url.Error)
	if !ok {
		t.Fatalf("expected a *url.Error but received: %T -> %v", err, err)
	}

	cveErr := (urlErr.Err.(*tls.CertificateVerificationError)).Err
	hostErr, ok := cveErr.(x509.HostnameError)
	if !ok {
		t.Fatalf("expected a x509.HostnameError but received: %T -> %v", urlErr.Err, urlErr.Err)
	}
	if expected := "127.0.0.1"; hostErr.Host != expected {
		t.Fatalf("expected hostname on error to be %q but found %q", expected, hostErr.Host)
	}

	// FAIL: Requests that specify a valid hostname but not the CA should
	// fail
	tlsConf := &tls.Config{
		ServerName: "client.regionFoo.nomad",
	}
	transport := &http.Transport{TLSClientConfig: tlsConf}
	client := &http.Client{Transport: transport}
	req, err := http.NewRequest("GET", reqURL, nil)
	if err != nil {
		t.Fatalf("error creating request: %v", err)
	}
	resp, err = client.Do(req)
	if err == nil {
		resp.Body.Close()
		t.Fatalf("expected non-nil error but received: %v", resp.StatusCode)
	}
	urlErr, ok = err.(*url.Error)
	if !ok {
		t.Fatalf("expected a *url.Error but received: %T -> %v", err, err)
	}

	cveErr = (urlErr.Err.(*tls.CertificateVerificationError)).Err
	_, ok = cveErr.(x509.UnknownAuthorityError)
	if !ok {
		t.Fatalf("expected a x509.UnknownAuthorityError but received: %T -> %v", urlErr.Err, urlErr.Err)
	}

	// FAIL: Requests that specify a valid hostname and CA cert but lack a
	// client certificate should fail
	cacertBytes, err := os.ReadFile(cafile)
	if err != nil {
		t.Fatalf("error reading cacert: %v", err)
	}
	tlsConf.RootCAs = x509.NewCertPool()
	tlsConf.RootCAs.AppendCertsFromPEM(cacertBytes)
	req, err = http.NewRequest("GET", reqURL, nil)
	if err != nil {
		t.Fatalf("error creating request: %v", err)
	}
	resp, err = client.Do(req)
	if err == nil {
		resp.Body.Close()
		t.Fatalf("expected non-nil error but received: %v", resp.StatusCode)
	}
	urlErr, ok = err.(*url.Error)
	if !ok {
		t.Fatalf("expected a *url.Error but received: %T -> %v", err, err)
	}

	var opErr *net.OpError
	ok = errors.As(urlErr.Err, &opErr)
	if !ok {
		t.Fatalf("expected a *net.OpErr but received: %T -> %v", urlErr.Err, urlErr.Err)
	}
	const badCertificate = "tls: bad certificate" // from crypto/tls/alert.go:52 and RFC 5246 § A.3
	if opErr.Err.Error() != badCertificate {
		t.Fatalf("expected tls.alert bad_certificate but received: %q", opErr.Err.Error())
	}

	// PASS: Requests that specify a valid hostname, CA cert, and client
	// certificate succeed.
	tlsConf.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
		c, err := tls.LoadX509KeyPair(foocert, fookey)
		if err != nil {
			return nil, err
		}
		return &c, nil
	}
	transport = &http.Transport{TLSClientConfig: tlsConf}
	client = &http.Client{Transport: transport}
	req, err = http.NewRequest("GET", reqURL, nil)
	if err != nil {
		t.Fatalf("error creating request: %v", err)
	}
	resp, err = client.Do(req)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	resp.Body.Close()
	if resp.StatusCode != 200 {
		t.Fatalf("expected 200 status code but got: %d", resp.StatusCode)
	}
}

func TestHTTP_VerifyHTTPSClient_AfterConfigReload(t *testing.T) {
	ci.Parallel(t)
	assert := assert.New(t)

	const (
		cafile   = "../../helper/tlsutil/testdata/ca.pem"
		foocert  = "../../helper/tlsutil/testdata/nomad-bad.pem"
		fookey   = "../../helper/tlsutil/testdata/nomad-bad-key.pem"
		foocert2 = "../../helper/tlsutil/testdata/nomad-foo.pem"
		fookey2  = "../../helper/tlsutil/testdata/nomad-foo-key.pem"
	)

	agentConfig := &Config{
		TLSConfig: &config.TLSConfig{
			EnableHTTP:        true,
			VerifyHTTPSClient: true,
			CAFile:            cafile,
			CertFile:          foocert,
			KeyFile:           fookey,
		},
	}

	newConfig := &Config{
		TLSConfig: &config.TLSConfig{
			EnableHTTP:        true,
			VerifyHTTPSClient: true,
			CAFile:            cafile,
			CertFile:          foocert2,
			KeyFile:           fookey2,
		},
	}

	s := makeHTTPServer(t, func(c *Config) {
		c.TLSConfig = agentConfig.TLSConfig
	})
	defer s.Shutdown()

	// Make an initial request that should fail.
	// Requests that specify a valid hostname, CA cert, and client
	// certificate succeed.
	tlsConf := &tls.Config{
		ServerName: "client.regionFoo.nomad",
		RootCAs:    x509.NewCertPool(),
		GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
			c, err := tls.LoadX509KeyPair(foocert, fookey)
			if err != nil {
				return nil, err
			}
			return &c, nil
		},
	}

	// HTTPS request should succeed
	httpsReqURL := fmt.Sprintf("https://%s/v1/agent/self", s.Agent.config.AdvertiseAddrs.HTTP)

	cacertBytes, err := os.ReadFile(cafile)
	assert.Nil(err)
	tlsConf.RootCAs.AppendCertsFromPEM(cacertBytes)

	transport := &http.Transport{TLSClientConfig: tlsConf}
	client := &http.Client{Transport: transport}
	req, err := http.NewRequest("GET", httpsReqURL, nil)
	assert.Nil(err)

	// Check that we get an error that the certificate isn't valid for the
	// region we are contacting.
	_, err = client.Do(req)
	assert.Contains(err.Error(), "certificate is valid for")

	// Reload the TLS configuration==
	assert.Nil(s.Agent.Reload(newConfig))

	// Requests that specify a valid hostname, CA cert, and client
	// certificate succeed.
	tlsConf = &tls.Config{
		ServerName: "client.regionFoo.nomad",
		RootCAs:    x509.NewCertPool(),
		GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
			c, err := tls.LoadX509KeyPair(foocert2, fookey2)
			if err != nil {
				return nil, err
			}
			return &c, nil
		},
	}

	cacertBytes, err = os.ReadFile(cafile)
	assert.Nil(err)
	tlsConf.RootCAs.AppendCertsFromPEM(cacertBytes)

	transport = &http.Transport{TLSClientConfig: tlsConf}
	client = &http.Client{Transport: transport}
	req, err = http.NewRequest("GET", httpsReqURL, nil)
	assert.Nil(err)

	resp, err := client.Do(req)
	if assert.NoError(err) {
		resp.Body.Close()
		assert.Equal(resp.StatusCode, 200)
	}
}

// TestHTTPServer_Limits_Error asserts invalid Limits cause errors. This is the
// HTTP counterpart to TestAgent_ServerConfig_Limits_Error.
func TestHTTPServer_Limits_Error(t *testing.T) {
	ci.Parallel(t)

	cases := []struct {
		tls         bool
		timeout     string
		limit       *int
		expectedErr string
	}{
		{
			tls:         true,
			timeout:     "",
			limit:       nil,
			expectedErr: "error parsing https_handshake_timeout: ",
		},
		{
			tls:         false,
			timeout:     "",
			limit:       nil,
			expectedErr: "error parsing https_handshake_timeout: ",
		},
		{
			tls:         true,
			timeout:     "-1s",
			limit:       nil,
			expectedErr: "https_handshake_timeout must be >= 0",
		},
		{
			tls:         false,
			timeout:     "-1s",
			limit:       nil,
			expectedErr: "https_handshake_timeout must be >= 0",
		},
		{
			tls:         true,
			timeout:     "5s",
			limit:       pointer.Of(-1),
			expectedErr: "http_max_conns_per_client must be >= 0",
		},
		{
			tls:         false,
			timeout:     "5s",
			limit:       pointer.Of(-1),
			expectedErr: "http_max_conns_per_client must be >= 0",
		},
	}

	for i := range cases {
		tc := cases[i]
		name := fmt.Sprintf("%d-tls-%t-timeout-%s-limit-%v", i, tc.tls, tc.timeout, tc.limit)
		t.Run(name, func(t *testing.T) {
			ci.Parallel(t)

			conf := &Config{
				normalizedAddrs: &NormalizedAddrs{
					HTTP: []string{"localhost:0"}, // port is never used
				},
				TLSConfig: &config.TLSConfig{
					EnableHTTP: tc.tls,
				},
				Limits: config.Limits{
					HTTPSHandshakeTimeout: tc.timeout,
					HTTPMaxConnsPerClient: tc.limit,
				},
			}

			// Use a fake agent since the HTTP server should never start
			agent := &Agent{
				logger:     testlog.HCLogger(t),
				httpLogger: testlog.HCLogger(t),
				config:     conf,
			}

			srv, err := NewHTTPServers(agent, conf)
			require.Error(t, err)
			require.Nil(t, srv)
			require.Contains(t, err.Error(), tc.expectedErr)
		})
	}
}

func limitStr(limit *int) string {
	if limit == nil {
		return "none"
	}
	return strconv.Itoa(*limit)
}

// TestHTTPServer_Limits_OK asserts that all valid limits combinations
// (tls/timeout/conns) work.
func TestHTTPServer_Limits_OK(t *testing.T) {
	ci.Parallel(t)

	const (
		cafile   = "../../helper/tlsutil/testdata/ca.pem"
		foocert  = "../../helper/tlsutil/testdata/nomad-foo.pem"
		fookey   = "../../helper/tlsutil/testdata/nomad-foo-key.pem"
		maxConns = 10 // limit must be < this for testing
		bufSize  = 1  // enough to know if something was written
	)

	cases := []struct {
		tls           bool
		timeout       string
		limit         *int
		assertTimeout bool
		assertLimit   bool
	}{
		{
			tls:           false,
			timeout:       "5s",
			limit:         nil,
			assertTimeout: false,
			assertLimit:   false,
		},
		{
			tls:           true,
			timeout:       "5s",
			limit:         nil,
			assertTimeout: true,
			assertLimit:   false,
		},
		{
			tls:           false,
			timeout:       "0",
			limit:         nil,
			assertTimeout: false,
			assertLimit:   false,
		},
		{
			tls:           true,
			timeout:       "0",
			limit:         nil,
			assertTimeout: false,
			assertLimit:   false,
		},
		{
			tls:           false,
			timeout:       "0",
			limit:         pointer.Of(2),
			assertTimeout: false,
			assertLimit:   true,
		},
		{
			tls:           true,
			timeout:       "0",
			limit:         pointer.Of(2),
			assertTimeout: false,
			assertLimit:   true,
		},
		{
			tls:           false,
			timeout:       "5s",
			limit:         pointer.Of(2),
			assertTimeout: false,
			assertLimit:   true,
		},
		{
			tls:           true,
			timeout:       "5s",
			limit:         pointer.Of(2),
			assertTimeout: true,
			assertLimit:   true,
		},
	}

	assertTimeout := func(t *testing.T, a *TestAgent, assertTimeout bool, timeout string) {
		timeoutDeadline, err := time.ParseDuration(timeout)
		require.NoError(t, err)

		// Increase deadline to detect timeouts
		deadline := timeoutDeadline + time.Second

		conn, err := net.DialTimeout("tcp", a.Server.Addr, deadline)
		require.NoError(t, err)
		defer func() {
			require.NoError(t, conn.Close())
		}()

		buf := []byte{0}
		readDeadline := time.Now().Add(deadline)
		err = conn.SetReadDeadline(readDeadline)
		require.NoError(t, err)
		n, err := conn.Read(buf)
		require.Zero(t, n)
		if assertTimeout {
			// Server timeouts == EOF
			require.Equal(t, io.EOF, err)

			// Perform blocking query to assert timeout is not
			// enabled post-TLS-handshake.
			q := &api.QueryOptions{
				WaitIndex: 10000, // wait a looong time
				WaitTime:  deadline,
			}

			// Assertions don't require certificate validation
			conf := api.DefaultConfig()
			conf.Address = a.HTTPAddr()
			conf.TLSConfig.Insecure = true
			client, err := api.NewClient(conf)
			require.NoError(t, err)
			defer client.Close()

			// Assert a blocking query isn't timed out by the
			// handshake timeout
			jobs, meta, err := client.Jobs().List(q)
			require.NoError(t, err)
			require.Len(t, jobs, 0)
			require.Truef(t, meta.RequestTime >= deadline,
				"expected RequestTime (%s) >= Deadline (%s)",
				meta.RequestTime, deadline)

			return
		}

		// HTTP Server should *not* have timed out.
		// Now() should always be after the read deadline, but
		// isn't a sufficient assertion for correctness as slow
		// tests may cause this to be true even if the server
		// timed out.
		require.True(t, time.Now().After(readDeadline))

		require.Truef(t, errors.Is(err, os.ErrDeadlineExceeded),
			"error does not wrap os.ErrDeadlineExceeded: (%T) %v", err, err)
	}

	assertNoLimit := func(t *testing.T, addr string) {
		var err error

		// Create max connections
		conns := make([]net.Conn, maxConns)
		errCh := make(chan error, maxConns)
		for i := 0; i < maxConns; i++ {
			conns[i], err = net.DialTimeout("tcp", addr, 1*time.Second)
			require.NoError(t, err)

			go func(i int) {
				buf := []byte{0}
				readDeadline := time.Now().Add(1 * time.Second)
				err = conns[i].SetReadDeadline(readDeadline)
				require.NoError(t, err)
				n, err := conns[i].Read(buf)
				if n > 0 {
					errCh <- fmt.Errorf("n > 0: %d", n)
					return
				}
				errCh <- err
			}(i)
		}

		// Now assert each error is a clientside read deadline error
		for i := 0; i < maxConns; i++ {
			select {
			case <-time.After(2 * time.Second):
				t.Fatalf("timed out waiting for conn error %d", i)
			case err := <-errCh:
				require.Truef(t, errors.Is(err, os.ErrDeadlineExceeded),
					"error does not wrap os.ErrDeadlineExceeded: (%T) %v", err, err)
			}
		}

		for i := 0; i < maxConns; i++ {
			require.NoError(t, conns[i].Close())
		}
	}

	dial := func(t *testing.T, addr string, useTLS bool) (net.Conn, error) {
		if useTLS {
			cert, err := tls.LoadX509KeyPair(foocert, fookey)
			require.NoError(t, err)
			return tls.Dial("tcp", addr, &tls.Config{
				Certificates:       []tls.Certificate{cert},
				InsecureSkipVerify: true, // good enough
			})
		} else {
			return net.DialTimeout("tcp", addr, 1*time.Second)
		}
	}

	assertLimit := func(t *testing.T, addr string, limit int, useTLS bool) {
		var err error

		// Create limit connections
		conns := make([]net.Conn, limit)
		errCh := make(chan error, limit)
		for i := range conns {
			conn, err := dial(t, addr, useTLS)
			require.NoError(t, err)
			conns[i] = conn

			go func(i int) {
				buf := []byte{0}
				n, err := conns[i].Read(buf)
				if n > 0 {
					errCh <- fmt.Errorf("n > 0: %d", n)
					return
				}
				errCh <- err
			}(i)
		}

		select {
		case err := <-errCh:
			t.Fatalf("unexpected error from connection prior to limit: %T %v", err, err)
		case <-time.After(500 * time.Millisecond):
		}

		// Create a new connection that will go over the connection limit.
		limitConn, err := dial(t, addr, useTLS)
		require.NoError(t, err)

		response := "HTTP/1.1 429"
		buf := make([]byte, len(response))
		deadline := time.Now().Add(10 * time.Second)
		require.NoError(t, limitConn.SetReadDeadline(deadline))
		n, err := limitConn.Read(buf)
		require.Equal(t, response, string(buf))
		require.Nil(t, err)
		require.Equal(t, len(response), n)
		require.NoError(t, limitConn.Close())

		// Assert existing connections are ok
		require.Len(t, errCh, 0)

		// Cleanup
		for _, conn := range conns {
			require.NoError(t, conn.Close())
		}

		for range conns {
			err := <-errCh
			require.Contains(t, err.Error(), "use of closed network connection")
		}
	}

	for i := range cases {
		tc := cases[i]
		name := fmt.Sprintf("%d-tls-%t-timeout-%s-limit-%v", i, tc.tls, tc.timeout, limitStr(tc.limit))
		t.Run(name, func(t *testing.T) {
			ci.Parallel(t)

			if tc.limit != nil && *tc.limit >= maxConns {
				t.Fatalf("test fixture failure: cannot assert limit (%d) >= max (%d)", *tc.limit, maxConns)
			}

			s := makeHTTPServer(t, func(c *Config) {
				if tc.tls {
					c.TLSConfig = &config.TLSConfig{
						EnableHTTP: true,
						CAFile:     cafile,
						CertFile:   foocert,
						KeyFile:    fookey,
					}
				}
				c.Limits.HTTPSHandshakeTimeout = tc.timeout
				c.Limits.HTTPMaxConnsPerClient = tc.limit
				c.LogLevel = "ERROR"
			})
			defer s.Shutdown()

			assertTimeout(t, s, tc.assertTimeout, tc.timeout)

			if tc.assertLimit {
				// There's a race between assertTimeout(false) closing
				// its connection and the HTTP server noticing and
				// un-tracking it. Since there's no way to coordinate
				// when this occurs, sleeping is the only way to avoid
				// asserting limits before the timed out connection is
				// untracked.
				time.Sleep(1 * time.Second)

				assertLimit(t, s.Server.Addr, *tc.limit, tc.tls)
			} else {
				assertNoLimit(t, s.Server.Addr)
			}
		})
	}
}

func TestHTTPServer_ResolveToken(t *testing.T) {
	ci.Parallel(t)

	// Setup two servers, one with ACL enabled and another with ACL disabled.
	noACLServer := makeHTTPServer(t, func(c *Config) {
		c.ACL = &ACLConfig{Enabled: false}
	})
	defer noACLServer.Shutdown()

	ACLServer := makeHTTPServer(t, func(c *Config) {
		c.ACL = &ACLConfig{Enabled: true}
	})
	defer ACLServer.Shutdown()

	// Register sample token.
	state := ACLServer.Agent.server.State()
	token := mock.CreatePolicyAndToken(t, state, 1000, "node", mock.NodePolicy(acl.PolicyWrite))

	// Tests cases.
	t.Run("acl disabled", func(t *testing.T) {
		req := &http.Request{Body: http.NoBody}
		got, err := noACLServer.Server.ResolveToken(req)
		require.NoError(t, err)
		require.Nil(t, got)
	})

	t.Run("token not found", func(t *testing.T) {
		req := &http.Request{
			Body:   http.NoBody,
			Header: make(map[string][]string),
		}
		setToken(req, mock.ACLToken())
		got, err := ACLServer.Server.ResolveToken(req)
		require.Nil(t, got)
		require.Error(t, err)
		require.Contains(t, err.Error(), "ACL token not found")
	})

	t.Run("set token", func(t *testing.T) {
		req := &http.Request{
			Body:   http.NoBody,
			Header: make(map[string][]string),
		}
		setToken(req, token)
		got, err := ACLServer.Server.ResolveToken(req)
		require.NoError(t, err)
		require.NotNil(t, got)
		require.True(t, got.AllowNodeWrite())
	})
}

func Test_IsAPIClientError(t *testing.T) {
	ci.Parallel(t)

	trueCases := []int{400, 403, 404, 499}
	for _, c := range trueCases {
		require.Truef(t, isAPIClientError(c), "code: %v", c)
	}

	falseCases := []int{100, 300, 500, 501, 505}
	for _, c := range falseCases {
		require.Falsef(t, isAPIClientError(c), "code: %v", c)
	}
}

func Test_decodeBody(t *testing.T) {
	ci.Parallel(t)

	testCases := []struct {
		inputReq      *http.Request
		inputOut      interface{}
		expectedOut   interface{}
		expectedError error
		name          string
	}{
		{
			inputReq:      &http.Request{Body: http.NoBody},
			expectedError: errors.New("Request body is empty"),
			name:          "empty input request body",
		},
		{
			inputReq: &http.Request{Body: io.NopCloser(strings.NewReader(`{"foo":"bar"}`))},
			inputOut: &struct {
				Foo string `json:"foo"`
			}{},
			expectedOut: &struct {
				Foo string `json:"foo"`
			}{Foo: "bar"},
			expectedError: nil,
			name:          "populated request body and correct out",
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			actualError := decodeBody(tc.inputReq, tc.inputOut)
			assert.Equal(t, tc.expectedError, actualError, tc.name)
			assert.Equal(t, tc.expectedOut, tc.inputOut, tc.name)
		})
	}
}

// BenchmarkHTTPServer_JSONEncodingWithExtensions benchmarks the performance of
// encoding JSON objects using extensions
func BenchmarkHTTPServer_JSONEncodingWithExtensions(b *testing.B) {
	benchmarkJsonEncoding(b, structs.JsonHandleWithExtensions)
}

// BenchmarkHTTPServer_JSONEncodingWithoutExtensions benchmarks the performance of
// encoding JSON objects using extensions
func BenchmarkHTTPServer_JSONEncodingWithoutExtensions(b *testing.B) {
	benchmarkJsonEncoding(b, structs.JsonHandle)
}

func benchmarkJsonEncoding(b *testing.B, handle *codec.JsonHandle) {
	n := mock.Node()
	var buf bytes.Buffer

	enc := codec.NewEncoder(&buf, handle)
	for i := 0; i < b.N; i++ {
		buf.Reset()
		err := enc.Encode(n)
		require.NoError(b, err)
	}
}

func httpTest(t testing.TB, cb func(c *Config), f func(srv *TestAgent)) {
	s := makeHTTPServer(t, cb)
	defer s.Shutdown()
	testutil.WaitForLeader(t, s.Agent.RPC)
	f(s)
}

func httpACLTest(t testing.TB, cb func(c *Config), f func(srv *TestAgent)) {
	s := makeHTTPServer(t, func(c *Config) {
		c.ACL.Enabled = true
		if cb != nil {
			cb(c)
		}
	})
	defer s.Shutdown()
	testutil.WaitForLeader(t, s.Agent.RPC)
	f(s)
}

func setToken(req *http.Request, token *structs.ACLToken) {
	req.Header.Set("X-Nomad-Token", token.SecretID)
}

func setNamespace(req *http.Request, ns string) {
	q := req.URL.Query()
	q.Add("namespace", ns)
	req.URL.RawQuery = q.Encode()
}

func encodeReq(obj interface{}) io.ReadCloser {
	buf := bytes.NewBuffer(nil)
	enc := json.NewEncoder(buf)
	enc.Encode(obj)
	return io.NopCloser(buf)
}