package agent import ( "bytes" "encoding/json" "errors" "fmt" "io/ioutil" "net" "net/http" "net/http/httptest" "net/url" "os" "reflect" "strings" "sync" "syscall" "testing" "time" msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" "github.com/hashicorp/nomad/acl" "github.com/hashicorp/nomad/api" "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/pool" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestHTTP_AgentSelf(t *testing.T) { ci.Parallel(t) require := require.New(t) httpTest(t, nil, func(s *TestAgent) { // Make the HTTP request req, err := http.NewRequest("GET", "/v1/agent/self", nil) require.NoError(err) respW := httptest.NewRecorder() // Make the request obj, err := s.Server.AgentSelfRequest(respW, req) require.NoError(err) // Check the job self := obj.(agentSelf) require.NotNil(self.Config) require.NotNil(self.Config.ACL) require.NotEmpty(self.Stats) // Check the Vault config require.Empty(self.Config.Vault.Token) // Assign a Vault token and require it is redacted. s.Config.Vault.Token = "badc0deb-adc0-deba-dc0d-ebadc0debadc" respW = httptest.NewRecorder() obj, err = s.Server.AgentSelfRequest(respW, req) require.NoError(err) self = obj.(agentSelf) require.Equal("", self.Config.Vault.Token) // Assign a ReplicationToken token and require it is redacted. s.Config.ACL.ReplicationToken = "badc0deb-adc0-deba-dc0d-ebadc0debadc" respW = httptest.NewRecorder() obj, err = s.Server.AgentSelfRequest(respW, req) require.NoError(err) self = obj.(agentSelf) require.Equal("", self.Config.ACL.ReplicationToken) // Check the Consul config require.Empty(self.Config.Consul.Token) // Assign a Consul token and require it is redacted. s.Config.Consul.Token = "badc0deb-adc0-deba-dc0d-ebadc0debadc" respW = httptest.NewRecorder() obj, err = s.Server.AgentSelfRequest(respW, req) require.NoError(err) self = obj.(agentSelf) require.Equal("", self.Config.Consul.Token) // Check the Circonus config require.Empty(self.Config.Telemetry.CirconusAPIToken) // Assign a Consul token and require it is redacted. s.Config.Telemetry.CirconusAPIToken = "badc0deb-adc0-deba-dc0d-ebadc0debadc" respW = httptest.NewRecorder() obj, err = s.Server.AgentSelfRequest(respW, req) require.NoError(err) self = obj.(agentSelf) require.Equal("", self.Config.Telemetry.CirconusAPIToken) }) } func TestHTTP_AgentSelf_ACL(t *testing.T) { ci.Parallel(t) require := require.New(t) httpACLTest(t, nil, func(s *TestAgent) { state := s.Agent.server.State() // Make the HTTP request req, err := http.NewRequest("GET", "/v1/agent/self", nil) require.Nil(err) // Try request without a token and expect failure { respW := httptest.NewRecorder() _, err := s.Server.AgentSelfRequest(respW, req) require.NotNil(err) require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) } // Try request with an invalid token and expect failure { respW := httptest.NewRecorder() token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyWrite)) setToken(req, token) _, err := s.Server.AgentSelfRequest(respW, req) require.NotNil(err) require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) } // Try request with a valid token { respW := httptest.NewRecorder() token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.AgentPolicy(acl.PolicyWrite)) setToken(req, token) obj, err := s.Server.AgentSelfRequest(respW, req) require.Nil(err) self := obj.(agentSelf) require.NotNil(self.Config) require.NotNil(self.Stats) } // Try request with a root token { respW := httptest.NewRecorder() setToken(req, s.RootToken) obj, err := s.Server.AgentSelfRequest(respW, req) require.Nil(err) self := obj.(agentSelf) require.NotNil(self.Config) require.NotNil(self.Stats) } }) } func TestHTTP_AgentJoin(t *testing.T) { ci.Parallel(t) httpTest(t, nil, func(s *TestAgent) { // Determine the join address member := s.Agent.Server().LocalMember() addr := fmt.Sprintf("%s:%d", member.Addr, member.Port) // Make the HTTP request req, err := http.NewRequest("PUT", fmt.Sprintf("/v1/agent/join?address=%s&address=%s", addr, addr), nil) if err != nil { t.Fatalf("err: %v", err) } respW := httptest.NewRecorder() // Make the request obj, err := s.Server.AgentJoinRequest(respW, req) if err != nil { t.Fatalf("err: %v", err) } // Check the job join := obj.(joinResult) if join.NumJoined != 2 { t.Fatalf("bad: %#v", join) } if join.Error != "" { t.Fatalf("bad: %#v", join) } }) } func TestHTTP_AgentMembers(t *testing.T) { ci.Parallel(t) httpTest(t, nil, func(s *TestAgent) { // Make the HTTP request req, err := http.NewRequest("GET", "/v1/agent/members", nil) if err != nil { t.Fatalf("err: %v", err) } respW := httptest.NewRecorder() // Make the request obj, err := s.Server.AgentMembersRequest(respW, req) if err != nil { t.Fatalf("err: %v", err) } // Check the job members := obj.(structs.ServerMembersResponse) if len(members.Members) != 1 { t.Fatalf("bad: %#v", members.Members) } }) } func TestHTTP_AgentMembers_ACL(t *testing.T) { ci.Parallel(t) require := require.New(t) httpACLTest(t, nil, func(s *TestAgent) { state := s.Agent.server.State() // Make the HTTP request req, err := http.NewRequest("GET", "/v1/agent/members", nil) require.Nil(err) // Try request without a token and expect failure { respW := httptest.NewRecorder() _, err := s.Server.AgentMembersRequest(respW, req) require.NotNil(err) require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) } // Try request with an invalid token and expect failure { respW := httptest.NewRecorder() token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.AgentPolicy(acl.PolicyWrite)) setToken(req, token) _, err := s.Server.AgentMembersRequest(respW, req) require.NotNil(err) require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) } // Try request with a valid token { respW := httptest.NewRecorder() token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.NodePolicy(acl.PolicyRead)) setToken(req, token) obj, err := s.Server.AgentMembersRequest(respW, req) require.Nil(err) members := obj.(structs.ServerMembersResponse) require.Len(members.Members, 1) } // Try request with a root token { respW := httptest.NewRecorder() setToken(req, s.RootToken) obj, err := s.Server.AgentMembersRequest(respW, req) require.Nil(err) members := obj.(structs.ServerMembersResponse) require.Len(members.Members, 1) } }) } func TestHTTP_AgentMonitor(t *testing.T) { ci.Parallel(t) t.Run("invalid log_json parameter", func(t *testing.T) { httpTest(t, nil, func(s *TestAgent) { req, err := http.NewRequest("GET", "/v1/agent/monitor?log_json=no", nil) require.NoError(t, err) resp := newClosableRecorder() // Make the request _, err = s.Server.AgentMonitor(resp, req) httpErr := err.(HTTPCodedError).Code() require.Equal(t, 400, httpErr) }) }) t.Run("unknown log_level", func(t *testing.T) { httpTest(t, nil, func(s *TestAgent) { req, err := http.NewRequest("GET", "/v1/agent/monitor?log_level=unknown", nil) require.NoError(t, err) resp := newClosableRecorder() // Make the request _, err = s.Server.AgentMonitor(resp, req) httpErr := err.(HTTPCodedError).Code() require.Equal(t, 400, httpErr) }) }) t.Run("check for specific log level", func(t *testing.T) { httpTest(t, nil, func(s *TestAgent) { req, err := http.NewRequest("GET", "/v1/agent/monitor?log_level=warn", nil) require.NoError(t, err) resp := newClosableRecorder() defer resp.Close() go func() { _, err = s.Server.AgentMonitor(resp, req) assert.NoError(t, err) }() // send the same log until monitor sink is set up maxLogAttempts := 10 tried := 0 testutil.WaitForResult(func() (bool, error) { if tried < maxLogAttempts { s.Server.logger.Warn("log that should be sent") tried++ } got := resp.Body.String() want := `{"Data":"` if strings.Contains(got, want) { return true, nil } return false, fmt.Errorf("missing expected log, got: %v, want: %v", got, want) }, func(err error) { require.Fail(t, err.Error()) }) }) }) t.Run("plain output", func(t *testing.T) { httpTest(t, nil, func(s *TestAgent) { req, err := http.NewRequest("GET", "/v1/agent/monitor?log_level=debug&plain=true", nil) require.NoError(t, err) resp := newClosableRecorder() defer resp.Close() go func() { _, err = s.Server.AgentMonitor(resp, req) assert.NoError(t, err) }() // send the same log until monitor sink is set up maxLogAttempts := 10 tried := 0 testutil.WaitForResult(func() (bool, error) { if tried < maxLogAttempts { s.Server.logger.Debug("log that should be sent") tried++ } got := resp.Body.String() want := `[DEBUG] http: log that should be sent` if strings.Contains(got, want) { return true, nil } return false, fmt.Errorf("missing expected log, got: %v, want: %v", got, want) }, func(err error) { require.Fail(t, err.Error()) }) }) }) t.Run("logs for a specific node", func(t *testing.T) { httpTest(t, nil, func(s *TestAgent) { req, err := http.NewRequest("GET", "/v1/agent/monitor?log_level=warn&node_id="+s.client.NodeID(), nil) require.NoError(t, err) resp := newClosableRecorder() defer resp.Close() go func() { _, err = s.Server.AgentMonitor(resp, req) assert.NoError(t, err) }() // send the same log until monitor sink is set up maxLogAttempts := 10 tried := 0 out := "" testutil.WaitForResult(func() (bool, error) { if tried < maxLogAttempts { s.Server.logger.Debug("log that should not be sent") s.Server.logger.Warn("log that should be sent") tried++ } output, err := ioutil.ReadAll(resp.Body) if err != nil { return false, err } out += string(output) want := `{"Data":"` if strings.Contains(out, want) { return true, nil } return false, fmt.Errorf("missing expected log, got: %v, want: %v", out, want) }, func(err error) { require.Fail(t, err.Error()) }) }) }) t.Run("logs for a local client with no server running on agent", func(t *testing.T) { httpTest(t, nil, func(s *TestAgent) { req, err := http.NewRequest("GET", "/v1/agent/monitor?log_level=warn", nil) require.NoError(t, err) resp := newClosableRecorder() defer resp.Close() go func() { // set server to nil to monitor as client s.Agent.server = nil _, err = s.Server.AgentMonitor(resp, req) assert.NoError(t, err) }() // send the same log until monitor sink is set up maxLogAttempts := 10 tried := 0 out := "" testutil.WaitForResult(func() (bool, error) { if tried < maxLogAttempts { s.Agent.logger.Warn("log that should be sent") tried++ } output, err := ioutil.ReadAll(resp.Body) if err != nil { return false, err } out += string(output) want := `{"Data":"` if strings.Contains(out, want) { return true, nil } return false, fmt.Errorf("missing expected log, got: %v, want: %v", out, want) }, func(err error) { require.Fail(t, err.Error()) }) }) }) } // Scenarios when Pprof requests should be available // see https://github.com/hashicorp/nomad/issues/6496 // +---------------+------------------+--------+------------------+ // | Endpoint | `enable_debug` | ACLs | **Available?** | // +---------------+------------------+--------+------------------+ // | /debug/pprof | unset | n/a | no | // | /debug/pprof | `true` | n/a | yes | // | /debug/pprof | `false` | n/a | no | // | /agent/pprof | unset | off | no | // | /agent/pprof | unset | on | **yes** | // | /agent/pprof | `true` | off | yes | // | /agent/pprof | `false` | on | **yes** | // +---------------+------------------+--------+------------------+ func TestAgent_PprofRequest_Permissions(t *testing.T) { ci.Parallel(t) trueP, falseP := helper.BoolToPtr(true), helper.BoolToPtr(false) cases := []struct { acl *bool debug *bool ok bool }{ // manually set to false because test helpers // enable to true by default // enableDebug: helper.BoolToPtr(false), {debug: nil, ok: false}, {debug: trueP, ok: true}, {debug: falseP, ok: false}, {debug: falseP, acl: falseP, ok: false}, {acl: trueP, ok: true}, {acl: falseP, debug: trueP, ok: true}, {debug: falseP, acl: trueP, ok: true}, } for _, tc := range cases { ptrToStr := func(val *bool) string { if val == nil { return "unset" } else if *val == true { return "true" } else { return "false" } } t.Run( fmt.Sprintf("debug %s, acl %s", ptrToStr(tc.debug), ptrToStr(tc.acl)), func(t *testing.T) { cb := func(c *Config) { if tc.acl != nil { c.ACL.Enabled = *tc.acl } if tc.debug == nil { var nodebug bool c.EnableDebug = nodebug } else { c.EnableDebug = *tc.debug } } httpTest(t, cb, func(s *TestAgent) { state := s.Agent.server.State() url := "/v1/agent/pprof/cmdline" req, err := http.NewRequest("GET", url, nil) require.NoError(t, err) respW := httptest.NewRecorder() if tc.acl != nil && *tc.acl { token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.AgentPolicy(acl.PolicyWrite)) setToken(req, token) } resp, err := s.Server.AgentPprofRequest(respW, req) if tc.ok { require.NoError(t, err) require.NotNil(t, resp) } else { require.Error(t, err) require.Equal(t, structs.ErrPermissionDenied.Error(), err.Error()) } }) }) } } func TestAgent_PprofRequest(t *testing.T) { ci.Parallel(t) cases := []struct { desc string url string addNodeID bool addServerID bool expectedErr string clientOnly bool }{ { desc: "cmdline local server request", url: "/v1/agent/pprof/cmdline", }, { desc: "cmdline local node request", url: "/v1/agent/pprof/cmdline", clientOnly: true, }, { desc: "cmdline node request", url: "/v1/agent/pprof/cmdline", addNodeID: true, }, { desc: "cmdline server request", url: "/v1/agent/pprof/cmdline", addServerID: true, }, { desc: "invalid server request", url: "/v1/agent/pprof/unknown", addServerID: true, expectedErr: "RPC Error:: 404,Pprof profile not found profile: unknown", }, { desc: "cpu profile request", url: "/v1/agent/pprof/profile", addNodeID: true, }, { desc: "trace request", url: "/v1/agent/pprof/trace", addNodeID: true, }, { desc: "pprof lookup request", url: "/v1/agent/pprof/goroutine", addNodeID: true, }, { desc: "unknown pprof lookup request", url: "/v1/agent/pprof/latency", addNodeID: true, expectedErr: "RPC Error:: 404,Pprof profile not found profile: latency", }, } for _, tc := range cases { t.Run(tc.desc, func(t *testing.T) { httpTest(t, nil, func(s *TestAgent) { // add node or server id query param url := tc.url if tc.addNodeID { url = url + "?node_id=" + s.client.NodeID() } else if tc.addServerID { url = url + "?server_id=" + s.server.LocalMember().Name } if tc.clientOnly { s.Agent.server = nil } req, err := http.NewRequest("GET", url, nil) require.NoError(t, err) respW := httptest.NewRecorder() resp, err := s.Server.AgentPprofRequest(respW, req) if tc.expectedErr != "" { require.Error(t, err) require.EqualError(t, err, tc.expectedErr) } else { require.NoError(t, err) require.NotNil(t, resp) } }) }) } } type closableRecorder struct { *httptest.ResponseRecorder closer chan bool } func newClosableRecorder() *closableRecorder { r := httptest.NewRecorder() closer := make(chan bool) return &closableRecorder{r, closer} } func (r *closableRecorder) Close() { close(r.closer) } func (r *closableRecorder) CloseNotify() <-chan bool { return r.closer } func TestHTTP_AgentForceLeave(t *testing.T) { ci.Parallel(t) httpTest(t, nil, func(s *TestAgent) { // Make the HTTP request req, err := http.NewRequest("PUT", "/v1/agent/force-leave?node=foo", nil) if err != nil { t.Fatalf("err: %v", err) } respW := httptest.NewRecorder() // Make the request _, err = s.Server.AgentForceLeaveRequest(respW, req) if err != nil { t.Fatalf("err: %v", err) } }) } func TestHTTP_AgentForceLeave_ACL(t *testing.T) { ci.Parallel(t) require := require.New(t) httpACLTest(t, nil, func(s *TestAgent) { state := s.Agent.server.State() // Make the HTTP request req, err := http.NewRequest("PUT", "/v1/agent/force-leave?node=foo", nil) require.Nil(err) // Try request without a token and expect failure { respW := httptest.NewRecorder() _, err := s.Server.AgentForceLeaveRequest(respW, req) require.NotNil(err) require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) } // Try request with an invalid token and expect failure { respW := httptest.NewRecorder() token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyRead)) setToken(req, token) _, err := s.Server.AgentForceLeaveRequest(respW, req) require.NotNil(err) require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) } // Try request with a valid token { respW := httptest.NewRecorder() token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.AgentPolicy(acl.PolicyWrite)) setToken(req, token) _, err := s.Server.AgentForceLeaveRequest(respW, req) require.Nil(err) require.Equal(http.StatusOK, respW.Code) } // Try request with a root token { respW := httptest.NewRecorder() setToken(req, s.RootToken) _, err := s.Server.AgentForceLeaveRequest(respW, req) require.Nil(err) require.Equal(http.StatusOK, respW.Code) } }) } func TestHTTP_AgentSetServers(t *testing.T) { ci.Parallel(t) require := require.New(t) httpTest(t, nil, func(s *TestAgent) { addr := s.Config.AdvertiseAddrs.RPC testutil.WaitForResult(func() (bool, error) { conn, err := net.DialTimeout("tcp", addr, 100*time.Millisecond) if err != nil { return false, err } defer conn.Close() // Write the Nomad RPC byte to set the mode if _, err := conn.Write([]byte{byte(pool.RpcNomad)}); err != nil { return false, err } codec := pool.NewClientCodec(conn) args := &structs.GenericRequest{} var leader string err = msgpackrpc.CallWithCodec(codec, "Status.Leader", args, &leader) return leader != "", err }, func(err error) { t.Fatalf("failed to find leader: %v", err) }) // Create the request req, err := http.NewRequest("PUT", "/v1/agent/servers", nil) require.Nil(err) // Send the request respW := httptest.NewRecorder() _, err = s.Server.AgentServersRequest(respW, req) require.NotNil(err) require.Contains(err.Error(), "missing server address") // Create a valid request req, err = http.NewRequest("PUT", "/v1/agent/servers?address=127.0.0.1%3A4647&address=127.0.0.2%3A4647&address=127.0.0.3%3A4647", nil) require.Nil(err) // Send the request which should fail respW = httptest.NewRecorder() _, err = s.Server.AgentServersRequest(respW, req) require.NotNil(err) // Retrieve the servers again req, err = http.NewRequest("GET", "/v1/agent/servers", nil) require.Nil(err) respW = httptest.NewRecorder() // Make the request and check the result expected := []string{ s.GetConfig().AdvertiseAddrs.RPC, } out, err := s.Server.AgentServersRequest(respW, req) require.Nil(err) servers := out.([]string) require.Len(servers, len(expected)) require.Equal(expected, servers) }) } func TestHTTP_AgentSetServers_ACL(t *testing.T) { ci.Parallel(t) require := require.New(t) httpACLTest(t, nil, func(s *TestAgent) { state := s.Agent.server.State() addr := s.Config.AdvertiseAddrs.RPC testutil.WaitForResult(func() (bool, error) { conn, err := net.DialTimeout("tcp", addr, 100*time.Millisecond) if err != nil { return false, err } defer conn.Close() // Write the Consul RPC byte to set the mode if _, err := conn.Write([]byte{byte(pool.RpcNomad)}); err != nil { return false, err } codec := pool.NewClientCodec(conn) args := &structs.GenericRequest{} var leader string err = msgpackrpc.CallWithCodec(codec, "Status.Leader", args, &leader) return leader != "", err }, func(err error) { t.Fatalf("failed to find leader: %v", err) }) // Make the HTTP request path := fmt.Sprintf("/v1/agent/servers?address=%s", url.QueryEscape(s.GetConfig().AdvertiseAddrs.RPC)) req, err := http.NewRequest("PUT", path, nil) require.Nil(err) // Try request without a token and expect failure { respW := httptest.NewRecorder() _, err := s.Server.AgentServersRequest(respW, req) require.NotNil(err) require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) } // Try request with an invalid token and expect failure { respW := httptest.NewRecorder() token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyRead)) setToken(req, token) _, err := s.Server.AgentServersRequest(respW, req) require.NotNil(err) require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) } // Try request with a valid token { respW := httptest.NewRecorder() token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.AgentPolicy(acl.PolicyWrite)) setToken(req, token) _, err := s.Server.AgentServersRequest(respW, req) require.Nil(err) require.Equal(http.StatusOK, respW.Code) } // Try request with a root token { respW := httptest.NewRecorder() setToken(req, s.RootToken) _, err := s.Server.AgentServersRequest(respW, req) require.Nil(err) require.Equal(http.StatusOK, respW.Code) } }) } func TestHTTP_AgentListServers_ACL(t *testing.T) { ci.Parallel(t) require := require.New(t) httpACLTest(t, nil, func(s *TestAgent) { state := s.Agent.server.State() // Create list request req, err := http.NewRequest("GET", "/v1/agent/servers", nil) require.Nil(err) expected := []string{ s.GetConfig().AdvertiseAddrs.RPC, } // Try request without a token and expect failure { respW := httptest.NewRecorder() _, err := s.Server.AgentServersRequest(respW, req) require.NotNil(err) require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) } // Try request with an invalid token and expect failure { respW := httptest.NewRecorder() token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyRead)) setToken(req, token) _, err := s.Server.AgentServersRequest(respW, req) require.NotNil(err) require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) } // Wait for client to have a server testutil.WaitForResult(func() (bool, error) { return len(s.client.GetServers()) != 0, fmt.Errorf("no servers") }, func(err error) { t.Fatal(err) }) // Try request with a valid token { respW := httptest.NewRecorder() token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.AgentPolicy(acl.PolicyRead)) setToken(req, token) out, err := s.Server.AgentServersRequest(respW, req) require.Nil(err) servers := out.([]string) require.Len(servers, len(expected)) require.Equal(expected, servers) } // Try request with a root token { respW := httptest.NewRecorder() setToken(req, s.RootToken) out, err := s.Server.AgentServersRequest(respW, req) require.Nil(err) servers := out.([]string) require.Len(servers, len(expected)) require.Equal(expected, servers) } }) } func TestHTTP_AgentListKeys(t *testing.T) { ci.Parallel(t) key1 := "HS5lJ+XuTlYKWaeGYyG+/A==" httpTest(t, func(c *Config) { c.Server.EncryptKey = key1 }, func(s *TestAgent) { req, err := http.NewRequest("GET", "/v1/agent/keyring/list", nil) if err != nil { t.Fatalf("err: %s", err) } respW := httptest.NewRecorder() out, err := s.Server.KeyringOperationRequest(respW, req) require.NoError(t, err) kresp := out.(structs.KeyringResponse) require.Len(t, kresp.Keys, 1) }) } func TestHTTP_AgentListKeys_ACL(t *testing.T) { ci.Parallel(t) require := require.New(t) key1 := "HS5lJ+XuTlYKWaeGYyG+/A==" cb := func(c *Config) { c.Server.EncryptKey = key1 } httpACLTest(t, cb, func(s *TestAgent) { state := s.Agent.server.State() // Make the HTTP request req, err := http.NewRequest("GET", "/v1/agent/keyring/list", nil) require.Nil(err) // Try request without a token and expect failure { respW := httptest.NewRecorder() _, err := s.Server.KeyringOperationRequest(respW, req) require.NotNil(err) require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) } // Try request with an invalid token and expect failure { respW := httptest.NewRecorder() token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.AgentPolicy(acl.PolicyRead)) setToken(req, token) _, err := s.Server.KeyringOperationRequest(respW, req) require.NotNil(err) require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) } // Try request with a valid token { respW := httptest.NewRecorder() token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.AgentPolicy(acl.PolicyWrite)) setToken(req, token) out, err := s.Server.KeyringOperationRequest(respW, req) require.Nil(err) kresp := out.(structs.KeyringResponse) require.Len(kresp.Keys, 1) require.Contains(kresp.Keys, key1) } // Try request with a root token { respW := httptest.NewRecorder() setToken(req, s.RootToken) out, err := s.Server.KeyringOperationRequest(respW, req) require.Nil(err) kresp := out.(structs.KeyringResponse) require.Len(kresp.Keys, 1) require.Contains(kresp.Keys, key1) } }) } func TestHTTP_AgentInstallKey(t *testing.T) { ci.Parallel(t) key1 := "HS5lJ+XuTlYKWaeGYyG+/A==" key2 := "wH1Bn9hlJ0emgWB1JttVRA==" httpTest(t, func(c *Config) { c.Server.EncryptKey = key1 }, func(s *TestAgent) { b, err := json.Marshal(&structs.KeyringRequest{Key: key2}) if err != nil { t.Fatalf("err: %v", err) } req, err := http.NewRequest("GET", "/v1/agent/keyring/install", bytes.NewReader(b)) if err != nil { t.Fatalf("err: %s", err) } respW := httptest.NewRecorder() _, err = s.Server.KeyringOperationRequest(respW, req) if err != nil { t.Fatalf("err: %s", err) } req, err = http.NewRequest("GET", "/v1/agent/keyring/list", bytes.NewReader(b)) if err != nil { t.Fatalf("err: %s", err) } respW = httptest.NewRecorder() out, err := s.Server.KeyringOperationRequest(respW, req) if err != nil { t.Fatalf("err: %s", err) } kresp := out.(structs.KeyringResponse) if len(kresp.Keys) != 2 { t.Fatalf("bad: %v", kresp) } }) } func TestHTTP_AgentRemoveKey(t *testing.T) { ci.Parallel(t) key1 := "HS5lJ+XuTlYKWaeGYyG+/A==" key2 := "wH1Bn9hlJ0emgWB1JttVRA==" httpTest(t, func(c *Config) { c.Server.EncryptKey = key1 }, func(s *TestAgent) { b, err := json.Marshal(&structs.KeyringRequest{Key: key2}) if err != nil { t.Fatalf("err: %v", err) } req, err := http.NewRequest("GET", "/v1/agent/keyring/install", bytes.NewReader(b)) if err != nil { t.Fatalf("err: %s", err) } respW := httptest.NewRecorder() _, err = s.Server.KeyringOperationRequest(respW, req) if err != nil { t.Fatalf("err: %s", err) } req, err = http.NewRequest("GET", "/v1/agent/keyring/remove", bytes.NewReader(b)) if err != nil { t.Fatalf("err: %s", err) } respW = httptest.NewRecorder() if _, err = s.Server.KeyringOperationRequest(respW, req); err != nil { t.Fatalf("err: %s", err) } req, err = http.NewRequest("GET", "/v1/agent/keyring/list", nil) if err != nil { t.Fatalf("err: %s", err) } respW = httptest.NewRecorder() out, err := s.Server.KeyringOperationRequest(respW, req) if err != nil { t.Fatalf("err: %s", err) } kresp := out.(structs.KeyringResponse) if len(kresp.Keys) != 1 { t.Fatalf("bad: %v", kresp) } }) } func TestHTTP_AgentHealth_Ok(t *testing.T) { ci.Parallel(t) require := require.New(t) // Enable ACLs to ensure they're not enforced httpACLTest(t, nil, func(s *TestAgent) { // No ?type= { req, err := http.NewRequest("GET", "/v1/agent/health", nil) require.Nil(err) respW := httptest.NewRecorder() healthI, err := s.Server.HealthRequest(respW, req) require.Nil(err) require.Equal(http.StatusOK, respW.Code) require.NotNil(healthI) health := healthI.(*healthResponse) require.NotNil(health.Client) require.True(health.Client.Ok) require.Equal("ok", health.Client.Message) require.NotNil(health.Server) require.True(health.Server.Ok) require.Equal("ok", health.Server.Message) } // type=client { req, err := http.NewRequest("GET", "/v1/agent/health?type=client", nil) require.Nil(err) respW := httptest.NewRecorder() healthI, err := s.Server.HealthRequest(respW, req) require.Nil(err) require.Equal(http.StatusOK, respW.Code) require.NotNil(healthI) health := healthI.(*healthResponse) require.NotNil(health.Client) require.True(health.Client.Ok) require.Equal("ok", health.Client.Message) require.Nil(health.Server) } // type=server { req, err := http.NewRequest("GET", "/v1/agent/health?type=server", nil) require.Nil(err) respW := httptest.NewRecorder() healthI, err := s.Server.HealthRequest(respW, req) require.Nil(err) require.Equal(http.StatusOK, respW.Code) require.NotNil(healthI) health := healthI.(*healthResponse) require.NotNil(health.Server) require.True(health.Server.Ok) require.Equal("ok", health.Server.Message) require.Nil(health.Client) } // type=client&type=server { req, err := http.NewRequest("GET", "/v1/agent/health?type=client&type=server", nil) require.Nil(err) respW := httptest.NewRecorder() healthI, err := s.Server.HealthRequest(respW, req) require.Nil(err) require.Equal(http.StatusOK, respW.Code) require.NotNil(healthI) health := healthI.(*healthResponse) require.NotNil(health.Client) require.True(health.Client.Ok) require.Equal("ok", health.Client.Message) require.NotNil(health.Server) require.True(health.Server.Ok) require.Equal("ok", health.Server.Message) } }) } func TestHTTP_AgentHealth_BadServer(t *testing.T) { ci.Parallel(t) require := require.New(t) serverAgent := NewTestAgent(t, "server", nil) defer serverAgent.Shutdown() s := makeHTTPServer(t, func(c *Config) { // Disable server to make server health unhealthy if requested c.Server.Enabled = false c.Client.Servers = []string{fmt.Sprintf("localhost:%d", serverAgent.Config.Ports.RPC)} }) defer s.Shutdown() // No ?type= means server is just skipped { req, err := http.NewRequest("GET", "/v1/agent/health", nil) require.Nil(err) respW := httptest.NewRecorder() healthI, err := s.Server.HealthRequest(respW, req) require.Nil(err) require.Equal(http.StatusOK, respW.Code) require.NotNil(healthI) health := healthI.(*healthResponse) require.NotNil(health.Client) require.True(health.Client.Ok) require.Equal("ok", health.Client.Message) require.Nil(health.Server) } // type=server means server is considered unhealthy { req, err := http.NewRequest("GET", "/v1/agent/health?type=server", nil) require.Nil(err) respW := httptest.NewRecorder() _, err = s.Server.HealthRequest(respW, req) require.NotNil(err) httpErr, ok := err.(HTTPCodedError) require.True(ok) require.Equal(500, httpErr.Code()) require.Equal(`{"server":{"ok":false,"message":"server not enabled"}}`, err.Error()) } } func TestHTTP_AgentHealth_BadClient(t *testing.T) { ci.Parallel(t) require := require.New(t) // Disable client to make server unhealthy if requested cb := func(c *Config) { c.Client.Enabled = false } // Enable ACLs to ensure they're not enforced httpACLTest(t, cb, func(s *TestAgent) { // No ?type= means client is just skipped { req, err := http.NewRequest("GET", "/v1/agent/health", nil) require.Nil(err) respW := httptest.NewRecorder() healthI, err := s.Server.HealthRequest(respW, req) require.Nil(err) require.Equal(http.StatusOK, respW.Code) require.NotNil(healthI) health := healthI.(*healthResponse) require.NotNil(health.Server) require.True(health.Server.Ok) require.Equal("ok", health.Server.Message) require.Nil(health.Client) } // type=client means client is considered unhealthy { req, err := http.NewRequest("GET", "/v1/agent/health?type=client", nil) require.Nil(err) respW := httptest.NewRecorder() _, err = s.Server.HealthRequest(respW, req) require.NotNil(err) httpErr, ok := err.(HTTPCodedError) require.True(ok) require.Equal(500, httpErr.Code()) require.Equal(`{"client":{"ok":false,"message":"client not enabled"}}`, err.Error()) } }) } var ( errorPipe = &net.OpError{ Op: "write", Net: "tcp", Source: &net.TCPAddr{}, Addr: &net.TCPAddr{}, Err: &os.SyscallError{ Syscall: "write", Err: syscall.EPIPE, }, } ) // fakeRW is a fake response writer to ease polling streaming responses in a // data-race-free way. type fakeRW struct { Code int HeaderMap http.Header buf *bytes.Buffer closed bool mu sync.Mutex // Written is ticked whenever a Write occurs and on WriteHeaders if it // is explicitly called Written chan int // ClosedErr is the error Write will return once the writer is closed. // Defaults to EPIPE. Must not be mutated concurrently with writes. ClosedErr error } // Header is for setting headers before writing a response. Tests should check // the HeaderMap field directly. func (f *fakeRW) Header() http.Header { f.mu.Lock() defer f.mu.Unlock() if f.Code != 0 { panic("cannot set headers after WriteHeader has been called") } return f.HeaderMap } func (f *fakeRW) Write(p []byte) (int, error) { f.mu.Lock() defer f.mu.Unlock() if f.closed { // Mimic an EPIPE error return 0, f.ClosedErr } if f.Code == 0 { f.Code = 200 } n, err := f.buf.Write(p) select { case f.Written <- 1: default: } return n, err } // WriteHeader sets Code and FinalHeaders func (f *fakeRW) WriteHeader(statusCode int) { f.mu.Lock() defer f.mu.Unlock() if f.Code != 0 { panic("cannot call WriteHeader more than once") } f.Code = statusCode select { case f.Written <- 1: default: } } // Bytes returns the body bytes written to the buffer. Safe for calling // concurrent with writes. func (f *fakeRW) Bytes() []byte { f.mu.Lock() defer f.mu.Unlock() return f.buf.Bytes() } // Close the writer causing an EPIPE error on future writes. Safe to call // concurrently with other methods. Safe to call more than once. func (f *fakeRW) Close() { f.mu.Lock() defer f.mu.Unlock() f.closed = true } func NewFakeRW() *fakeRW { return &fakeRW{ HeaderMap: make(map[string][]string), buf: &bytes.Buffer{}, Written: make(chan int, 1), ClosedErr: errorPipe, } } // TestHTTP_XSS_Monitor asserts /v1/agent/monitor is safe against XSS attacks // even when log output contains HTML+Javascript. func TestHTTP_XSS_Monitor(t *testing.T) { ci.Parallel(t) cases := []struct { Name string Logline string JSON bool }{ { Name: "Plain", Logline: "--TEST 123--", JSON: false, }, { Name: "JSON", Logline: "--TEST 123--", JSON: true, }, { Name: "XSSPlain", Logline: "", JSON: false, }, { Name: "XSSJson", Logline: "", JSON: true, }, } for i := range cases { tc := cases[i] t.Run(tc.Name, func(t *testing.T) { ci.Parallel(t) s := makeHTTPServer(t, nil) defer s.Shutdown() path := fmt.Sprintf("%s/v1/agent/monitor?error_level=error&plain=%t", s.HTTPAddr(), !tc.JSON) req, err := http.NewRequest("GET", path, nil) require.NoError(t, err) resp := NewFakeRW() closedErr := errors.New("sentinel error") resp.ClosedErr = closedErr defer resp.Close() errCh := make(chan error, 1) go func() { _, err := s.Server.AgentMonitor(resp, req) errCh <- err }() deadline := time.After(3 * time.Second) OUTER: for { // Log a needle and look for it in the response haystack s.Server.logger.Error(tc.Logline) select { case <-time.After(30 * time.Millisecond): // Give AgentMonitor handler goroutine time to start case <-resp.Written: // Something was written, check it case <-deadline: t.Fatalf("timed out waiting for expected log line; body:\n%s", string(resp.Bytes())) case err := <-errCh: t.Fatalf("AgentMonitor exited unexpectedly: err=%v", err) } if !tc.JSON { if bytes.Contains(resp.Bytes(), []byte(tc.Logline)) { // Found needle! break } else { // Try again continue } } // Decode JSON r := bytes.NewReader(resp.Bytes()) dec := json.NewDecoder(r) for { data := struct{ Data []byte }{} if err := dec.Decode(&data); err != nil { // Probably a partial write, continue continue OUTER } if bytes.Contains(data.Data, []byte(tc.Logline)) { // Found needle! break OUTER } } } // Assert default logs are application/json ct := "text/plain" if tc.JSON { ct = "application/json" } require.Equal(t, []string{ct}, resp.HeaderMap.Values("Content-Type")) // Close response writer and log to make AgentMonitor exit resp.Close() s.Server.logger.Error("log again to force a write that detects the closed connection") select { case err := <-errCh: require.EqualError(t, closedErr, err.Error()) case <-deadline: t.Fatalf("timed out waiting for closing error from handler") } }) } } // ---------------------------- // SchedulerWorkerInfoAPI tests // ---------------------------- type schedulerWorkerAPITest_testCase struct { name string // test case name request schedulerWorkerAPITest_testRequest whenACLNotEnabled schedulerWorkerAPITest_testExpect whenACLEnabled schedulerWorkerAPITest_testExpect } type schedulerWorkerAPITest_testRequest struct { verb string aclToken string requestBody string } type schedulerWorkerAPITest_testExpect struct { statusCode int response interface{} err error isError bool } func (te schedulerWorkerAPITest_testExpect) Code() int { return te.statusCode } func schedulerWorkerInfoTest_testCases() []schedulerWorkerAPITest_testCase { forbidden := schedulerWorkerAPITest_testExpect{ statusCode: http.StatusForbidden, response: structs.ErrPermissionDenied.Error(), isError: true, } invalidMethod := schedulerWorkerAPITest_testExpect{ statusCode: http.StatusMethodNotAllowed, response: ErrInvalidMethod, isError: true, } success := schedulerWorkerAPITest_testExpect{ statusCode: http.StatusOK, response: &api.AgentSchedulerWorkersInfo{ Schedulers: []api.AgentSchedulerWorkerInfo{ { ID: "9b3713e0-6f74-0e1b-3b3e-d94f0c22dbf9", EnabledSchedulers: []string{"_core", "batch"}, Started: "2021-12-10 22:13:12.595366 -0500 EST m=+0.039016232", Status: "Pausing", WorkloadStatus: "WaitingToDequeue", }, { ID: "ebda23e2-7f68-0c82-f0b2-f91d4581094d", EnabledSchedulers: []string{"_core", "batch"}, Started: "2021-12-10 22:13:12.595478 -0500 EST m=+0.039127886", Status: "Pausing", WorkloadStatus: "WaitingToDequeue", }, { ID: "b3869c9b-64ff-686c-a003-e7d059d3a573", EnabledSchedulers: []string{"_core", "batch"}, Started: "2021-12-10 22:13:12.595501 -0500 EST m=+0.039151276", Status: "Pausing", WorkloadStatus: "WaitingToDequeue", }, { ID: "cc5907c0-552e-bf36-0ca1-f150af7273c2", EnabledSchedulers: []string{"_core", "batch"}, Started: "2021-12-10 22:13:12.595691 -0500 EST m=+0.039341541", Status: "Starting", WorkloadStatus: "WaitingToDequeue", }, }, }, } return []schedulerWorkerAPITest_testCase{ { name: "bad verb", request: schedulerWorkerAPITest_testRequest{ verb: "FOO", aclToken: "", requestBody: "", }, whenACLNotEnabled: invalidMethod, whenACLEnabled: invalidMethod, }, { name: "get without token", request: schedulerWorkerAPITest_testRequest{ verb: "GET", aclToken: "", requestBody: "", }, whenACLNotEnabled: success, whenACLEnabled: forbidden, }, { name: "get with management token", request: schedulerWorkerAPITest_testRequest{ verb: "GET", aclToken: "management", requestBody: "", }, whenACLNotEnabled: success, whenACLEnabled: success, }, { name: "get with read token", request: schedulerWorkerAPITest_testRequest{ verb: "GET", aclToken: "agent_read", requestBody: "", }, whenACLNotEnabled: success, whenACLEnabled: success, }, { name: "get with invalid token", request: schedulerWorkerAPITest_testRequest{ verb: "GET", aclToken: "node_write", requestBody: "", }, whenACLNotEnabled: success, whenACLEnabled: forbidden, }, } } func TestHTTP_AgentSchedulerWorkerInfoRequest(t *testing.T) { ci.Parallel(t) configFn := func(c *Config) { var numSchedulers = 4 c.Server.NumSchedulers = &numSchedulers c.Server.EnabledSchedulers = []string{"_core", "batch"} c.Client.Enabled = false } for _, runACL := range []string{"no_acl", "acl"} { t.Run(runACL, func(t *testing.T) { tests := func(s *TestAgent) { testingACLS := s.Config.ACL.Enabled var tokens map[string]*structs.ACLToken if s.Config.ACL.Enabled { state := s.Agent.server.State() tokens = make(map[string]*structs.ACLToken) tokens["management"] = s.RootToken tokens["agent_read"] = mock.CreatePolicyAndToken(t, state, 1005, "agent_read", mock.AgentPolicy(acl.PolicyRead)) tokens["agent_write"] = mock.CreatePolicyAndToken(t, state, 1007, "agent_write", mock.AgentPolicy(acl.PolicyWrite)) tokens["node_write"] = mock.CreatePolicyAndToken(t, state, 1009, "node_write", mock.NodePolicy(acl.PolicyWrite)) } for _, tc := range schedulerWorkerInfoTest_testCases() { t.Run(tc.name, func(t *testing.T) { req, err := http.NewRequest(tc.request.verb, "/v1/agent/schedulers", bytes.NewReader([]byte(tc.request.requestBody))) if testingACLS && tc.request.aclToken != "" { setToken(req, tokens[tc.request.aclToken]) } require.NoError(t, err) respW := httptest.NewRecorder() workerInfoResp, err := s.Server.AgentSchedulerWorkerInfoRequest(respW, req) expected := tc.whenACLNotEnabled if testingACLS { expected = tc.whenACLEnabled } if expected.isError { require.Error(t, err) codedErr, ok := err.(HTTPCodedError) require.True(t, ok, "expected a HTTPCodedError") require.Equal(t, expected.Code(), codedErr.Code()) require.Equal(t, expected.response, codedErr.Error()) return } require.NoError(t, err) workerInfo, ok := workerInfoResp.(*api.AgentSchedulerWorkersInfo) require.True(t, ok, "expected an *AgentSchedulersWorkersInfo. received:%s", reflect.TypeOf(workerInfoResp)) expectWorkerInfo, ok := expected.response.(*api.AgentSchedulerWorkersInfo) require.True(t, ok, "error casting test case to *AgentSchedulersWorkersInfo. received:%s", reflect.TypeOf(workerInfoResp)) var schedCount int = *s.Config.Server.NumSchedulers require.Equal(t, schedCount, len(workerInfo.Schedulers), "must match num_schedulers") require.Equal(t, len(expectWorkerInfo.Schedulers), len(workerInfo.Schedulers), "lengths must match") for i, info := range expectWorkerInfo.Schedulers { require.ElementsMatch(t, info.EnabledSchedulers, workerInfo.Schedulers[i].EnabledSchedulers) } }) } } if runACL == "acl" { httpACLTest(t, configFn, tests) } else { httpTest(t, configFn, tests) } }) } } // ---------------------------- // SchedulerWorkerConfigAPI tests // ---------------------------- type scheduleWorkerConfigTest_workerRequestTest struct { name string // test case name request schedulerWorkerConfigTest_testRequest whenACLNotEnabled schedulerWorkerConfigTest_testExpect whenACLEnabled schedulerWorkerConfigTest_testExpect } type schedulerWorkerConfigTest_testRequest struct { verb string aclToken string requestBody string } type schedulerWorkerConfigTest_testExpect struct { expectedResponseCode int expectedResponse interface{} } // These test cases are run for both the ACL and Non-ACL enabled servers. When // ACLS are not enabled, the request.aclTokens are ignored. func schedulerWorkerConfigTest_testCases() []scheduleWorkerConfigTest_workerRequestTest { forbidden := schedulerWorkerConfigTest_testExpect{ expectedResponseCode: http.StatusForbidden, expectedResponse: structs.ErrPermissionDenied.Error(), } invalidMethod := schedulerWorkerConfigTest_testExpect{ expectedResponseCode: http.StatusMethodNotAllowed, expectedResponse: ErrInvalidMethod, } invalidRequest := schedulerWorkerConfigTest_testExpect{ expectedResponseCode: http.StatusBadRequest, expectedResponse: "Invalid request", } success1 := schedulerWorkerConfigTest_testExpect{ expectedResponseCode: http.StatusOK, expectedResponse: &api.AgentSchedulerWorkerConfigResponse{EnabledSchedulers: []string{"_core", "batch"}, NumSchedulers: 8}, } success2 := schedulerWorkerConfigTest_testExpect{ expectedResponseCode: http.StatusOK, expectedResponse: &api.AgentSchedulerWorkerConfigResponse{EnabledSchedulers: []string{"_core", "batch"}, NumSchedulers: 9}, } return []scheduleWorkerConfigTest_workerRequestTest{ { name: "bad verb", request: schedulerWorkerConfigTest_testRequest{ verb: "FOO", aclToken: "", requestBody: "", }, whenACLNotEnabled: invalidMethod, whenACLEnabled: invalidMethod, }, { name: "get without token", request: schedulerWorkerConfigTest_testRequest{ verb: "GET", aclToken: "", requestBody: "", }, whenACLNotEnabled: success1, whenACLEnabled: forbidden, }, { name: "get with management token", request: schedulerWorkerConfigTest_testRequest{ verb: "GET", aclToken: "management", requestBody: "", }, whenACLNotEnabled: success1, whenACLEnabled: success1, }, { name: "get with read token", request: schedulerWorkerConfigTest_testRequest{ verb: "GET", aclToken: "agent_read", requestBody: "", }, whenACLNotEnabled: success1, whenACLEnabled: success1, }, { name: "get with write token", request: schedulerWorkerConfigTest_testRequest{ verb: "GET", aclToken: "agent_write", requestBody: "", }, whenACLNotEnabled: success1, whenACLEnabled: success1, }, { name: "post with no token", request: schedulerWorkerConfigTest_testRequest{ verb: "POST", aclToken: "", requestBody: `{"num_schedulers":9,"enabled_schedulers":["_core", "batch"]}`, }, whenACLNotEnabled: success2, whenACLEnabled: forbidden, }, { name: "put with no token", request: schedulerWorkerConfigTest_testRequest{ verb: "PUT", aclToken: "", requestBody: `{"num_schedulers":8,"enabled_schedulers":["_core", "batch"]}`, }, whenACLNotEnabled: success1, whenACLEnabled: forbidden, }, { name: "post with invalid token", request: schedulerWorkerConfigTest_testRequest{ verb: "POST", aclToken: "node_write", requestBody: `{"num_schedulers":9,"enabled_schedulers":["_core", "batch"]}`, }, whenACLNotEnabled: success2, whenACLEnabled: forbidden, }, { name: "put with invalid token", request: schedulerWorkerConfigTest_testRequest{ verb: "PUT", aclToken: "node_write", requestBody: `{"num_schedulers":8,"enabled_schedulers":["_core", "batch"]}`, }, whenACLNotEnabled: success1, whenACLEnabled: forbidden, }, { name: "post with valid token", request: schedulerWorkerConfigTest_testRequest{ verb: "POST", aclToken: "agent_write", requestBody: `{"num_schedulers":9,"enabled_schedulers":["_core", "batch"]}`, }, whenACLNotEnabled: success2, whenACLEnabled: success2, }, { name: "put with valid token", request: schedulerWorkerConfigTest_testRequest{ verb: "PUT", aclToken: "agent_write", requestBody: `{"num_schedulers":8,"enabled_schedulers":["_core", "batch"]}`, }, whenACLNotEnabled: success1, whenACLEnabled: success1, }, { name: "post with good token and bad value", request: schedulerWorkerConfigTest_testRequest{ verb: "POST", aclToken: "agent_write", requestBody: `{"num_schedulers":-1,"enabled_schedulers":["_core", "batch"]}`, }, whenACLNotEnabled: invalidRequest, whenACLEnabled: invalidRequest, }, { name: "post with bad token and bad value", request: schedulerWorkerConfigTest_testRequest{ verb: "POST", aclToken: "node_write", requestBody: `{"num_schedulers":-1,"enabled_schedulers":["_core", "batch"]}`, }, whenACLNotEnabled: invalidRequest, whenACLEnabled: forbidden, }, { name: "put with good token and bad value", request: schedulerWorkerConfigTest_testRequest{ verb: "PUT", aclToken: "agent_write", requestBody: `{"num_schedulers":-1,"enabled_schedulers":["_core", "batch"]}`, }, whenACLNotEnabled: invalidRequest, whenACLEnabled: invalidRequest, }, { name: "put with bad token and bad value", request: schedulerWorkerConfigTest_testRequest{ verb: "PUT", aclToken: "node_write", requestBody: `{"num_schedulers":-1,"enabled_schedulers":["_core", "batch"]}`, }, whenACLNotEnabled: invalidRequest, whenACLEnabled: forbidden, }, { name: "post with bad json", request: schedulerWorkerConfigTest_testRequest{ verb: "POST", aclToken: "agent_write", requestBody: `{num_schedulers:-1,"enabled_schedulers":["_core", "batch"]}`, }, whenACLNotEnabled: invalidRequest, whenACLEnabled: invalidRequest, }, { name: "put with bad json", request: schedulerWorkerConfigTest_testRequest{ verb: "PUT", aclToken: "agent_write", requestBody: `{num_schedulers:-1,"enabled_schedulers":["_core", "batch"]}`, }, whenACLNotEnabled: invalidRequest, whenACLEnabled: invalidRequest, }, } } func TestHTTP_AgentSchedulerWorkerConfigRequest_NoACL(t *testing.T) { ci.Parallel(t) configFn := func(c *Config) { var numSchedulers = 8 c.Server.NumSchedulers = &numSchedulers c.Server.EnabledSchedulers = []string{"_core", "batch"} c.Client.Enabled = false } testFn := func(s *TestAgent) { for _, tc := range schedulerWorkerConfigTest_testCases() { t.Run(tc.name, func(t *testing.T) { req, err := http.NewRequest(tc.request.verb, "/v1/agent/schedulers/config", bytes.NewReader([]byte(tc.request.requestBody))) require.NoError(t, err) respW := httptest.NewRecorder() workersI, err := s.Server.AgentSchedulerWorkerConfigRequest(respW, req) switch tc.whenACLNotEnabled.expectedResponseCode { case http.StatusBadRequest, http.StatusForbidden, http.StatusMethodNotAllowed: schedulerWorkerTest_parseError(t, false, tc, workersI, err) case http.StatusOK: schedulerWorkerTest_parseSuccess(t, false, tc, workersI, err) default: require.Failf(t, "unexpected status code", "code: %v", tc.whenACLNotEnabled.expectedResponseCode) } }) } } httpTest(t, configFn, testFn) } func TestHTTP_AgentSchedulerWorkerConfigRequest_ACL(t *testing.T) { ci.Parallel(t) configFn := func(c *Config) { var numSchedulers = 8 c.Server.NumSchedulers = &numSchedulers c.Server.EnabledSchedulers = []string{"_core", "batch"} c.Client.Enabled = false } tests := func(s *TestAgent) { state := s.Agent.server.State() var tokens map[string]*structs.ACLToken = make(map[string]*structs.ACLToken) tokens["management"] = s.RootToken tokens["agent_read"] = mock.CreatePolicyAndToken(t, state, 1005, "agent_read", mock.AgentPolicy(acl.PolicyRead)) tokens["agent_write"] = mock.CreatePolicyAndToken(t, state, 1007, "agent_write", mock.AgentPolicy(acl.PolicyWrite)) tokens["node_write"] = mock.CreatePolicyAndToken(t, state, 1009, "node_write", mock.NodePolicy(acl.PolicyWrite)) for _, tc := range schedulerWorkerConfigTest_testCases() { t.Run(tc.name, func(t *testing.T) { req, err := http.NewRequest(tc.request.verb, "/v1/agent/schedulers", bytes.NewReader([]byte(tc.request.requestBody))) if tc.request.aclToken != "" { setToken(req, tokens[tc.request.aclToken]) } require.NoError(t, err) respW := httptest.NewRecorder() workersI, err := s.Server.AgentSchedulerWorkerConfigRequest(respW, req) switch tc.whenACLEnabled.expectedResponseCode { case http.StatusOK: schedulerWorkerTest_parseSuccess(t, true, tc, workersI, err) case http.StatusBadRequest, http.StatusForbidden, http.StatusMethodNotAllowed: schedulerWorkerTest_parseError(t, true, tc, workersI, err) default: require.Failf(t, "unexpected status code", "code: %v", tc.whenACLEnabled.expectedResponseCode) } }) } } httpACLTest(t, configFn, tests) } func schedulerWorkerTest_parseSuccess(t *testing.T, isACLEnabled bool, tc scheduleWorkerConfigTest_workerRequestTest, workersI interface{}, err error) { require.NoError(t, err) require.NotNil(t, workersI) testExpect := tc.whenACLNotEnabled if isACLEnabled { testExpect = tc.whenACLNotEnabled } // test into the response when we expect an okay tcConfig, ok := testExpect.expectedResponse.(*api.AgentSchedulerWorkerConfigResponse) require.True(t, ok, "expected response malformed - this is an issue with a test case.") workersConfig, ok := workersI.(*api.AgentSchedulerWorkerConfigResponse) require.True(t, ok, "response can not cast to an agentSchedulerWorkerConfig") require.NotNil(t, workersConfig) require.Equal(t, tcConfig.NumSchedulers, workersConfig.NumSchedulers) require.ElementsMatch(t, tcConfig.EnabledSchedulers, workersConfig.EnabledSchedulers) } // schedulerWorkerTest_parseError parses the error response given // from the API call to make sure that it's a coded error and is the // expected value from the test case func schedulerWorkerTest_parseError(t *testing.T, isACLEnabled bool, tc scheduleWorkerConfigTest_workerRequestTest, workersI interface{}, err error) { require.Error(t, err) require.Nil(t, workersI) codedError, ok := err.(HTTPCodedError) require.True(t, ok, "expected an HTTPCodedError") testExpect := tc.whenACLNotEnabled if isACLEnabled { testExpect = tc.whenACLEnabled } require.Equal(t, testExpect.expectedResponseCode, codedError.Code()) // this is a relaxed test to allow us to not have to create a case // for concatenated error strings. require.Contains(t, codedError.Error(), testExpect.expectedResponse) } func TestHTTP_AgentSchedulerWorkerInfoRequest_Client(t *testing.T) { ci.Parallel(t) verbs := []string{"GET", "POST", "PUT"} path := "schedulers" for _, verb := range verbs { t.Run(verb, func(t *testing.T) { httpTest(t, nil, func(s *TestAgent) { s.Agent.server = nil req, err := http.NewRequest(verb, fmt.Sprintf("/v1/agent/%v", path), nil) require.NoError(t, err) respW := httptest.NewRecorder() _, err = s.Server.AgentSchedulerWorkerInfoRequest(respW, req) require.Error(t, err) codedErr, ok := err.(HTTPCodedError) require.True(t, ok, "expected a HTTPCodedError") require.Equal(t, http.StatusBadRequest, codedErr.Code()) require.Equal(t, ErrServerOnly, codedErr.Error()) }) }) } } func TestHTTP_AgentSchedulerWorkerConfigRequest_Client(t *testing.T) { ci.Parallel(t) verbs := []string{"GET", "POST", "PUT"} path := "schedulers/config" for _, verb := range verbs { t.Run(verb, func(t *testing.T) { httpTest(t, nil, func(s *TestAgent) { s.Agent.server = nil req, err := http.NewRequest(verb, fmt.Sprintf("/v1/agent/%v", path), nil) require.NoError(t, err) respW := httptest.NewRecorder() _, err = s.Server.AgentSchedulerWorkerInfoRequest(respW, req) require.Error(t, err) codedErr, ok := err.(HTTPCodedError) require.True(t, ok, "expected a HTTPCodedError") require.Equal(t, http.StatusBadRequest, codedErr.Code()) require.Equal(t, ErrServerOnly, codedErr.Error()) }) }) } }