4b2ba62e35
Fix a bug where a millicious user can access or manipulate an alloc in a namespace they don't have access to. The allocation endpoints perform ACL checks against the request namespace, not the allocation namespace, and performs the allocation lookup independently from namespaces. Here, we check that the requested can access the alloc namespace regardless of the declared request namespace. Ideally, we'd enforce that the declared request namespace matches the actual allocation namespace. Unfortunately, we haven't documented alloc endpoints as namespaced functions; we suspect starting to enforce this will be very disruptive and inappropriate for a nomad point release. As such, we maintain current behavior that doesn't require passing the proper namespace in request. A future major release may start enforcing checking declared namespace.
962 lines
26 KiB
Go
962 lines
26 KiB
Go
package agent
|
|
|
|
import (
|
|
"archive/tar"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"reflect"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/golang/snappy"
|
|
"github.com/hashicorp/nomad/acl"
|
|
"github.com/hashicorp/nomad/client/allocdir"
|
|
"github.com/hashicorp/nomad/helper"
|
|
"github.com/hashicorp/nomad/helper/uuid"
|
|
"github.com/hashicorp/nomad/nomad/mock"
|
|
"github.com/hashicorp/nomad/nomad/structs"
|
|
"github.com/hashicorp/nomad/testutil"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestHTTP_AllocsList(t *testing.T) {
|
|
t.Parallel()
|
|
httpTest(t, nil, func(s *TestAgent) {
|
|
// Directly manipulate the state
|
|
state := s.Agent.server.State()
|
|
alloc1 := mock.Alloc()
|
|
testEvent := structs.NewTaskEvent(structs.TaskSiblingFailed)
|
|
var events1 []*structs.TaskEvent
|
|
events1 = append(events1, testEvent)
|
|
taskState := &structs.TaskState{Events: events1}
|
|
alloc1.TaskStates = make(map[string]*structs.TaskState)
|
|
alloc1.TaskStates["test"] = taskState
|
|
|
|
alloc2 := mock.Alloc()
|
|
alloc2.TaskStates = make(map[string]*structs.TaskState)
|
|
alloc2.TaskStates["test"] = taskState
|
|
|
|
state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID))
|
|
state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID))
|
|
err := state.UpsertAllocs(1000,
|
|
[]*structs.Allocation{alloc1, alloc2})
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// Make the HTTP request
|
|
req, err := http.NewRequest("GET", "/v1/allocations", nil)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
respW := httptest.NewRecorder()
|
|
|
|
// Make the request
|
|
obj, err := s.Server.AllocsRequest(respW, req)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// Check for the index
|
|
if respW.HeaderMap.Get("X-Nomad-Index") == "" {
|
|
t.Fatalf("missing index")
|
|
}
|
|
if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" {
|
|
t.Fatalf("missing known leader")
|
|
}
|
|
if respW.HeaderMap.Get("X-Nomad-LastContact") == "" {
|
|
t.Fatalf("missing last contact")
|
|
}
|
|
|
|
// Check the alloc
|
|
allocs := obj.([]*structs.AllocListStub)
|
|
if len(allocs) != 2 {
|
|
t.Fatalf("bad: %#v", allocs)
|
|
}
|
|
expectedMsg := "Task's sibling failed"
|
|
displayMsg1 := allocs[0].TaskStates["test"].Events[0].DisplayMessage
|
|
require.Equal(t, expectedMsg, displayMsg1, "DisplayMessage should be set")
|
|
displayMsg2 := allocs[0].TaskStates["test"].Events[0].DisplayMessage
|
|
require.Equal(t, expectedMsg, displayMsg2, "DisplayMessage should be set")
|
|
})
|
|
}
|
|
|
|
func TestHTTP_AllocsPrefixList(t *testing.T) {
|
|
t.Parallel()
|
|
httpTest(t, nil, func(s *TestAgent) {
|
|
// Directly manipulate the state
|
|
state := s.Agent.server.State()
|
|
|
|
alloc1 := mock.Alloc()
|
|
alloc1.ID = "aaaaaaaa-e8f7-fd38-c855-ab94ceb89706"
|
|
alloc2 := mock.Alloc()
|
|
alloc2.ID = "aaabbbbb-e8f7-fd38-c855-ab94ceb89706"
|
|
|
|
testEvent := structs.NewTaskEvent(structs.TaskSiblingFailed)
|
|
var events1 []*structs.TaskEvent
|
|
events1 = append(events1, testEvent)
|
|
taskState := &structs.TaskState{Events: events1}
|
|
alloc2.TaskStates = make(map[string]*structs.TaskState)
|
|
alloc2.TaskStates["test"] = taskState
|
|
|
|
summary1 := mock.JobSummary(alloc1.JobID)
|
|
summary2 := mock.JobSummary(alloc2.JobID)
|
|
if err := state.UpsertJobSummary(998, summary1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := state.UpsertJobSummary(999, summary2); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := state.UpsertAllocs(1000,
|
|
[]*structs.Allocation{alloc1, alloc2}); err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// Make the HTTP request
|
|
req, err := http.NewRequest("GET", "/v1/allocations?prefix=aaab", nil)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
respW := httptest.NewRecorder()
|
|
|
|
// Make the request
|
|
obj, err := s.Server.AllocsRequest(respW, req)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// Check for the index
|
|
if respW.HeaderMap.Get("X-Nomad-Index") == "" {
|
|
t.Fatalf("missing index")
|
|
}
|
|
if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" {
|
|
t.Fatalf("missing known leader")
|
|
}
|
|
if respW.HeaderMap.Get("X-Nomad-LastContact") == "" {
|
|
t.Fatalf("missing last contact")
|
|
}
|
|
|
|
// Check the alloc
|
|
n := obj.([]*structs.AllocListStub)
|
|
if len(n) != 1 {
|
|
t.Fatalf("bad: %#v", n)
|
|
}
|
|
|
|
// Check the identifier
|
|
if n[0].ID != alloc2.ID {
|
|
t.Fatalf("expected alloc ID: %v, Actual: %v", alloc2.ID, n[0].ID)
|
|
}
|
|
expectedMsg := "Task's sibling failed"
|
|
displayMsg1 := n[0].TaskStates["test"].Events[0].DisplayMessage
|
|
require.Equal(t, expectedMsg, displayMsg1, "DisplayMessage should be set")
|
|
|
|
})
|
|
}
|
|
|
|
func TestHTTP_AllocQuery(t *testing.T) {
|
|
t.Parallel()
|
|
httpTest(t, nil, func(s *TestAgent) {
|
|
// Directly manipulate the state
|
|
state := s.Agent.server.State()
|
|
alloc := mock.Alloc()
|
|
if err := state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err := state.UpsertAllocs(1000,
|
|
[]*structs.Allocation{alloc})
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// Make the HTTP request
|
|
req, err := http.NewRequest("GET", "/v1/allocation/"+alloc.ID, nil)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
respW := httptest.NewRecorder()
|
|
|
|
// Make the request
|
|
obj, err := s.Server.AllocSpecificRequest(respW, req)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// Check for the index
|
|
if respW.HeaderMap.Get("X-Nomad-Index") == "" {
|
|
t.Fatalf("missing index")
|
|
}
|
|
if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" {
|
|
t.Fatalf("missing known leader")
|
|
}
|
|
if respW.HeaderMap.Get("X-Nomad-LastContact") == "" {
|
|
t.Fatalf("missing last contact")
|
|
}
|
|
|
|
// Check the job
|
|
a := obj.(*structs.Allocation)
|
|
if a.ID != alloc.ID {
|
|
t.Fatalf("bad: %#v", a)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestHTTP_AllocQuery_Payload(t *testing.T) {
|
|
t.Parallel()
|
|
httpTest(t, nil, func(s *TestAgent) {
|
|
// Directly manipulate the state
|
|
state := s.Agent.server.State()
|
|
alloc := mock.Alloc()
|
|
if err := state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Insert Payload compressed
|
|
expected := []byte("hello world")
|
|
compressed := snappy.Encode(nil, expected)
|
|
alloc.Job.Payload = compressed
|
|
|
|
err := state.UpsertAllocs(1000, []*structs.Allocation{alloc})
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// Make the HTTP request
|
|
req, err := http.NewRequest("GET", "/v1/allocation/"+alloc.ID, nil)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
respW := httptest.NewRecorder()
|
|
|
|
// Make the request
|
|
obj, err := s.Server.AllocSpecificRequest(respW, req)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// Check for the index
|
|
if respW.HeaderMap.Get("X-Nomad-Index") == "" {
|
|
t.Fatalf("missing index")
|
|
}
|
|
if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" {
|
|
t.Fatalf("missing known leader")
|
|
}
|
|
if respW.HeaderMap.Get("X-Nomad-LastContact") == "" {
|
|
t.Fatalf("missing last contact")
|
|
}
|
|
|
|
// Check the job
|
|
a := obj.(*structs.Allocation)
|
|
if a.ID != alloc.ID {
|
|
t.Fatalf("bad: %#v", a)
|
|
}
|
|
|
|
// Check the payload is decompressed
|
|
if !reflect.DeepEqual(a.Job.Payload, expected) {
|
|
t.Fatalf("Payload not decompressed properly; got %#v; want %#v", a.Job.Payload, expected)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestHTTP_AllocRestart(t *testing.T) {
|
|
t.Parallel()
|
|
require := require.New(t)
|
|
|
|
// Validates that all methods of forwarding the request are processed correctly
|
|
httpTest(t, nil, func(s *TestAgent) {
|
|
// Local node, local resp
|
|
{
|
|
// Make the HTTP request
|
|
buf := encodeReq(map[string]string{})
|
|
req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
respW := httptest.NewRecorder()
|
|
|
|
// Make the request
|
|
_, err = s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.True(structs.IsErrUnknownAllocation(err))
|
|
}
|
|
|
|
// Local node, server resp
|
|
{
|
|
srv := s.server
|
|
s.server = nil
|
|
|
|
buf := encodeReq(map[string]string{})
|
|
req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf)
|
|
require.Nil(err)
|
|
|
|
respW := httptest.NewRecorder()
|
|
_, err = s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.True(structs.IsErrUnknownAllocation(err))
|
|
|
|
s.server = srv
|
|
}
|
|
|
|
// no client, server resp
|
|
{
|
|
c := s.client
|
|
s.client = nil
|
|
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
n, err := s.server.State().NodeByID(nil, c.NodeID())
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return n != nil, nil
|
|
}, func(err error) {
|
|
t.Fatalf("should have client: %v", err)
|
|
})
|
|
|
|
buf := encodeReq(map[string]string{})
|
|
req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf)
|
|
require.Nil(err)
|
|
|
|
respW := httptest.NewRecorder()
|
|
_, err = s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.True(structs.IsErrUnknownAllocation(err))
|
|
|
|
s.client = c
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestHTTP_AllocRestart_ACL(t *testing.T) {
|
|
t.Parallel()
|
|
require := require.New(t)
|
|
|
|
httpACLTest(t, nil, func(s *TestAgent) {
|
|
state := s.Agent.server.State()
|
|
|
|
// If there's no token, we expect the request to fail.
|
|
{
|
|
buf := encodeReq(map[string]string{})
|
|
req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf)
|
|
require.NoError(err)
|
|
|
|
respW := httptest.NewRecorder()
|
|
_, err = s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.True(structs.IsErrUnknownAllocation(err), "(%T) %v", err, err)
|
|
}
|
|
|
|
// Try request with an invalid token and expect it to fail
|
|
{
|
|
buf := encodeReq(map[string]string{})
|
|
req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf)
|
|
require.NoError(err)
|
|
|
|
respW := httptest.NewRecorder()
|
|
token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyWrite))
|
|
setToken(req, token)
|
|
_, err = s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.True(structs.IsErrUnknownAllocation(err), "(%T) %v", err, err)
|
|
}
|
|
|
|
// Try request with a valid token
|
|
// Still returns an error because the alloc does not exist
|
|
{
|
|
buf := encodeReq(map[string]string{})
|
|
req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf)
|
|
require.NoError(err)
|
|
|
|
respW := httptest.NewRecorder()
|
|
policy := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityAllocLifecycle})
|
|
token := mock.CreatePolicyAndToken(t, state, 1007, "valid", policy)
|
|
setToken(req, token)
|
|
_, err = s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.True(structs.IsErrUnknownAllocation(err), "(%T) %v", err, err)
|
|
}
|
|
|
|
// Try request with a management token
|
|
// Still returns an error because the alloc does not exist
|
|
{
|
|
buf := encodeReq(map[string]string{})
|
|
req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf)
|
|
require.NoError(err)
|
|
|
|
respW := httptest.NewRecorder()
|
|
setToken(req, s.RootToken)
|
|
_, err = s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.True(structs.IsErrUnknownAllocation(err))
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestHTTP_AllocStop(t *testing.T) {
|
|
t.Parallel()
|
|
httpTest(t, nil, func(s *TestAgent) {
|
|
// Directly manipulate the state
|
|
state := s.Agent.server.State()
|
|
alloc := mock.Alloc()
|
|
require := require.New(t)
|
|
require.NoError(state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID)))
|
|
|
|
require.NoError(state.UpsertAllocs(1000, []*structs.Allocation{alloc}))
|
|
|
|
// Test that the happy path works
|
|
{
|
|
// Make the HTTP request
|
|
req, err := http.NewRequest("POST", "/v1/allocation/"+alloc.ID+"/stop", nil)
|
|
require.NoError(err)
|
|
respW := httptest.NewRecorder()
|
|
|
|
// Make the request
|
|
obj, err := s.Server.AllocSpecificRequest(respW, req)
|
|
require.NoError(err)
|
|
|
|
a := obj.(*structs.AllocStopResponse)
|
|
require.NotEmpty(a.EvalID, "missing eval")
|
|
require.NotEmpty(a.Index, "missing index")
|
|
headerIndex, _ := strconv.ParseUint(respW.Header().Get("X-Nomad-Index"), 10, 64)
|
|
require.Equal(a.Index, headerIndex)
|
|
}
|
|
|
|
// Test that we 404 when the allocid is invalid
|
|
{
|
|
// Make the HTTP request
|
|
req, err := http.NewRequest("POST", "/v1/allocation/"+uuid.Generate()+"/stop", nil)
|
|
require.NoError(err)
|
|
respW := httptest.NewRecorder()
|
|
|
|
// Make the request
|
|
_, err = s.Server.AllocSpecificRequest(respW, req)
|
|
require.NotNil(err)
|
|
if !strings.Contains(err.Error(), allocNotFoundErr) {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestHTTP_AllocStats(t *testing.T) {
|
|
t.Parallel()
|
|
require := require.New(t)
|
|
|
|
httpTest(t, nil, func(s *TestAgent) {
|
|
// Local node, local resp
|
|
{
|
|
// Make the HTTP request
|
|
req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/stats", uuid.Generate()), nil)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
respW := httptest.NewRecorder()
|
|
|
|
// Make the request
|
|
_, err = s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.True(structs.IsErrUnknownAllocation(err))
|
|
}
|
|
|
|
// Local node, server resp
|
|
{
|
|
srv := s.server
|
|
s.server = nil
|
|
|
|
req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/stats", uuid.Generate()), nil)
|
|
require.Nil(err)
|
|
|
|
respW := httptest.NewRecorder()
|
|
_, err = s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.True(structs.IsErrUnknownAllocation(err))
|
|
|
|
s.server = srv
|
|
}
|
|
|
|
// no client, server resp
|
|
{
|
|
c := s.client
|
|
s.client = nil
|
|
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
n, err := s.server.State().NodeByID(nil, c.NodeID())
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return n != nil, nil
|
|
}, func(err error) {
|
|
t.Fatalf("should have client: %v", err)
|
|
})
|
|
|
|
req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/stats", uuid.Generate()), nil)
|
|
require.Nil(err)
|
|
|
|
respW := httptest.NewRecorder()
|
|
_, err = s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.True(structs.IsErrUnknownAllocation(err))
|
|
|
|
s.client = c
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestHTTP_AllocStats_ACL(t *testing.T) {
|
|
t.Parallel()
|
|
require := require.New(t)
|
|
|
|
httpACLTest(t, nil, func(s *TestAgent) {
|
|
state := s.Agent.server.State()
|
|
|
|
// Make the HTTP request
|
|
req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/stats", uuid.Generate()), nil)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// Try request without a token and expect failure
|
|
{
|
|
respW := httptest.NewRecorder()
|
|
_, err := s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.True(structs.IsErrUnknownAllocation(err), "(%T) %v", err, err)
|
|
}
|
|
|
|
// Try request with an invalid token and expect failure
|
|
{
|
|
respW := httptest.NewRecorder()
|
|
token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyWrite))
|
|
setToken(req, token)
|
|
_, err := s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.True(structs.IsErrUnknownAllocation(err), "(%T) %v", err, err)
|
|
}
|
|
|
|
// Try request with a valid token
|
|
// Still returns an error because the alloc does not exist
|
|
{
|
|
respW := httptest.NewRecorder()
|
|
policy := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})
|
|
token := mock.CreatePolicyAndToken(t, state, 1007, "valid", policy)
|
|
setToken(req, token)
|
|
_, err := s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.True(structs.IsErrUnknownAllocation(err), "(%T) %v", err, err)
|
|
}
|
|
|
|
// Try request with a management token
|
|
// Still returns an error because the alloc does not exist
|
|
{
|
|
respW := httptest.NewRecorder()
|
|
setToken(req, s.RootToken)
|
|
_, err := s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.True(structs.IsErrUnknownAllocation(err))
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestHTTP_AllocSnapshot(t *testing.T) {
|
|
t.Parallel()
|
|
httpTest(t, nil, func(s *TestAgent) {
|
|
// Make the HTTP request
|
|
req, err := http.NewRequest("GET", "/v1/client/allocation/123/snapshot", nil)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
respW := httptest.NewRecorder()
|
|
|
|
// Make the request
|
|
_, err = s.Server.ClientAllocRequest(respW, req)
|
|
if !strings.Contains(err.Error(), allocNotFoundErr) {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestHTTP_AllocSnapshot_WithMigrateToken(t *testing.T) {
|
|
t.Parallel()
|
|
require := require.New(t)
|
|
httpACLTest(t, nil, func(s *TestAgent) {
|
|
// Request without a token fails
|
|
req, err := http.NewRequest("GET", "/v1/client/allocation/123/snapshot", nil)
|
|
require.Nil(err)
|
|
|
|
// Make the unauthorized request
|
|
respW := httptest.NewRecorder()
|
|
_, err = s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.EqualError(err, structs.ErrPermissionDenied.Error())
|
|
|
|
// Create an allocation
|
|
alloc := mock.Alloc()
|
|
|
|
validMigrateToken, err := structs.GenerateMigrateToken(alloc.ID, s.Agent.Client().Node().SecretID)
|
|
require.Nil(err)
|
|
|
|
// Request with a token succeeds
|
|
url := fmt.Sprintf("/v1/client/allocation/%s/snapshot", alloc.ID)
|
|
req, err = http.NewRequest("GET", url, nil)
|
|
require.Nil(err)
|
|
|
|
req.Header.Set("X-Nomad-Token", validMigrateToken)
|
|
|
|
// Make the unauthorized request
|
|
respW = httptest.NewRecorder()
|
|
_, err = s.Server.ClientAllocRequest(respW, req)
|
|
require.NotContains(err.Error(), structs.ErrPermissionDenied.Error())
|
|
})
|
|
}
|
|
|
|
// TestHTTP_AllocSnapshot_Atomic ensures that when a client encounters an error
|
|
// snapshotting a valid tar is not returned.
|
|
func TestHTTP_AllocSnapshot_Atomic(t *testing.T) {
|
|
t.Parallel()
|
|
httpTest(t, func(c *Config) {
|
|
// Disable the schedulers
|
|
c.Server.NumSchedulers = helper.IntToPtr(0)
|
|
}, func(s *TestAgent) {
|
|
// Create an alloc
|
|
state := s.server.State()
|
|
alloc := mock.Alloc()
|
|
alloc.Job.TaskGroups[0].Tasks[0].Driver = "mock_driver"
|
|
alloc.Job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{
|
|
"run_for": "30s",
|
|
}
|
|
alloc.NodeID = s.client.NodeID()
|
|
state.UpsertJobSummary(998, mock.JobSummary(alloc.JobID))
|
|
if err := state.UpsertAllocs(1000, []*structs.Allocation{alloc.Copy()}); err != nil {
|
|
t.Fatalf("error upserting alloc: %v", err)
|
|
}
|
|
|
|
// Wait for the client to run it
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
if _, err := s.client.GetAllocState(alloc.ID); err != nil {
|
|
return false, err
|
|
}
|
|
|
|
serverAlloc, err := state.AllocByID(nil, alloc.ID)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
return serverAlloc.ClientStatus == structs.AllocClientStatusRunning, fmt.Errorf(serverAlloc.ClientStatus)
|
|
}, func(err error) {
|
|
t.Fatalf("client not running alloc: %v", err)
|
|
})
|
|
|
|
// Now write to its shared dir
|
|
allocDirI, err := s.client.GetAllocFS(alloc.ID)
|
|
if err != nil {
|
|
t.Fatalf("unable to find alloc dir: %v", err)
|
|
}
|
|
allocDir := allocDirI.(*allocdir.AllocDir)
|
|
|
|
// Remove the task dir to break Snapshot
|
|
os.RemoveAll(allocDir.TaskDirs["web"].LocalDir)
|
|
|
|
// require Snapshot fails
|
|
if err := allocDir.Snapshot(ioutil.Discard); err != nil {
|
|
t.Logf("[DEBUG] agent.test: snapshot returned error: %v", err)
|
|
} else {
|
|
t.Errorf("expected Snapshot() to fail but it did not")
|
|
}
|
|
|
|
// Make the HTTP request to ensure the Snapshot error is
|
|
// propagated through to the HTTP layer. Since the tar is
|
|
// streamed over a 200 HTTP response the only way to signal an
|
|
// error is by writing a marker file.
|
|
respW := httptest.NewRecorder()
|
|
req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/snapshot", alloc.ID), nil)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// Make the request via the mux to make sure the error returned
|
|
// by Snapshot is properly propagated via HTTP
|
|
s.Server.mux.ServeHTTP(respW, req)
|
|
resp := respW.Result()
|
|
r := tar.NewReader(resp.Body)
|
|
errorFilename := allocdir.SnapshotErrorFilename(alloc.ID)
|
|
markerFound := false
|
|
markerContents := ""
|
|
for {
|
|
header, err := r.Next()
|
|
if err != nil {
|
|
if err != io.EOF {
|
|
// Huh, I wonder how a non-EOF error can happen?
|
|
t.Errorf("Unexpected error while streaming: %v", err)
|
|
}
|
|
break
|
|
}
|
|
|
|
if markerFound {
|
|
// No more files should be found after the failure marker
|
|
t.Errorf("Next file found after error marker: %s", header.Name)
|
|
break
|
|
}
|
|
|
|
if header.Name == errorFilename {
|
|
// Found it!
|
|
markerFound = true
|
|
buf := make([]byte, int(header.Size))
|
|
if _, err := r.Read(buf); err != nil && err != io.EOF {
|
|
t.Errorf("Unexpected error reading error marker %s: %v", errorFilename, err)
|
|
} else {
|
|
markerContents = string(buf)
|
|
}
|
|
}
|
|
}
|
|
|
|
if !markerFound {
|
|
t.Fatalf("marker file %s not written; bad tar will be treated as good!", errorFilename)
|
|
}
|
|
if markerContents == "" {
|
|
t.Fatalf("marker file %s empty", markerContents)
|
|
} else {
|
|
t.Logf("EXPECTED snapshot error: %s", markerContents)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestHTTP_AllocGC(t *testing.T) {
|
|
t.Parallel()
|
|
require := require.New(t)
|
|
path := fmt.Sprintf("/v1/client/allocation/%s/gc", uuid.Generate())
|
|
httpTest(t, nil, func(s *TestAgent) {
|
|
// Local node, local resp
|
|
{
|
|
req, err := http.NewRequest("GET", path, nil)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
respW := httptest.NewRecorder()
|
|
_, err = s.Server.ClientAllocRequest(respW, req)
|
|
if !structs.IsErrUnknownAllocation(err) {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
}
|
|
|
|
// Local node, server resp
|
|
{
|
|
srv := s.server
|
|
s.server = nil
|
|
|
|
req, err := http.NewRequest("GET", path, nil)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
respW := httptest.NewRecorder()
|
|
_, err = s.Server.ClientAllocRequest(respW, req)
|
|
if !structs.IsErrUnknownAllocation(err) {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
|
|
s.server = srv
|
|
}
|
|
|
|
// no client, server resp
|
|
{
|
|
c := s.client
|
|
s.client = nil
|
|
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
n, err := s.server.State().NodeByID(nil, c.NodeID())
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return n != nil, nil
|
|
}, func(err error) {
|
|
t.Fatalf("should have client: %v", err)
|
|
})
|
|
|
|
req, err := http.NewRequest("GET", path, nil)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
respW := httptest.NewRecorder()
|
|
_, err = s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
if !structs.IsErrUnknownAllocation(err) {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
|
|
s.client = c
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestHTTP_AllocGC_ACL(t *testing.T) {
|
|
t.Parallel()
|
|
require := require.New(t)
|
|
path := fmt.Sprintf("/v1/client/allocation/%s/gc", uuid.Generate())
|
|
|
|
httpACLTest(t, nil, func(s *TestAgent) {
|
|
state := s.Agent.server.State()
|
|
|
|
// Make the HTTP request
|
|
req, err := http.NewRequest("GET", path, nil)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
// Try request without a token and expect failure
|
|
{
|
|
respW := httptest.NewRecorder()
|
|
_, err := s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.True(structs.IsErrUnknownAllocation(err), "(%T) %v", err, err)
|
|
}
|
|
|
|
// Try request with an invalid token and expect failure
|
|
{
|
|
respW := httptest.NewRecorder()
|
|
token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyWrite))
|
|
setToken(req, token)
|
|
_, err := s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.True(structs.IsErrUnknownAllocation(err), "(%T) %v", err, err)
|
|
}
|
|
|
|
// Try request with a valid token
|
|
// Still returns an error because the alloc does not exist
|
|
{
|
|
respW := httptest.NewRecorder()
|
|
policy := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})
|
|
token := mock.CreatePolicyAndToken(t, state, 1007, "valid", policy)
|
|
setToken(req, token)
|
|
_, err := s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.True(structs.IsErrUnknownAllocation(err), "(%T) %v", err, err)
|
|
}
|
|
|
|
// Try request with a management token
|
|
// Still returns an error because the alloc does not exist
|
|
{
|
|
respW := httptest.NewRecorder()
|
|
setToken(req, s.RootToken)
|
|
_, err := s.Server.ClientAllocRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.True(structs.IsErrUnknownAllocation(err))
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestHTTP_AllocAllGC(t *testing.T) {
|
|
t.Parallel()
|
|
require := require.New(t)
|
|
httpTest(t, nil, func(s *TestAgent) {
|
|
// Local node, local resp
|
|
{
|
|
req, err := http.NewRequest("GET", "/v1/client/gc", nil)
|
|
if err != nil {
|
|
t.Fatalf("err: %v", err)
|
|
}
|
|
|
|
respW := httptest.NewRecorder()
|
|
_, err = s.Server.ClientGCRequest(respW, req)
|
|
if err != nil {
|
|
t.Fatalf("unexpected err: %v", err)
|
|
}
|
|
}
|
|
|
|
// Local node, server resp
|
|
{
|
|
srv := s.server
|
|
s.server = nil
|
|
|
|
req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/gc?node_id=%s", uuid.Generate()), nil)
|
|
require.Nil(err)
|
|
|
|
respW := httptest.NewRecorder()
|
|
_, err = s.Server.ClientGCRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.Contains(err.Error(), "Unknown node")
|
|
|
|
s.server = srv
|
|
}
|
|
|
|
// client stats from server, should not error
|
|
{
|
|
c := s.client
|
|
s.client = nil
|
|
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
n, err := s.server.State().NodeByID(nil, c.NodeID())
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return n != nil, nil
|
|
}, func(err error) {
|
|
t.Fatalf("should have client: %v", err)
|
|
})
|
|
|
|
req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/gc?node_id=%s", c.NodeID()), nil)
|
|
require.Nil(err)
|
|
|
|
respW := httptest.NewRecorder()
|
|
_, err = s.Server.ClientGCRequest(respW, req)
|
|
require.Nil(err)
|
|
|
|
s.client = c
|
|
}
|
|
})
|
|
|
|
}
|
|
|
|
func TestHTTP_AllocAllGC_ACL(t *testing.T) {
|
|
t.Parallel()
|
|
require := require.New(t)
|
|
httpACLTest(t, nil, func(s *TestAgent) {
|
|
state := s.Agent.server.State()
|
|
|
|
// Make the HTTP request
|
|
req, err := http.NewRequest("GET", "/v1/client/gc", nil)
|
|
require.Nil(err)
|
|
|
|
// Try request without a token and expect failure
|
|
{
|
|
respW := httptest.NewRecorder()
|
|
_, err := s.Server.ClientGCRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.Equal(err.Error(), structs.ErrPermissionDenied.Error())
|
|
}
|
|
|
|
// Try request with an invalid token and expect failure
|
|
{
|
|
respW := httptest.NewRecorder()
|
|
token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyRead))
|
|
setToken(req, token)
|
|
_, err := s.Server.ClientGCRequest(respW, req)
|
|
require.NotNil(err)
|
|
require.Equal(err.Error(), structs.ErrPermissionDenied.Error())
|
|
}
|
|
|
|
// Try request with a valid token
|
|
{
|
|
respW := httptest.NewRecorder()
|
|
token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.NodePolicy(acl.PolicyWrite))
|
|
setToken(req, token)
|
|
_, err := s.Server.ClientGCRequest(respW, req)
|
|
require.Nil(err)
|
|
require.Equal(http.StatusOK, respW.Code)
|
|
}
|
|
|
|
// Try request with a management token
|
|
{
|
|
respW := httptest.NewRecorder()
|
|
setToken(req, s.RootToken)
|
|
_, err := s.Server.ClientGCRequest(respW, req)
|
|
require.Nil(err)
|
|
require.Equal(http.StatusOK, respW.Code)
|
|
}
|
|
})
|
|
}
|