backport of commit 21eccf8b8df7868c7d454f8ba42d5bec5235a69e (#20866)
Co-authored-by: Anton Averchenkov <84287187+averche@users.noreply.github.com>
This commit is contained in:
hc-github-team-secure-vault-core
GitHub
parent
3a3e931cfd
commit
afef4629c8
|
|
@ -98,6 +98,7 @@ test_packages[6]+=" $base/command/agentproxyshared/cache/cacheboltdb"
|
|||
test_packages[6]+=" $base/command/agentproxyshared/cache/cachememdb"
|
||||
test_packages[6]+=" $base/command/agentproxyshared/cache/keymanager"
|
||||
test_packages[6]+=" $base/command/agent/config"
|
||||
test_packages[6]+=" $base/command/agent/exec"
|
||||
test_packages[6]+=" $base/command/proxy/config"
|
||||
test_packages[6]+=" $base/command/config"
|
||||
test_packages[6]+=" $base/command/token"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
```release-note:improvement
|
||||
agent: Add integration tests for agent running in process supervisor mode
|
||||
```
|
||||
|
|
@ -0,0 +1,321 @@
|
|||
package exec
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ctconfig "github.com/hashicorp/consul-template/config"
|
||||
"github.com/hashicorp/go-hclog"
|
||||
"github.com/hashicorp/go-retryablehttp"
|
||||
|
||||
"github.com/hashicorp/vault/command/agent/config"
|
||||
"github.com/hashicorp/vault/sdk/helper/logging"
|
||||
"github.com/hashicorp/vault/sdk/helper/pointerutil"
|
||||
)
|
||||
|
||||
func fakeVaultServer() *httptest.Server {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/v1/kv/my-app/creds", func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintln(w, `{
|
||||
"request_id": "8af096e9-518c-7351-eff5-5ba20554b21f",
|
||||
"lease_id": "",
|
||||
"renewable": false,
|
||||
"lease_duration": 0,
|
||||
"data": {
|
||||
"data": {
|
||||
"password": "s3cr3t",
|
||||
"user": "app-user"
|
||||
},
|
||||
"metadata": {
|
||||
"created_time": "2019-10-07T22:18:44.233247Z",
|
||||
"deletion_time": "",
|
||||
"destroyed": false,
|
||||
"version": 3
|
||||
}
|
||||
},
|
||||
"wrap_info": null,
|
||||
"warnings": null,
|
||||
"auth": null
|
||||
}`)
|
||||
})
|
||||
|
||||
return httptest.NewServer(mux)
|
||||
}
|
||||
|
||||
// TestExecServer_Run tests various scenarios of using vault agent as a process
|
||||
// supervisor. At its core is a sample application referred to as 'test app',
|
||||
// compiled from ./test-app/main.go. Each test case verifies that the test app
|
||||
// is started and/or stopped correctly by exec.Server.Run(). There are 3
|
||||
// high-level scenarios we want to test for:
|
||||
//
|
||||
// 1. test app is started and is injected with environment variables
|
||||
// 2. test app exits early (either with zero or non-zero extit code)
|
||||
// 3. test app needs to be stopped (and restarted) by exec.Server
|
||||
func TestExecServer_Run(t *testing.T) {
|
||||
fakeVault := fakeVaultServer()
|
||||
defer fakeVault.Close()
|
||||
|
||||
// we must build a test-app binary since 'go run' does not propagate signals correctly
|
||||
goBinary, err := exec.LookPath("go")
|
||||
if err != nil {
|
||||
t.Fatalf("could not find go binary on path: %s", err)
|
||||
}
|
||||
|
||||
testAppBinary := filepath.Join(os.TempDir(), "test-app")
|
||||
|
||||
if err := exec.Command(goBinary, "build", "-o", testAppBinary, "./test-app").Run(); err != nil {
|
||||
t.Fatalf("could not build the test application: %s", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := os.Remove(testAppBinary); err != nil {
|
||||
t.Fatalf("could not remove %q test application: %s", testAppBinary, err)
|
||||
}
|
||||
}()
|
||||
|
||||
testCases := map[string]struct {
|
||||
// skip this test case
|
||||
skip bool
|
||||
skipReason string
|
||||
|
||||
// inputs to the exec server
|
||||
envTemplates []*ctconfig.TemplateConfig
|
||||
|
||||
// test app parameters
|
||||
testAppArgs []string
|
||||
testAppStopSignal os.Signal
|
||||
testAppPort int
|
||||
|
||||
// simulate a shutdown of agent, which, in turn stops the test app
|
||||
simulateShutdown bool
|
||||
simulateShutdownWaitDuration time.Duration
|
||||
|
||||
// expected results
|
||||
expected map[string]string
|
||||
expectedTestDuration time.Duration
|
||||
expectedError error
|
||||
}{
|
||||
"ensure_environment_variables_are_injected": {
|
||||
envTemplates: []*ctconfig.TemplateConfig{{
|
||||
Contents: pointerutil.StringPtr(`{{ with secret "kv/my-app/creds" }}{{ .Data.data.user }}{{ end }}`),
|
||||
MapToEnvironmentVariable: pointerutil.StringPtr("MY_USER"),
|
||||
}, {
|
||||
Contents: pointerutil.StringPtr(`{{ with secret "kv/my-app/creds" }}{{ .Data.data.password }}{{ end }}`),
|
||||
MapToEnvironmentVariable: pointerutil.StringPtr("MY_PASSWORD"),
|
||||
}},
|
||||
testAppArgs: []string{"--stop-after", "10s"},
|
||||
testAppStopSignal: syscall.SIGTERM,
|
||||
testAppPort: 34001,
|
||||
expected: map[string]string{
|
||||
"MY_USER": "app-user",
|
||||
"MY_PASSWORD": "s3cr3t",
|
||||
},
|
||||
expectedTestDuration: 15 * time.Second,
|
||||
expectedError: nil,
|
||||
},
|
||||
|
||||
"test_app_exits_early": {
|
||||
envTemplates: []*ctconfig.TemplateConfig{{
|
||||
Contents: pointerutil.StringPtr(`{{ with secret "kv/my-app/creds" }}{{ .Data.data.user }}{{ end }}`),
|
||||
MapToEnvironmentVariable: pointerutil.StringPtr("MY_USER"),
|
||||
}},
|
||||
testAppArgs: []string{"--stop-after", "1s"},
|
||||
testAppStopSignal: syscall.SIGTERM,
|
||||
testAppPort: 34002,
|
||||
expectedTestDuration: 15 * time.Second,
|
||||
expectedError: &ProcessExitError{0},
|
||||
},
|
||||
|
||||
"test_app_exits_early_non_zero": {
|
||||
envTemplates: []*ctconfig.TemplateConfig{{
|
||||
Contents: pointerutil.StringPtr(`{{ with secret "kv/my-app/creds" }}{{ .Data.data.user }}{{ end }}`),
|
||||
MapToEnvironmentVariable: pointerutil.StringPtr("MY_USER"),
|
||||
}},
|
||||
testAppArgs: []string{"--stop-after", "1s", "--exit-code", "5"},
|
||||
testAppStopSignal: syscall.SIGTERM,
|
||||
testAppPort: 34003,
|
||||
expectedTestDuration: 15 * time.Second,
|
||||
expectedError: &ProcessExitError{5},
|
||||
},
|
||||
|
||||
"send_sigterm_expect_test_app_exit": {
|
||||
envTemplates: []*ctconfig.TemplateConfig{{
|
||||
Contents: pointerutil.StringPtr(`{{ with secret "kv/my-app/creds" }}{{ .Data.data.user }}{{ end }}`),
|
||||
MapToEnvironmentVariable: pointerutil.StringPtr("MY_USER"),
|
||||
}},
|
||||
testAppArgs: []string{"--stop-after", "30s", "--sleep-after-stop-signal", "1s"},
|
||||
testAppStopSignal: syscall.SIGTERM,
|
||||
testAppPort: 34004,
|
||||
simulateShutdown: true,
|
||||
simulateShutdownWaitDuration: 3 * time.Second,
|
||||
expectedTestDuration: 15 * time.Second,
|
||||
expectedError: nil,
|
||||
},
|
||||
|
||||
"send_sigusr1_expect_test_app_exit": {
|
||||
envTemplates: []*ctconfig.TemplateConfig{{
|
||||
Contents: pointerutil.StringPtr(`{{ with secret "kv/my-app/creds" }}{{ .Data.data.user }}{{ end }}`),
|
||||
MapToEnvironmentVariable: pointerutil.StringPtr("MY_USER"),
|
||||
}},
|
||||
testAppArgs: []string{"--stop-after", "30s", "--sleep-after-stop-signal", "1s", "--use-sigusr1"},
|
||||
testAppStopSignal: syscall.SIGUSR1,
|
||||
testAppPort: 34005,
|
||||
simulateShutdown: true,
|
||||
simulateShutdownWaitDuration: 3 * time.Second,
|
||||
expectedTestDuration: 15 * time.Second,
|
||||
expectedError: nil,
|
||||
},
|
||||
|
||||
"test_app_ignores_stop_signal": {
|
||||
skip: true,
|
||||
skipReason: "This test currently fails with 'go test -race' (see hashicorp/consul-template/issues/1753).",
|
||||
envTemplates: []*ctconfig.TemplateConfig{{
|
||||
Contents: pointerutil.StringPtr(`{{ with secret "kv/my-app/creds" }}{{ .Data.data.user }}{{ end }}`),
|
||||
MapToEnvironmentVariable: pointerutil.StringPtr("MY_USER"),
|
||||
}},
|
||||
testAppArgs: []string{"--stop-after", "60s", "--sleep-after-stop-signal", "60s"},
|
||||
testAppStopSignal: syscall.SIGTERM,
|
||||
testAppPort: 34006,
|
||||
simulateShutdown: true,
|
||||
simulateShutdownWaitDuration: 32 * time.Second, // the test app should be stopped immediately after 30s
|
||||
expectedTestDuration: 45 * time.Second,
|
||||
expectedError: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for name, testCase := range testCases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if testCase.skip {
|
||||
t.Skip(testCase.skipReason)
|
||||
}
|
||||
|
||||
ctx, cancelContextFunc := context.WithTimeout(context.Background(), testCase.expectedTestDuration)
|
||||
defer cancelContextFunc()
|
||||
|
||||
testAppCommand := []string{
|
||||
testAppBinary,
|
||||
"--port",
|
||||
strconv.Itoa(testCase.testAppPort),
|
||||
}
|
||||
|
||||
execServer := NewServer(&ServerConfig{
|
||||
Logger: logging.NewVaultLogger(hclog.Trace),
|
||||
AgentConfig: &config.Config{
|
||||
Vault: &config.Vault{
|
||||
Address: fakeVault.URL,
|
||||
Retry: &config.Retry{
|
||||
NumRetries: 3,
|
||||
},
|
||||
},
|
||||
Exec: &config.ExecConfig{
|
||||
RestartOnSecretChanges: "always",
|
||||
Command: append(testAppCommand, testCase.testAppArgs...),
|
||||
RestartStopSignal: testCase.testAppStopSignal,
|
||||
},
|
||||
EnvTemplates: testCase.envTemplates,
|
||||
},
|
||||
LogLevel: hclog.Trace,
|
||||
LogWriter: hclog.DefaultOutput,
|
||||
})
|
||||
|
||||
// start the exec server
|
||||
var (
|
||||
execServerErrCh = make(chan error)
|
||||
execServerTokenCh = make(chan string, 1)
|
||||
)
|
||||
go func() {
|
||||
execServerErrCh <- execServer.Run(ctx, execServerTokenCh)
|
||||
}()
|
||||
|
||||
// send a dummy token to kick off the server
|
||||
execServerTokenCh <- "my-token"
|
||||
|
||||
// ensure the test app is running after 3 seconds
|
||||
var (
|
||||
testAppAddr = fmt.Sprintf("http://localhost:%d", testCase.testAppPort)
|
||||
testAppStartedCh = make(chan error)
|
||||
)
|
||||
if testCase.expectedError == nil {
|
||||
time.AfterFunc(500*time.Millisecond, func() {
|
||||
_, err := retryablehttp.Head(testAppAddr)
|
||||
testAppStartedCh <- err
|
||||
})
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatal("timeout reached before templates were rendered")
|
||||
|
||||
case err := <-execServerErrCh:
|
||||
if testCase.expectedError == nil && err != nil {
|
||||
t.Fatalf("exec server did not expect an error, got: %v", err)
|
||||
}
|
||||
|
||||
if errors.Is(err, testCase.expectedError) {
|
||||
t.Fatalf("exec server expected error %v; got %v", testCase.expectedError, err)
|
||||
}
|
||||
|
||||
t.Log("exec server exited without an error")
|
||||
|
||||
return
|
||||
|
||||
case err := <-testAppStartedCh:
|
||||
if testCase.expectedError == nil && err != nil {
|
||||
t.Fatalf("test app could not be started")
|
||||
}
|
||||
|
||||
t.Log("test app started successfully")
|
||||
}
|
||||
|
||||
// simulate a shutdown of agent, which, in turn stops the test app
|
||||
if testCase.simulateShutdown {
|
||||
cancelContextFunc()
|
||||
|
||||
time.Sleep(testCase.simulateShutdownWaitDuration)
|
||||
|
||||
// check if the test app is still alive
|
||||
if _, err := http.Head(testAppAddr); err == nil {
|
||||
t.Fatalf("the test app is still alive %v after a simulated shutdown!", testCase.simulateShutdownWaitDuration)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// verify the environment variables
|
||||
resp, err := http.Get(testAppAddr)
|
||||
if err != nil {
|
||||
t.Fatalf("error making request to the test app: %s", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
decoder := json.NewDecoder(resp.Body)
|
||||
var response struct {
|
||||
EnvironmentVariables map[string]string `json:"environment_variables"`
|
||||
ProcessID int `json:"process_id"`
|
||||
}
|
||||
if err := decoder.Decode(&response); err != nil {
|
||||
t.Fatalf("unable to parse response from test app: %s", err)
|
||||
}
|
||||
|
||||
for key, expectedValue := range testCase.expected {
|
||||
actualValue, ok := response.EnvironmentVariables[key]
|
||||
if !ok {
|
||||
t.Fatalf("expected the test app to return %q environment variable", key)
|
||||
}
|
||||
if expectedValue != actualValue {
|
||||
t.Fatalf("expected environment variable %s to have a value of %q but it has a value of %q", key, expectedValue, actualValue)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,150 @@
|
|||
package main
|
||||
|
||||
// This is a test application that is used by TestExecServer_Run to verify
|
||||
// the behavior of vault agent running as a process supervisor.
|
||||
//
|
||||
// The app will automatically exit after 1 minute or the --stop-after interval,
|
||||
// whichever comes first. It also can serve its loaded environment variables on
|
||||
// the given --port. This app will also return the given --exit-code and
|
||||
// terminate on SIGTERM unless --use-sigusr1 is specified.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
port uint
|
||||
ignoreStopSignal bool
|
||||
sleepAfterStopSignal time.Duration
|
||||
useSigusr1StopSignal bool
|
||||
stopAfter time.Duration
|
||||
exitCode int
|
||||
)
|
||||
|
||||
func init() {
|
||||
flag.UintVar(&port, "port", 34000, "port to run the test app on")
|
||||
flag.DurationVar(&sleepAfterStopSignal, "sleep-after-stop-signal", 1*time.Second, "time to sleep after getting the signal before exiting")
|
||||
flag.BoolVar(&useSigusr1StopSignal, "use-sigusr1", false, "use SIGUSR1 as the stop signal, instead of the default SIGTERM")
|
||||
flag.DurationVar(&stopAfter, "stop-after", 0, "stop the process after duration (overrides all other flags if set)")
|
||||
flag.IntVar(&exitCode, "exit-code", 0, "exit code to return when this script exits")
|
||||
}
|
||||
|
||||
type Response struct {
|
||||
EnvironmentVariables map[string]string `json:"environment_variables"`
|
||||
ProcessID int `json:"process_id"`
|
||||
}
|
||||
|
||||
func newResponse() Response {
|
||||
respEnv := make(map[string]string, len(os.Environ()))
|
||||
for _, envVar := range os.Environ() {
|
||||
tokens := strings.Split(envVar, "=")
|
||||
respEnv[tokens[0]] = tokens[1]
|
||||
}
|
||||
|
||||
return Response{
|
||||
EnvironmentVariables: respEnv,
|
||||
ProcessID: os.Getpid(),
|
||||
}
|
||||
}
|
||||
|
||||
func handler(w http.ResponseWriter, r *http.Request) {
|
||||
var buf bytes.Buffer
|
||||
encoder := json.NewEncoder(&buf)
|
||||
if r.URL.Query().Get("pretty") == "1" {
|
||||
encoder.SetIndent("", " ")
|
||||
}
|
||||
if err := encoder.Encode(newResponse()); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write(buf.Bytes())
|
||||
}
|
||||
|
||||
func main() {
|
||||
logger := log.New(os.Stderr, "test-app: ", log.LstdFlags)
|
||||
|
||||
if err := run(logger); err != nil {
|
||||
log.Fatalf("error: %v\n", err)
|
||||
}
|
||||
|
||||
logger.Printf("exit code: %d\n", exitCode)
|
||||
|
||||
os.Exit(exitCode)
|
||||
}
|
||||
|
||||
func run(logger *log.Logger) error {
|
||||
/* */ logger.Println("run: started")
|
||||
defer logger.Println("run: done")
|
||||
|
||||
ctx, cancelContextFunc := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancelContextFunc()
|
||||
|
||||
flag.Parse()
|
||||
|
||||
server := http.Server{
|
||||
Addr: fmt.Sprintf(":%d", port),
|
||||
Handler: http.HandlerFunc(handler),
|
||||
ReadTimeout: 20 * time.Second,
|
||||
WriteTimeout: 20 * time.Second,
|
||||
IdleTimeout: 20 * time.Second,
|
||||
}
|
||||
|
||||
doneCh := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
defer close(doneCh)
|
||||
|
||||
stopSignal := make(chan os.Signal, 1)
|
||||
if useSigusr1StopSignal {
|
||||
signal.Notify(stopSignal, syscall.SIGUSR1)
|
||||
} else {
|
||||
signal.Notify(stopSignal, syscall.SIGTERM)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
logger.Println("context done: exiting")
|
||||
|
||||
case s := <-stopSignal:
|
||||
logger.Printf("signal %q: received\n", s)
|
||||
|
||||
if sleepAfterStopSignal > 0 {
|
||||
logger.Printf("signal %q: sleeping for %v simulate cleanup\n", s, sleepAfterStopSignal)
|
||||
time.Sleep(sleepAfterStopSignal)
|
||||
}
|
||||
|
||||
case <-time.After(stopAfter):
|
||||
logger.Printf("stopping after: %v\n", stopAfter)
|
||||
}
|
||||
|
||||
if err := server.Shutdown(context.Background()); err != nil {
|
||||
log.Printf("server shutdown error: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
logger.Printf("server %s: started\n", server.Addr)
|
||||
|
||||
if err := server.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
|
||||
return fmt.Errorf("could not start the server: %v", err)
|
||||
}
|
||||
|
||||
logger.Printf("server %s: done\n", server.Addr)
|
||||
|
||||
<-doneCh
|
||||
|
||||
return nil
|
||||
}
|
||||
Loading…
Reference in New Issue