package proxyprocess import ( "io/ioutil" "os" "os/exec" "path/filepath" "strconv" "syscall" "testing" "time" "github.com/hashicorp/consul/sdk/testutil/retry" "github.com/hashicorp/go-uuid" "github.com/stretchr/testify/require" ) func TestDaemon_impl(t *testing.T) { var _ Proxy = new(Daemon) } func TestDaemonStartStop(t *testing.T) { t.Parallel() require := require.New(t) td, closer := testTempDir(t) defer closer() path := filepath.Join(td, "file") uuid, err := uuid.GenerateUUID() require.NoError(err) cmd, destroy := helperProcess("start-stop", path) defer destroy() d := &Daemon{ Command: cmd, ProxyID: "tubes", ProxyToken: uuid, Logger: testLogger, } require.NoError(d.Start()) defer d.Stop() // Wait for the file to exist retry.Run(t, func(r *retry.R) { _, err := os.Stat(path) if err == nil { return } r.Fatalf("error: %s", err) }) // Verify that the contents of the file is the token. This verifies // that we properly passed the token as an env var. data, err := ioutil.ReadFile(path) require.NoError(err) require.Equal("tubes:"+uuid, string(data)) // Stop the process require.NoError(d.Stop()) // File should no longer exist. retry.Run(t, func(r *retry.R) { _, err := os.Stat(path) if os.IsNotExist(err) { return } // err might be nil here but that's okay r.Fatalf("should not exist: %s", err) }) } func TestDaemonRestart(t *testing.T) { t.Parallel() require := require.New(t) td, closer := testTempDir(t) defer closer() path := filepath.Join(td, "file") cmd, destroy := helperProcess("restart", path) defer destroy() d := &Daemon{ Command: cmd, Logger: testLogger, } require.NoError(d.Start()) defer d.Stop() // Wait for the file to exist. We save the func so we can reuse the test. waitFile := func() { retry.Run(t, func(r *retry.R) { _, err := os.Stat(path) if err == nil { return } r.Fatalf("error waiting for path: %s", err) }) } waitFile() // Delete the file require.NoError(os.Remove(path)) // File should re-appear because the process is restart waitFile() } func TestDaemonLaunchesNewProcessGroup(t *testing.T) { t.Parallel() require := require.New(t) td, closer := testTempDir(t) defer closer() path := filepath.Join(td, "file") pidPath := filepath.Join(td, "child.pid") // Start the parent process wrapping a start-stop test. The parent is acting // as our "agent". We need an extra indirection to be able to kill the "agent" // and still be running the test process. parentCmd, destroy := helperProcess("parent", pidPath, "start-stop", path) defer destroy() // We MUST run this as a separate process group otherwise the Kill below will // kill this test process (and possibly your shell/editor that launched it!) parentCmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} require.NoError(parentCmd.Start()) // Wait for the pid file to exist so we know parent is running retry.Run(t, func(r *retry.R) { _, err := os.Stat(pidPath) if err == nil { return } r.Fatalf("error: %s", err) }) // And wait for the actual file to be sure the child is running (it should be // since parent doesn't write PID until child starts but the child might not // have completed the write to disk yet which causes flakiness below). retry.Run(t, func(r *retry.R) { _, err := os.Stat(path) if err == nil { return } r.Fatalf("error: %s", err) }) // Get the child PID bs, err := ioutil.ReadFile(pidPath) require.NoError(err) pid, err := strconv.Atoi(string(bs)) require.NoError(err) proc, err := os.FindProcess(pid) require.NoError(err) // Always cleanup child process after defer func() { if proc != nil { proc.Kill() } }() // Now kill the parent's whole process group and wait for it pgid, err := syscall.Getpgid(parentCmd.Process.Pid) require.NoError(err) // Yep the minus PGid is how you kill a whole process group in unix... no idea // how this works on windows. We TERM no KILL since we rely on the child // catching the signal and deleting it's file to detect correct behavior. require.NoError(syscall.Kill(-pgid, syscall.SIGTERM)) _, err = parentCmd.Process.Wait() require.NoError(err) // The child should still be running so file should still be there _, err = os.Stat(path) require.NoError(err, "child should still be running") // TEST PART 2 - verify that adopting an existing process works and picks up // monitoring even though it's not a child. We can't do this accurately with // Restart test since even if we create a new `Daemon` object the test process // is still the parent. We need the indirection of the `parent` test helper to // actually verify "adoption" on restart works. // Start a new parent that will "adopt" the existing child even though it will // not be an actual child process. fosterCmd, destroy := helperProcess("parent", pidPath, "start-stop", path) defer destroy() // Don't care about it being same process group this time as we will just kill // it normally. require.NoError(fosterCmd.Start()) defer func() { // Clean up the daemon and wait for it to prevent it becoming a zombie. fosterCmd.Process.Kill() fosterCmd.Wait() }() // The child should still be running so file should still be there _, err = os.Stat(path) require.NoError(err, "child should still be running") { // Get the child PID - it shouldn't have changed and should be running bs2, err := ioutil.ReadFile(pidPath) require.NoError(err) pid2, err := strconv.Atoi(string(bs2)) require.NoError(err) // Defer a cleanup (til end of test function) proc, err := os.FindProcess(pid) require.NoError(err) defer func() { proc.Kill() }() require.Equal(pid, pid2) t.Logf("Child PID was %d and still %d", pid, pid2) } // Now killing the child directly should still be restarted by the Daemon require.NoError(proc.Kill()) proc = nil retry.Run(t, func(r *retry.R) { // Get the child PID - it should have changed bs, err := ioutil.ReadFile(pidPath) r.Check(err) newPid, err := strconv.Atoi(string(bs)) r.Check(err) if newPid == pid { r.Fatalf("Child PID file not changed, Daemon not restarting it") } t.Logf("Child PID was %d and is now %d", pid, newPid) }) // I had to run through this test in debugger a lot of times checking ps state // by hand at different points to convince myself it was doing the right // thing. It doesn't help that with verbose logs on it seems that the stdio // from the `parent` process can sometimes miss lines out due to timing. For // example the `[INFO] agent/proxy: daemon exited...` log from Daemon that // indicates that the child was detected to have failed and is restarting is // never output on my Mac at full speed. But if I run in debugger and have it // pause at the step after the child is killed above, then it shows. The // `[DEBUG] agent/proxy: starting proxy:` for the restart does always come // through though which is odd. I assume this is some odd quirk of timing // between processes and stdio or something but it makes debugging this stuff // even harder! // Let defer clean up the child process(es) // Get the NEW child PID bs, err = ioutil.ReadFile(pidPath) require.NoError(err) pid, err = strconv.Atoi(string(bs)) require.NoError(err) proc2, err := os.FindProcess(pid) require.NoError(err) // Always cleanup child process after defer func() { if proc2 != nil { proc2.Kill() } }() } func TestDaemonStop_kill(t *testing.T) { t.Parallel() require := require.New(t) td, closer := testTempDir(t) defer closer() path := filepath.Join(td, "file") cmd, destroy := helperProcess("stop-kill", path) defer destroy() d := &Daemon{ Command: cmd, ProxyToken: "hello", Logger: testLogger, gracefulWait: 200 * time.Millisecond, } require.NoError(d.Start()) // Wait for the file to exist retry.Run(t, func(r *retry.R) { _, err := os.Stat(path) if err == nil { return } r.Fatalf("error: %s", err) }) // Stop the process require.NoError(d.Stop()) // Stat the file so that we can get the mtime fi, err := os.Stat(path) require.NoError(err) mtime := fi.ModTime() // The mtime shouldn't change time.Sleep(100 * time.Millisecond) fi, err = os.Stat(path) require.NoError(err) require.Equal(mtime, fi.ModTime()) } func TestDaemonStop_killAdopted(t *testing.T) { t.Parallel() require := require.New(t) td, closer := testTempDir(t) defer closer() path := filepath.Join(td, "file") // In this test we want to ensure that graceful/ungraceful stop works with // processes that were adopted by current process but not started by it. (i.e. // we have to poll them not use Wait). // // We could use `parent` indirection to get a child that is actually not // started by this process but that's a lot of hoops to jump through on top of // an already complex multi-process test case. // // For now we rely on an implementation detail of Daemon which is potentially // brittle but beats lots of extra complexity here. Currently, if // Daemon.process is non-nil, the keepAlive loop will explicitly assume it's // not a child and so will use polling to monitor it. If we ever change that // it might invalidate this test and we would either need more indirection // here, or an alternative explicit signal on Daemon like Daemon.forcePoll to // ensure we are exercising that code path. // Start the "child" process childCmd, destroy := helperProcess("stop-kill", path) defer destroy() require.NoError(childCmd.Start()) go func() { childCmd.Wait() }() // Prevent it becoming a zombie when killed defer func() { childCmd.Process.Kill() }() // Create the Daemon cmd, destroy := helperProcess("stop-kill", path) defer destroy() d := &Daemon{ Command: cmd, ProxyToken: "hello", Logger: testLogger, gracefulWait: 200 * time.Millisecond, // Can't just set process as it will bypass intializing stopCh etc. } // Adopt the pid from a fake state snapshot (this correctly initializes Daemon // for adoption) fakeSnap := map[string]interface{}{ "Pid": childCmd.Process.Pid, "CommandPath": childCmd.Path, "CommandArgs": childCmd.Args, "CommandDir": childCmd.Dir, "CommandEnv": childCmd.Env, "ProxyToken": d.ProxyToken, } require.NoError(d.UnmarshalSnapshot(fakeSnap)) require.NoError(d.Start()) // Wait for the file to exist (child was already running so this doesn't // guarantee that Daemon is in "polling" state) retry.Run(t, func(r *retry.R) { _, err := os.Stat(path) if err == nil { return } r.Fatalf("error: %s", err) }) // Stop the process require.NoError(d.Stop()) // Stat the file so that we can get the mtime fi, err := os.Stat(path) require.NoError(err) mtime := fi.ModTime() // The mtime shouldn't change time.Sleep(100 * time.Millisecond) fi, err = os.Stat(path) require.NoError(err) require.Equal(mtime, fi.ModTime()) } func TestDaemonStart_pidFile(t *testing.T) { t.Parallel() require := require.New(t) td, closer := testTempDir(t) defer closer() path := filepath.Join(td, "file") pidPath := filepath.Join(td, "pid") uuid, err := uuid.GenerateUUID() require.NoError(err) cmd, destroy := helperProcess("start-once", path) defer destroy() d := &Daemon{ Command: cmd, ProxyToken: uuid, Logger: testLogger, PidPath: pidPath, } require.NoError(d.Start()) defer d.Stop() // Wait for the file to exist retry.Run(t, func(r *retry.R) { _, err := os.Stat(pidPath) if err == nil { return } r.Fatalf("error: %s", err) }) // Check the pid file pidRaw, err := ioutil.ReadFile(pidPath) require.NoError(err) require.NotEmpty(pidRaw) // Stop require.NoError(d.Stop()) // Pid file should be gone _, err = os.Stat(pidPath) require.True(os.IsNotExist(err)) } // Verify the pid file changes on restart func TestDaemonRestart_pidFile(t *testing.T) { t.Parallel() require := require.New(t) td, closer := testTempDir(t) defer closer() path := filepath.Join(td, "file") pidPath := filepath.Join(td, "pid") cmd, destroy := helperProcess("restart", path) defer destroy() d := &Daemon{ Command: cmd, Logger: testLogger, PidPath: pidPath, } require.NoError(d.Start()) defer d.Stop() // Wait for the file to exist. We save the func so we can reuse the test. waitFile := func(path string) { retry.Run(t, func(r *retry.R) { _, err := os.Stat(path) if err == nil { return } r.Fatalf("error waiting for path: %s", err) }) } waitFile(path) waitFile(pidPath) // Check the pid file pidRaw, err := ioutil.ReadFile(pidPath) require.NoError(err) require.NotEmpty(pidRaw) // Delete the file require.NoError(os.Remove(pidPath)) require.NoError(os.Remove(path)) // File should re-appear because the process is restart waitFile(path) waitFile(pidPath) // Check the pid file and it should not equal pidRaw2, err := ioutil.ReadFile(pidPath) require.NoError(err) require.NotEmpty(pidRaw2) require.NotEqual(pidRaw, pidRaw2) } func TestDaemonEqual(t *testing.T) { cases := []struct { Name string D1, D2 Proxy Expected bool }{ { "Different type", &Daemon{ Command: &exec.Cmd{}, }, &Noop{}, false, }, { "Nil", &Daemon{ Command: &exec.Cmd{}, }, nil, false, }, { "Equal", &Daemon{ Command: &exec.Cmd{}, }, &Daemon{ Command: &exec.Cmd{}, }, true, }, { "Different proxy ID", &Daemon{ Command: &exec.Cmd{Path: "/foo"}, ProxyID: "web", }, &Daemon{ Command: &exec.Cmd{Path: "/foo"}, ProxyID: "db", }, false, }, { "Different path", &Daemon{ Command: &exec.Cmd{Path: "/foo"}, }, &Daemon{ Command: &exec.Cmd{Path: "/bar"}, }, false, }, { "Different dir", &Daemon{ Command: &exec.Cmd{Dir: "/foo"}, }, &Daemon{ Command: &exec.Cmd{Dir: "/bar"}, }, false, }, { "Different args", &Daemon{ Command: &exec.Cmd{Args: []string{"foo"}}, }, &Daemon{ Command: &exec.Cmd{Args: []string{"bar"}}, }, false, }, { "Different token", &Daemon{ Command: &exec.Cmd{}, ProxyToken: "one", }, &Daemon{ Command: &exec.Cmd{}, ProxyToken: "two", }, false, }, } for _, tc := range cases { t.Run(tc.Name, func(t *testing.T) { actual := tc.D1.Equal(tc.D2) require.Equal(t, tc.Expected, actual) }) } } func TestDaemonMarshalSnapshot(t *testing.T) { cases := []struct { Name string Proxy Proxy Expected map[string]interface{} }{ { "stopped daemon", &Daemon{ Command: &exec.Cmd{Path: "/foo"}, }, nil, }, { "basic", &Daemon{ Command: &exec.Cmd{Path: "/foo"}, ProxyID: "web", process: &os.Process{Pid: 42}, }, map[string]interface{}{ "Pid": 42, "CommandPath": "/foo", "CommandArgs": []string(nil), "CommandDir": "", "CommandEnv": []string(nil), "ProxyToken": "", "ProxyID": "web", }, }, } for _, tc := range cases { t.Run(tc.Name, func(t *testing.T) { actual := tc.Proxy.MarshalSnapshot() require.Equal(t, tc.Expected, actual) }) } } func TestDaemonUnmarshalSnapshot(t *testing.T) { t.Parallel() require := require.New(t) td, closer := testTempDir(t) defer closer() path := filepath.Join(td, "file") uuid, err := uuid.GenerateUUID() require.NoError(err) cmd, destroy := helperProcess("start-stop", path) defer destroy() d := &Daemon{ Command: cmd, ProxyToken: uuid, Logger: testLogger, } defer d.Stop() require.NoError(d.Start()) // Wait for the file to exist retry.Run(t, func(r *retry.R) { _, err := os.Stat(path) if err == nil { return } r.Fatalf("error: %s", err) }) // Snapshot snap := d.MarshalSnapshot() // Stop the original daemon but keep it alive require.NoError(d.Close()) // Restore the second daemon d2 := &Daemon{Logger: testLogger} require.NoError(d2.UnmarshalSnapshot(snap)) // Verify the daemon is still running _, err = os.Stat(path) require.NoError(err) // Stop the process require.NoError(d2.Stop()) // File should no longer exist. retry.Run(t, func(r *retry.R) { _, err := os.Stat(path) if os.IsNotExist(err) { return } // err might be nil here but that's okay r.Fatalf("should not exist: %s", err) }) } func TestDaemonUnmarshalSnapshot_notRunning(t *testing.T) { t.Parallel() require := require.New(t) td, closer := testTempDir(t) defer closer() path := filepath.Join(td, "file") uuid, err := uuid.GenerateUUID() require.NoError(err) cmd, destroy := helperProcess("start-stop", path) defer destroy() d := &Daemon{ Command: cmd, ProxyToken: uuid, Logger: testLogger, } defer d.Stop() require.NoError(d.Start()) // Wait for the file to exist retry.Run(t, func(r *retry.R) { _, err := os.Stat(path) if err == nil { return } r.Fatalf("error: %s", err) }) // Snapshot snap := d.MarshalSnapshot() // Stop the original daemon require.NoError(d.Stop()) // Restore the second daemon d2 := &Daemon{Logger: testLogger} require.Error(d2.UnmarshalSnapshot(snap)) }