60ee243149
Fixes https://github.com/hashicorp/nomad/issues/5593 Executor seems to die unexpectedly after nomad agent dies or is restarted. The crash seems to occur at the first log message after the nomad agent dies. To ease debugging we forward executor log messages to executor.log as well as to Stderr. `go-plugin` sets up plugins with Stderr pointing to a pipe being read by plugin client, the nomad agent in our case[1]. When the nomad agent dies, the pipe is closed, and any subsequent executor logs fail with ErrClosedPipe and SIGPIPE signal. SIGPIPE results into executor process dying. I considered adding a handler to ignore SIGPIPE, but hc-log library currently panics when logging write operation fails[2] This we opt to revert to v0.8 behavior of exclusively writing logs to executor.log, while we investigate alternative options. [1] https://github.com/hashicorp/nomad/blob/v0.9.0/vendor/github.com/hashicorp/go-plugin/client.go#L528-L535 [2] https://github.com/hashicorp/nomad/blob/v0.9.0/vendor/github.com/hashicorp/go-hclog/int.go#L320-L323
67 lines
1.4 KiB
Go
67 lines
1.4 KiB
Go
package command
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"strings"
|
|
|
|
hclog "github.com/hashicorp/go-hclog"
|
|
log "github.com/hashicorp/go-hclog"
|
|
plugin "github.com/hashicorp/go-plugin"
|
|
|
|
"github.com/hashicorp/nomad/drivers/shared/executor"
|
|
"github.com/hashicorp/nomad/plugins/base"
|
|
)
|
|
|
|
type ExecutorPluginCommand struct {
|
|
Meta
|
|
}
|
|
|
|
func (e *ExecutorPluginCommand) Help() string {
|
|
helpText := `
|
|
This is a command used by Nomad internally to launch an executor plugin"
|
|
`
|
|
return strings.TrimSpace(helpText)
|
|
}
|
|
|
|
func (e *ExecutorPluginCommand) Synopsis() string {
|
|
return "internal - launch an executor plugin"
|
|
}
|
|
|
|
func (e *ExecutorPluginCommand) Run(args []string) int {
|
|
if len(args) != 1 {
|
|
e.Ui.Error("json configuration not provided")
|
|
return 1
|
|
}
|
|
|
|
config := args[0]
|
|
var executorConfig executor.ExecutorConfig
|
|
if err := json.Unmarshal([]byte(config), &executorConfig); err != nil {
|
|
return 1
|
|
}
|
|
|
|
f, err := os.OpenFile(executorConfig.LogFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
|
|
if err != nil {
|
|
e.Ui.Error(err.Error())
|
|
return 1
|
|
}
|
|
|
|
// Create the logger
|
|
logger := log.New(&log.LoggerOptions{
|
|
Level: hclog.LevelFromString(executorConfig.LogLevel),
|
|
JSONFormat: true,
|
|
Output: f,
|
|
})
|
|
|
|
plugin.Serve(&plugin.ServeConfig{
|
|
HandshakeConfig: base.Handshake,
|
|
Plugins: executor.GetPluginMap(
|
|
logger,
|
|
executorConfig.FSIsolation,
|
|
),
|
|
GRPCServer: plugin.DefaultGRPCServer,
|
|
Logger: logger,
|
|
})
|
|
return 0
|
|
}
|