962921f86c
Currently, nomad "plugin" processes (e.g. executor, logmon, docker_logger) are started as CLI
commands to be handled by command CLI framework. Plugin launchers use
`discover.NomadBinary()` to identify the binary and start it.
This has few downsides: The trivial one is that when running tests, one
must re-compile the nomad binary as the tests need to invoke the nomad
executable to start plugin. This is frequently overlooked, resulting in
puzzlement.
The more significant issue with `executor` in particular is in relation
to external driver:
* Plugin must identify the path of invoking nomad binary, which is not
trivial; `discvoer.NomadBinary()` now returns the path to the plugin
rather than to nomad, preventing external drivers from launching
executors.
* The external driver may get a different version of executor than it
expects (specially if we make a binary incompatible change in future).
This commit addresses both downside by having the plugin invocation
handling through an `init()` call, similar to how libcontainer init
handler is done in [1] and recommened by libcontainer [2]. `init()`
will be invoked and handled properly in tests and external drivers.
For external drivers, this change will cause external drivers to launch
the executor that's compiled against.
There a are a couple of downsides to this approach:
* These specific packages (i.e executor, logmon, and dockerlog) need to
be careful in use of `init()`, package initializers. Must avoid having
command execution rely on any other init in the package. I prefixed
files with `z_` (golang processes files in lexical order), but ensured
we don't depend on order.
* The command handling is spread in multiple packages making it a bit
less obvious how plugin starts are handled.
[1] drivers/shared/executor/libcontainer_nsenter_linux.go
[2] eb4aeed24f/libcontainer (using-libcontainer)
101 lines
2.4 KiB
Go
101 lines
2.4 KiB
Go
package docklog
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"os/exec"
|
|
|
|
hclog "github.com/hashicorp/go-hclog"
|
|
plugin "github.com/hashicorp/go-plugin"
|
|
"github.com/hashicorp/nomad/drivers/docker/docklog/proto"
|
|
"github.com/hashicorp/nomad/plugins/base"
|
|
"google.golang.org/grpc"
|
|
)
|
|
|
|
const PluginName = "docker_logger"
|
|
|
|
// LaunchDockerLogger launches an instance of DockerLogger
|
|
func LaunchDockerLogger(logger hclog.Logger) (DockerLogger, *plugin.Client, error) {
|
|
logger = logger.Named(PluginName)
|
|
bin, err := os.Executable()
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
client := plugin.NewClient(&plugin.ClientConfig{
|
|
HandshakeConfig: base.Handshake,
|
|
Plugins: map[string]plugin.Plugin{
|
|
PluginName: &Plugin{impl: NewDockerLogger(logger)},
|
|
},
|
|
Cmd: exec.Command(bin, PluginName),
|
|
AllowedProtocols: []plugin.Protocol{
|
|
plugin.ProtocolGRPC,
|
|
},
|
|
Logger: logger,
|
|
})
|
|
|
|
rpcClient, err := client.Client()
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
raw, err := rpcClient.Dispense(PluginName)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
l := raw.(DockerLogger)
|
|
return l, client, nil
|
|
|
|
}
|
|
|
|
func ReattachDockerLogger(reattachCfg *plugin.ReattachConfig) (DockerLogger, *plugin.Client, error) {
|
|
client := plugin.NewClient(&plugin.ClientConfig{
|
|
HandshakeConfig: base.Handshake,
|
|
Plugins: map[string]plugin.Plugin{
|
|
PluginName: &Plugin{impl: NewDockerLogger(hclog.L().Named(PluginName))},
|
|
},
|
|
Reattach: reattachCfg,
|
|
AllowedProtocols: []plugin.Protocol{
|
|
plugin.ProtocolGRPC,
|
|
},
|
|
})
|
|
|
|
rpcClient, err := client.Client()
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
raw, err := rpcClient.Dispense(PluginName)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
l := raw.(DockerLogger)
|
|
return l, client, nil
|
|
}
|
|
|
|
// Plugin is the go-plugin implementation
|
|
type Plugin struct {
|
|
plugin.NetRPCUnsupportedPlugin
|
|
impl DockerLogger
|
|
}
|
|
|
|
func NewPlugin(impl DockerLogger) *Plugin {
|
|
return &Plugin{impl: impl}
|
|
}
|
|
|
|
// GRPCServer registered the server side implementation with the grpc server
|
|
func (p *Plugin) GRPCServer(broker *plugin.GRPCBroker, s *grpc.Server) error {
|
|
proto.RegisterDockerLoggerServer(s, &dockerLoggerServer{
|
|
impl: p.impl,
|
|
broker: broker,
|
|
})
|
|
return nil
|
|
}
|
|
|
|
// GRPCClient returns a client side implementation of the plugin
|
|
func (p *Plugin) GRPCClient(ctx context.Context, broker *plugin.GRPCBroker, c *grpc.ClientConn) (interface{}, error) {
|
|
return &dockerLoggerClient{client: proto.NewDockerLoggerClient(c)}, nil
|
|
}
|