2020-07-13 21:53:10 +00:00
|
|
|
package allocrunner
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"net"
|
|
|
|
"os"
|
|
|
|
"path/filepath"
|
|
|
|
"sync"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
hclog "github.com/hashicorp/go-hclog"
|
|
|
|
"github.com/hashicorp/nomad/client/allocdir"
|
|
|
|
"github.com/hashicorp/nomad/client/allocrunner/interfaces"
|
|
|
|
"github.com/hashicorp/nomad/nomad/structs"
|
|
|
|
"github.com/hashicorp/nomad/nomad/structs/config"
|
|
|
|
"github.com/pkg/errors"
|
|
|
|
)
|
|
|
|
|
2020-07-29 14:11:42 +00:00
|
|
|
func tgFirstNetworkIsBridge(tg *structs.TaskGroup) bool {
|
|
|
|
if len(tg.Networks) < 1 || tg.Networks[0].Mode != "bridge" {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2020-07-13 21:53:10 +00:00
|
|
|
const (
|
|
|
|
consulHTTPSocketHookName = "consul_http_socket"
|
|
|
|
)
|
|
|
|
|
|
|
|
type consulHTTPSockHook struct {
|
|
|
|
logger hclog.Logger
|
|
|
|
|
2020-07-29 14:14:24 +00:00
|
|
|
// lock synchronizes proxy and alloc which may be mutated and read concurrently
|
|
|
|
// via Prerun, Update, and Postrun.
|
|
|
|
lock sync.Mutex
|
|
|
|
alloc *structs.Allocation
|
|
|
|
proxy *httpSocketProxy
|
2020-07-13 21:53:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func newConsulHTTPSocketHook(logger hclog.Logger, alloc *structs.Allocation, allocDir *allocdir.AllocDir, config *config.ConsulConfig) *consulHTTPSockHook {
|
|
|
|
return &consulHTTPSockHook{
|
|
|
|
alloc: alloc,
|
|
|
|
proxy: newHTTPSocketProxy(logger, allocDir, config),
|
|
|
|
logger: logger.Named(consulHTTPSocketHookName),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (*consulHTTPSockHook) Name() string {
|
|
|
|
return consulHTTPSocketHookName
|
|
|
|
}
|
|
|
|
|
|
|
|
// shouldRun returns true if the alloc contains at least one connect native
|
|
|
|
// task and has a network configured in bridge mode
|
|
|
|
//
|
|
|
|
// todo(shoenig): what about CNI networks?
|
|
|
|
func (h *consulHTTPSockHook) shouldRun() bool {
|
|
|
|
tg := h.alloc.Job.LookupTaskGroup(h.alloc.TaskGroup)
|
2020-07-29 14:11:42 +00:00
|
|
|
|
|
|
|
// we must be in bridge networking and at least one connect native task
|
|
|
|
if !tgFirstNetworkIsBridge(tg) {
|
2020-07-13 21:53:10 +00:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, service := range tg.Services {
|
|
|
|
if service.Connect.IsNative() {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
func (h *consulHTTPSockHook) Prerun() error {
|
|
|
|
h.lock.Lock()
|
|
|
|
defer h.lock.Unlock()
|
|
|
|
|
|
|
|
if !h.shouldRun() {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
return h.proxy.run(h.alloc)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (h *consulHTTPSockHook) Update(req *interfaces.RunnerUpdateRequest) error {
|
|
|
|
h.lock.Lock()
|
|
|
|
defer h.lock.Unlock()
|
|
|
|
|
|
|
|
h.alloc = req.Alloc
|
|
|
|
|
|
|
|
if !h.shouldRun() {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
return h.proxy.run(h.alloc)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (h *consulHTTPSockHook) Postrun() error {
|
|
|
|
h.lock.Lock()
|
|
|
|
defer h.lock.Unlock()
|
|
|
|
|
|
|
|
if err := h.proxy.stop(); err != nil {
|
|
|
|
// Only log a failure to stop, worst case is the proxy leaks a goroutine.
|
2020-07-29 14:25:29 +00:00
|
|
|
h.logger.Warn("error stopping Consul HTTP proxy", "error", err)
|
2020-07-13 21:53:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
type httpSocketProxy struct {
|
|
|
|
logger hclog.Logger
|
|
|
|
allocDir *allocdir.AllocDir
|
|
|
|
config *config.ConsulConfig
|
|
|
|
|
|
|
|
ctx context.Context
|
|
|
|
cancel func()
|
|
|
|
doneCh chan struct{}
|
|
|
|
runOnce bool
|
|
|
|
}
|
|
|
|
|
|
|
|
func newHTTPSocketProxy(logger hclog.Logger, allocDir *allocdir.AllocDir, config *config.ConsulConfig) *httpSocketProxy {
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
return &httpSocketProxy{
|
|
|
|
logger: logger,
|
|
|
|
allocDir: allocDir,
|
|
|
|
config: config,
|
|
|
|
ctx: ctx,
|
|
|
|
cancel: cancel,
|
|
|
|
doneCh: make(chan struct{}),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// run the httpSocketProxy for the given allocation.
|
|
|
|
//
|
|
|
|
// Assumes locking done by the calling alloc runner.
|
|
|
|
func (p *httpSocketProxy) run(alloc *structs.Allocation) error {
|
|
|
|
// Only run once.
|
|
|
|
if p.runOnce {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Never restart.
|
|
|
|
select {
|
|
|
|
case <-p.doneCh:
|
|
|
|
p.logger.Trace("consul http socket proxy already shutdown; exiting")
|
|
|
|
return nil
|
|
|
|
case <-p.ctx.Done():
|
|
|
|
p.logger.Trace("consul http socket proxy already done; exiting")
|
|
|
|
return nil
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
|
|
|
|
// consul http dest addr
|
|
|
|
destAddr := p.config.Addr
|
|
|
|
if destAddr == "" {
|
|
|
|
return errors.New("consul address must be set on nomad client")
|
|
|
|
}
|
|
|
|
|
|
|
|
hostHTTPSockPath := filepath.Join(p.allocDir.AllocDir, allocdir.AllocHTTPSocket)
|
|
|
|
if err := maybeRemoveOldSocket(hostHTTPSockPath); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
listener, err := net.Listen("unix", hostHTTPSockPath)
|
|
|
|
if err != nil {
|
|
|
|
return errors.Wrap(err, "unable to create unix socket for Consul HTTP endpoint")
|
|
|
|
}
|
|
|
|
|
|
|
|
// The Consul HTTP socket should be usable by all users in case a task is
|
|
|
|
// running as a non-privileged user. Unix does not allow setting domain
|
|
|
|
// socket permissions when creating the file, so we must manually call
|
2020-07-29 14:25:29 +00:00
|
|
|
// chmod afterwards.
|
2020-07-13 21:53:10 +00:00
|
|
|
if err := os.Chmod(hostHTTPSockPath, os.ModePerm); err != nil {
|
|
|
|
return errors.Wrap(err, "unable to set permissions on unix socket")
|
|
|
|
}
|
|
|
|
|
|
|
|
go func() {
|
|
|
|
proxy(p.ctx, p.logger, destAddr, listener)
|
|
|
|
p.cancel()
|
|
|
|
close(p.doneCh)
|
|
|
|
}()
|
|
|
|
|
|
|
|
p.runOnce = true
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *httpSocketProxy) stop() error {
|
|
|
|
p.cancel()
|
|
|
|
|
|
|
|
// if proxy was never run, no need to wait before shutdown
|
|
|
|
if !p.runOnce {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
select {
|
|
|
|
case <-p.doneCh:
|
2020-07-29 14:25:29 +00:00
|
|
|
case <-time.After(socketProxyStopWaitTime):
|
2020-07-13 21:53:10 +00:00
|
|
|
return errSocketProxyTimeout
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func maybeRemoveOldSocket(socketPath string) error {
|
|
|
|
_, err := os.Stat(socketPath)
|
|
|
|
if err == nil {
|
|
|
|
if err = os.Remove(socketPath); err != nil {
|
|
|
|
return errors.Wrap(err, "unable to remove existing unix socket")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|