2015-08-16 01:34:47 +00:00
package agent
import (
2015-08-16 01:54:41 +00:00
"flag"
"fmt"
2015-08-16 20:54:49 +00:00
"io"
2016-02-12 00:05:31 +00:00
"log"
2015-08-16 20:54:49 +00:00
"os"
"os/signal"
2015-08-16 21:34:38 +00:00
"path/filepath"
2015-12-03 07:22:18 +00:00
"reflect"
2015-08-16 21:34:38 +00:00
"sort"
"strconv"
2015-08-16 01:34:47 +00:00
"strings"
2015-08-16 20:54:49 +00:00
"syscall"
2015-08-16 01:54:41 +00:00
"time"
2015-08-16 01:34:47 +00:00
2019-08-28 19:54:53 +00:00
metrics "github.com/armon/go-metrics"
2016-07-22 16:33:10 +00:00
"github.com/armon/go-metrics/circonus"
2016-10-15 08:49:33 +00:00
"github.com/armon/go-metrics/datadog"
2017-09-08 18:50:07 +00:00
"github.com/armon/go-metrics/prometheus"
2019-08-28 19:54:53 +00:00
checkpoint "github.com/hashicorp/go-checkpoint"
discover "github.com/hashicorp/go-discover"
hclog "github.com/hashicorp/go-hclog"
2019-02-21 18:41:54 +00:00
gsyslog "github.com/hashicorp/go-syslog"
2015-08-16 20:54:49 +00:00
"github.com/hashicorp/logutils"
2019-01-09 18:56:40 +00:00
"github.com/hashicorp/nomad/helper"
2020-12-14 16:07:27 +00:00
flaghelper "github.com/hashicorp/nomad/helper/flags"
2019-02-21 18:41:54 +00:00
gatedwriter "github.com/hashicorp/nomad/helper/gated-writer"
2019-02-19 23:53:14 +00:00
"github.com/hashicorp/nomad/helper/logging"
2019-11-11 20:16:07 +00:00
"github.com/hashicorp/nomad/helper/winsvc"
2021-10-13 22:50:46 +00:00
"github.com/hashicorp/nomad/nomad/structs"
2016-09-01 22:41:07 +00:00
"github.com/hashicorp/nomad/nomad/structs/config"
2017-08-16 22:42:15 +00:00
"github.com/hashicorp/nomad/version"
2015-08-16 01:34:47 +00:00
"github.com/mitchellh/cli"
2017-07-19 18:28:16 +00:00
"github.com/posener/complete"
2015-08-16 01:34:47 +00:00
)
2015-08-16 20:54:49 +00:00
// gracefulTimeout controls how long we wait before forcefully terminating
const gracefulTimeout = 5 * time . Second
2015-08-16 21:34:38 +00:00
// Command is a Command implementation that runs a Nomad agent.
2015-08-16 01:34:47 +00:00
// The command will not end unless a shutdown message is sent on the
// ShutdownCh. If two messages are sent on the ShutdownCh it will forcibly
// exit.
type Command struct {
2017-08-16 22:42:15 +00:00
Version * version . VersionInfo
Ui cli . Ui
ShutdownCh <- chan struct { }
2015-08-16 01:54:41 +00:00
2015-12-09 23:50:18 +00:00
args [ ] string
agent * Agent
2022-01-03 14:33:53 +00:00
httpServers [ ] * HTTPServer
2015-12-09 23:50:18 +00:00
logFilter * logutils . LevelFilter
logOutput io . Writer
2015-12-09 15:57:54 +00:00
retryJoinErrCh chan struct { }
2015-08-16 01:54:41 +00:00
}
func ( c * Command ) readConfig ( ) * Config {
2019-08-14 19:29:37 +00:00
var dev * devModeConfig
2015-08-16 01:54:41 +00:00
var configPath [ ] string
2015-10-02 00:19:52 +00:00
var servers string
var meta [ ] string
2015-09-22 03:38:43 +00:00
2015-09-22 17:44:31 +00:00
// Make a new, empty config.
cmdConfig := & Config {
2016-05-23 18:09:31 +00:00
Client : & ClientConfig { } ,
2017-10-05 13:26:26 +00:00
Consul : & config . ConsulConfig { } ,
2016-05-23 18:09:31 +00:00
Ports : & Ports { } ,
2018-05-30 23:51:55 +00:00
Server : & ServerConfig {
ServerJoin : & ServerJoin { } ,
} ,
Vault : & config . VaultConfig { } ,
ACL : & ACLConfig { } ,
2020-03-22 16:17:33 +00:00
Audit : & config . AuditConfig { } ,
2015-09-22 17:44:31 +00:00
}
2015-09-22 03:38:43 +00:00
2015-08-16 01:54:41 +00:00
flags := flag . NewFlagSet ( "agent" , flag . ContinueOnError )
flags . Usage = func ( ) { c . Ui . Error ( c . Help ( ) ) }
2015-09-22 04:37:55 +00:00
// Role options
2019-08-30 12:15:50 +00:00
var devMode bool
var devConnectMode bool
flags . BoolVar ( & devMode , "dev" , false , "" )
flags . BoolVar ( & devConnectMode , "dev-connect" , false , "" )
2015-09-22 03:38:43 +00:00
flags . BoolVar ( & cmdConfig . Server . Enabled , "server" , false , "" )
flags . BoolVar ( & cmdConfig . Client . Enabled , "client" , false , "" )
2015-09-22 04:37:55 +00:00
2015-09-22 19:40:21 +00:00
// Server-only options
flags . IntVar ( & cmdConfig . Server . BootstrapExpect , "bootstrap-expect" , 0 , "" )
2016-10-17 17:48:04 +00:00
flags . StringVar ( & cmdConfig . Server . EncryptKey , "encrypt" , "" , "gossip encryption key" )
2017-11-22 00:29:11 +00:00
flags . IntVar ( & cmdConfig . Server . RaftProtocol , "raft-protocol" , 0 , "" )
2018-05-30 23:51:55 +00:00
flags . BoolVar ( & cmdConfig . Server . RejoinAfterLeave , "rejoin" , false , "" )
flags . Var ( ( * flaghelper . StringFlag ) ( & cmdConfig . Server . ServerJoin . StartJoin ) , "join" , "" )
flags . Var ( ( * flaghelper . StringFlag ) ( & cmdConfig . Server . ServerJoin . RetryJoin ) , "retry-join" , "" )
flags . IntVar ( & cmdConfig . Server . ServerJoin . RetryMaxAttempts , "retry-max" , 0 , "" )
flags . Var ( ( flaghelper . FuncDurationVar ) ( func ( d time . Duration ) error {
cmdConfig . Server . ServerJoin . RetryInterval = d
return nil
} ) , "retry-interval" , "" )
2015-09-22 19:40:21 +00:00
2015-10-01 23:46:42 +00:00
// Client-only options
flags . StringVar ( & cmdConfig . Client . StateDir , "state-dir" , "" , "" )
flags . StringVar ( & cmdConfig . Client . AllocDir , "alloc-dir" , "" , "" )
flags . StringVar ( & cmdConfig . Client . NodeClass , "node-class" , "" , "" )
flags . StringVar ( & servers , "servers" , "" , "" )
2016-10-11 01:04:39 +00:00
flags . Var ( ( * flaghelper . StringFlag ) ( & meta ) , "meta" , "" )
2015-10-03 00:32:11 +00:00
flags . StringVar ( & cmdConfig . Client . NetworkInterface , "network-interface" , "" , "" )
flags . IntVar ( & cmdConfig . Client . NetworkSpeed , "network-speed" , 0 , "" )
2015-10-01 23:46:42 +00:00
2015-09-22 04:37:55 +00:00
// General options
2016-10-11 01:04:39 +00:00
flags . Var ( ( * flaghelper . StringFlag ) ( & configPath ) , "config" , "config" )
2015-09-22 04:37:55 +00:00
flags . StringVar ( & cmdConfig . BindAddr , "bind" , "" , "" )
flags . StringVar ( & cmdConfig . Region , "region" , "" , "" )
2015-09-22 03:38:43 +00:00
flags . StringVar ( & cmdConfig . DataDir , "data-dir" , "" , "" )
2018-08-30 20:43:09 +00:00
flags . StringVar ( & cmdConfig . PluginDir , "plugin-dir" , "" , "" )
2015-09-22 04:37:55 +00:00
flags . StringVar ( & cmdConfig . Datacenter , "dc" , "" , "" )
2015-09-22 22:23:30 +00:00
flags . StringVar ( & cmdConfig . LogLevel , "log-level" , "" , "" )
2019-01-11 19:36:26 +00:00
flags . BoolVar ( & cmdConfig . LogJson , "log-json" , false , "" )
2015-09-22 04:37:55 +00:00
flags . StringVar ( & cmdConfig . NodeName , "node" , "" , "" )
2017-10-05 13:26:26 +00:00
// Consul options
flags . StringVar ( & cmdConfig . Consul . Auth , "consul-auth" , "" , "" )
flags . Var ( ( flaghelper . FuncBoolVar ) ( func ( b bool ) error {
cmdConfig . Consul . AutoAdvertise = & b
return nil
} ) , "consul-auto-advertise" , "" )
flags . StringVar ( & cmdConfig . Consul . CAFile , "consul-ca-file" , "" , "" )
flags . StringVar ( & cmdConfig . Consul . CertFile , "consul-cert-file" , "" , "" )
2018-09-28 17:09:01 +00:00
flags . StringVar ( & cmdConfig . Consul . KeyFile , "consul-key-file" , "" , "" )
2017-10-05 13:26:26 +00:00
flags . Var ( ( flaghelper . FuncBoolVar ) ( func ( b bool ) error {
cmdConfig . Consul . ChecksUseAdvertise = & b
return nil
} ) , "consul-checks-use-advertise" , "" )
flags . Var ( ( flaghelper . FuncBoolVar ) ( func ( b bool ) error {
cmdConfig . Consul . ClientAutoJoin = & b
return nil
} ) , "consul-client-auto-join" , "" )
flags . StringVar ( & cmdConfig . Consul . ClientServiceName , "consul-client-service-name" , "" , "" )
2018-03-20 15:03:58 +00:00
flags . StringVar ( & cmdConfig . Consul . ClientHTTPCheckName , "consul-client-http-check-name" , "" , "" )
2017-10-05 13:26:26 +00:00
flags . StringVar ( & cmdConfig . Consul . ServerServiceName , "consul-server-service-name" , "" , "" )
2018-03-20 15:03:58 +00:00
flags . StringVar ( & cmdConfig . Consul . ServerHTTPCheckName , "consul-server-http-check-name" , "" , "" )
flags . StringVar ( & cmdConfig . Consul . ServerSerfCheckName , "consul-server-serf-check-name" , "" , "" )
flags . StringVar ( & cmdConfig . Consul . ServerRPCCheckName , "consul-server-rpc-check-name" , "" , "" )
2017-10-05 13:26:26 +00:00
flags . Var ( ( flaghelper . FuncBoolVar ) ( func ( b bool ) error {
cmdConfig . Consul . ServerAutoJoin = & b
return nil
} ) , "consul-server-auto-join" , "" )
flags . Var ( ( flaghelper . FuncBoolVar ) ( func ( b bool ) error {
cmdConfig . Consul . EnableSSL = & b
return nil
} ) , "consul-ssl" , "" )
flags . StringVar ( & cmdConfig . Consul . Token , "consul-token" , "" , "" )
flags . Var ( ( flaghelper . FuncBoolVar ) ( func ( b bool ) error {
cmdConfig . Consul . VerifySSL = & b
return nil
} ) , "consul-verify-ssl" , "" )
2017-11-10 20:47:59 +00:00
flags . StringVar ( & cmdConfig . Consul . Addr , "consul-address" , "" , "" )
2019-11-18 21:05:06 +00:00
flags . Var ( ( flaghelper . FuncBoolVar ) ( func ( b bool ) error {
cmdConfig . Consul . AllowUnauthenticated = & b
return nil
} ) , "consul-allow-unauthenticated" , "" )
2017-10-05 13:26:26 +00:00
2016-09-01 22:41:07 +00:00
// Vault options
2016-10-11 01:04:39 +00:00
flags . Var ( ( flaghelper . FuncBoolVar ) ( func ( b bool ) error {
cmdConfig . Vault . Enabled = & b
return nil
} ) , "vault-enabled" , "" )
flags . Var ( ( flaghelper . FuncBoolVar ) ( func ( b bool ) error {
cmdConfig . Vault . AllowUnauthenticated = & b
return nil
} ) , "vault-allow-unauthenticated" , "" )
2016-09-01 22:41:07 +00:00
flags . StringVar ( & cmdConfig . Vault . Token , "vault-token" , "" , "" )
flags . StringVar ( & cmdConfig . Vault . Addr , "vault-address" , "" , "" )
2019-04-03 11:58:34 +00:00
flags . StringVar ( & cmdConfig . Vault . Namespace , "vault-namespace" , "" , "" )
2017-01-27 22:11:34 +00:00
flags . StringVar ( & cmdConfig . Vault . Role , "vault-create-from-role" , "" , "" )
2016-10-25 00:23:48 +00:00
flags . StringVar ( & cmdConfig . Vault . TLSCaFile , "vault-ca-file" , "" , "" )
flags . StringVar ( & cmdConfig . Vault . TLSCaPath , "vault-ca-path" , "" , "" )
flags . StringVar ( & cmdConfig . Vault . TLSCertFile , "vault-cert-file" , "" , "" )
flags . StringVar ( & cmdConfig . Vault . TLSKeyFile , "vault-key-file" , "" , "" )
flags . Var ( ( flaghelper . FuncBoolVar ) ( func ( b bool ) error {
cmdConfig . Vault . TLSSkipVerify = & b
return nil
} ) , "vault-tls-skip-verify" , "" )
flags . StringVar ( & cmdConfig . Vault . TLSServerName , "vault-tls-server-name" , "" , "" )
2016-09-01 22:41:07 +00:00
2017-10-12 20:51:39 +00:00
// ACL options
flags . BoolVar ( & cmdConfig . ACL . Enabled , "acl-enabled" , false , "" )
flags . StringVar ( & cmdConfig . ACL . ReplicationToken , "acl-replication-token" , "" , "" )
2015-08-16 01:54:41 +00:00
if err := flags . Parse ( c . args ) ; err != nil {
return nil
}
2015-10-01 23:46:42 +00:00
// Split the servers.
if servers != "" {
cmdConfig . Client . Servers = strings . Split ( servers , "," )
}
// Parse the meta flags.
2015-10-02 00:21:32 +00:00
metaLength := len ( meta )
if metaLength != 0 {
cmdConfig . Client . Meta = make ( map [ string ] string , metaLength )
2015-10-01 23:46:42 +00:00
for _ , kv := range meta {
2015-10-02 00:19:52 +00:00
parts := strings . SplitN ( kv , "=" , 2 )
2015-10-01 23:46:42 +00:00
if len ( parts ) != 2 {
c . Ui . Error ( fmt . Sprintf ( "Error parsing Client.Meta value: %v" , kv ) )
return nil
}
cmdConfig . Client . Meta [ parts [ 0 ] ] = parts [ 1 ]
}
}
2015-08-16 01:54:41 +00:00
// Load the configuration
2019-08-30 12:15:50 +00:00
dev , err := newDevModeConfig ( devMode , devConnectMode )
if err != nil {
c . Ui . Error ( err . Error ( ) )
return nil
}
2015-08-16 01:54:41 +00:00
var config * Config
2019-08-14 19:29:37 +00:00
if dev != nil {
config = DevConfig ( dev )
2015-08-16 20:54:49 +00:00
} else {
config = DefaultConfig ( )
2015-08-16 01:54:41 +00:00
}
2018-03-22 20:53:08 +00:00
// Merge in the enterprise overlay
2020-10-14 07:35:16 +00:00
config = config . Merge ( DefaultEntConfig ( ) )
2018-03-22 20:53:08 +00:00
2015-08-16 01:54:41 +00:00
for _ , path := range configPath {
current , err := LoadConfig ( path )
if err != nil {
c . Ui . Error ( fmt . Sprintf (
"Error loading configuration from %s: %s" , path , err ) )
return nil
}
2015-12-03 07:22:18 +00:00
// The user asked us to load some config here but we didn't find any,
// so we'll complain but continue.
if current == nil || reflect . DeepEqual ( current , & Config { } ) {
c . Ui . Warn ( fmt . Sprintf ( "No configuration loaded from %s" , path ) )
}
2015-08-16 01:54:41 +00:00
if config == nil {
config = current
} else {
config = config . Merge ( current )
}
}
2015-08-16 21:34:38 +00:00
// Ensure the sub-structs at least exist
if config . Client == nil {
config . Client = & ClientConfig { }
}
2019-12-06 20:46:46 +00:00
2020-01-07 14:14:24 +00:00
if config . Server == nil {
config . Server = & ServerConfig { }
}
2015-08-16 22:10:11 +00:00
2015-09-22 03:38:43 +00:00
// Merge any CLI options over config file options
config = config . Merge ( cmdConfig )
2015-08-16 22:10:11 +00:00
// Set the version info
config . Version = c . Version
2016-11-09 19:37:41 +00:00
// Normalize binds, ports, addresses, and advertise
if err := config . normalizeAddrs ( ) ; err != nil {
c . Ui . Error ( err . Error ( ) )
2016-11-04 22:24:28 +00:00
return nil
}
2016-10-17 17:48:04 +00:00
2016-11-10 19:47:43 +00:00
// Check to see if we should read the Vault token from the environment
if config . Vault . Token == "" {
2019-04-05 17:11:41 +00:00
config . Vault . Token = os . Getenv ( "VAULT_TOKEN" )
2016-11-10 19:47:43 +00:00
}
2019-04-05 01:16:51 +00:00
// Check to see if we should read the Vault namespace from the environment
if config . Vault . Namespace == "" {
2019-04-05 17:11:41 +00:00
config . Vault . Namespace = os . Getenv ( "VAULT_NAMESPACE" )
2019-04-05 01:16:51 +00:00
}
2018-08-30 20:43:09 +00:00
// Default the plugin directory to be under that of the data directory if it
// isn't explicitly specified.
if config . PluginDir == "" && config . DataDir != "" {
config . PluginDir = filepath . Join ( config . DataDir , "plugins" )
}
2021-03-23 13:08:14 +00:00
// License configuration options
config . Server . LicenseEnv = os . Getenv ( "NOMAD_LICENSE" )
if config . Server . LicensePath == "" {
config . Server . LicensePath = os . Getenv ( "NOMAD_LICENSE_PATH" )
}
2020-05-09 16:14:02 +00:00
config . Server . DefaultSchedulerConfig . Canonicalize ( )
2022-02-08 21:52:35 +00:00
if ! c . IsValidConfig ( config , cmdConfig ) {
2019-01-08 15:07:36 +00:00
return nil
}
return config
}
2022-02-08 21:52:35 +00:00
func ( c * Command ) IsValidConfig ( config , cmdConfig * Config ) bool {
2019-01-08 22:21:48 +00:00
// Check that the server is running in at least one mode.
if ! ( config . Server . Enabled || config . Client . Enabled ) {
c . Ui . Error ( "Must specify either server, client or dev mode for the agent." )
return false
}
2020-10-05 14:52:07 +00:00
// Check that the region does not contain invalid characters
if strings . ContainsAny ( config . Region , "\000" ) {
c . Ui . Error ( "Region contains invalid characters" )
return false
}
// Check that the datacenter name does not contain invalid characters
if strings . ContainsAny ( config . Datacenter , "\000" ) {
c . Ui . Error ( "Datacenter contains invalid characters" )
return false
}
2019-01-08 15:07:36 +00:00
// Set up the TLS configuration properly if we have one.
// XXX chelseakomlo: set up a TLSConfig New method which would wrap
// constructor-type actions like this.
if config . TLSConfig != nil && ! config . TLSConfig . IsEmpty ( ) {
if err := config . TLSConfig . SetChecksum ( ) ; err != nil {
c . Ui . Error ( fmt . Sprintf ( "WARNING: Error when parsing TLS configuration: %v" , err ) )
}
}
2016-10-17 17:48:04 +00:00
if config . Server . EncryptKey != "" {
if _ , err := config . Server . EncryptBytes ( ) ; err != nil {
c . Ui . Error ( fmt . Sprintf ( "Invalid encryption key: %s" , err ) )
2019-01-08 15:07:36 +00:00
return false
2016-10-17 17:48:04 +00:00
}
keyfile := filepath . Join ( config . DataDir , serfKeyring )
if _ , err := os . Stat ( keyfile ) ; err == nil {
2016-11-09 19:37:41 +00:00
c . Ui . Warn ( "WARNING: keyring exists but -encrypt given, using keyring" )
2016-10-17 17:48:04 +00:00
}
}
2015-12-22 19:14:41 +00:00
// Verify the paths are absolute.
dirs := map [ string ] string {
2018-08-30 20:43:09 +00:00
"data-dir" : config . DataDir ,
"plugin-dir" : config . PluginDir ,
"alloc-dir" : config . Client . AllocDir ,
"state-dir" : config . Client . StateDir ,
2015-12-22 19:14:41 +00:00
}
for k , dir := range dirs {
if dir == "" {
continue
}
if ! filepath . IsAbs ( dir ) {
c . Ui . Error ( fmt . Sprintf ( "%s must be given as an absolute path: got %v" , k , dir ) )
2019-01-08 15:07:36 +00:00
return false
2015-12-22 19:14:41 +00:00
}
}
2019-01-09 18:56:40 +00:00
if config . Client . Enabled {
for k := range config . Client . Meta {
if ! helper . IsValidInterpVariable ( k ) {
c . Ui . Error ( fmt . Sprintf ( "Invalid Client.Meta key: %v" , k ) )
return false
}
}
}
2020-05-14 18:16:12 +00:00
if err := config . Server . DefaultSchedulerConfig . Validate ( ) ; err != nil {
c . Ui . Error ( err . Error ( ) )
2019-01-08 15:07:36 +00:00
return false
2015-10-02 00:27:36 +00:00
}
2021-10-13 22:50:46 +00:00
if config . Client . MinDynamicPort < 0 || config . Client . MinDynamicPort > structs . MaxValidPort {
c . Ui . Error ( fmt . Sprintf ( "Invalid dynamic port range: min_dynamic_port=%d" , config . Client . MinDynamicPort ) )
return false
}
if config . Client . MaxDynamicPort < 0 || config . Client . MaxDynamicPort > structs . MaxValidPort {
c . Ui . Error ( fmt . Sprintf ( "Invalid dynamic port range: max_dynamic_port=%d" , config . Client . MaxDynamicPort ) )
return false
}
if config . Client . MinDynamicPort > config . Client . MaxDynamicPort {
c . Ui . Error ( fmt . Sprintf ( "Invalid dynamic port range: min_dynamic_port=%d and max_dynamic_port=%d" , config . Client . MinDynamicPort , config . Client . MaxDynamicPort ) )
2021-09-10 08:52:47 +00:00
return false
}
2022-01-12 20:04:31 +00:00
if config . Client . Reserved == nil {
// Coding error; should always be set by DefaultConfig()
c . Ui . Error ( "client.reserved must be initialized. Please report a bug." )
return false
}
if ports := config . Client . Reserved . ReservedPorts ; ports != "" {
if _ , err := structs . ParsePortRanges ( ports ) ; err != nil {
c . Ui . Error ( fmt . Sprintf ( "reserved.reserved_ports %q invalid: %v" , ports , err ) )
return false
}
}
for _ , hn := range config . Client . HostNetworks {
2022-07-12 21:40:25 +00:00
// Ensure port range is valid
2022-01-12 20:04:31 +00:00
if _ , err := structs . ParsePortRanges ( hn . ReservedPorts ) ; err != nil {
c . Ui . Error ( fmt . Sprintf ( "host_network[%q].reserved_ports %q invalid: %v" ,
hn . Name , hn . ReservedPorts , err ) )
return false
}
}
2022-05-03 22:38:32 +00:00
if err := config . Client . Artifact . Validate ( ) ; err != nil {
c . Ui . Error ( fmt . Sprintf ( "client.artifact stanza invalid: %v" , err ) )
return false
}
2020-05-14 18:16:12 +00:00
if ! config . DevMode {
// Ensure that we have the directories we need to run.
if config . Server . Enabled && config . DataDir == "" {
2021-10-28 13:05:56 +00:00
c . Ui . Error ( ` Must specify "data_dir" config option or "data-dir" CLI flag ` )
2019-01-08 15:07:36 +00:00
return false
}
2020-05-14 18:16:12 +00:00
// The config is valid if the top-level data-dir is set or if both
// alloc-dir and state-dir are set.
if config . Client . Enabled && config . DataDir == "" {
if config . Client . AllocDir == "" || config . Client . StateDir == "" || config . PluginDir == "" {
c . Ui . Error ( "Must specify the state, alloc dir, and plugin dir if data-dir is omitted." )
return false
}
}
2020-04-24 14:47:43 +00:00
2020-05-14 18:16:12 +00:00
// Check the bootstrap flags
if ! config . Server . Enabled && cmdConfig . Server . BootstrapExpect > 0 {
// report an error if BootstrapExpect is set in CLI but server is disabled
c . Ui . Error ( "Bootstrap requires server mode to be enabled" )
return false
}
if config . Server . Enabled && config . Server . BootstrapExpect == 1 {
c . Ui . Error ( "WARNING: Bootstrap mode enabled! Potentially unsafe operation." )
}
2022-06-06 13:22:59 +00:00
if config . Server . Enabled && config . Server . BootstrapExpect % 2 == 0 {
c . Ui . Error ( "WARNING: Number of bootstrap servers should ideally be set to an odd number." )
}
2020-04-24 14:47:43 +00:00
}
2015-09-22 19:40:21 +00:00
2021-12-01 22:36:02 +00:00
// ProtocolVersion has never been used. Warn if it is set as someone
// has probably made a mistake.
if config . Server . ProtocolVersion != 0 {
2022-05-13 07:28:43 +00:00
c . Ui . Warn ( "Please remove deprecated protocol_version field from config." )
2021-12-01 22:36:02 +00:00
}
2019-01-08 15:07:36 +00:00
return true
2015-08-16 01:34:47 +00:00
}
2021-08-30 09:08:12 +00:00
// SetupLoggers is used to set up the logGate, and our logOutput
2020-07-17 15:05:57 +00:00
func SetupLoggers ( ui cli . Ui , config * Config ) ( * logutils . LevelFilter , * gatedwriter . Writer , io . Writer ) {
2015-08-16 20:54:49 +00:00
// Setup logging. First create the gated log writer, which will
// store logs until we're ready to show them. Then create the level
// filter, filtering logs of the specified level.
2015-08-23 23:57:54 +00:00
logGate := & gatedwriter . Writer {
2020-07-17 15:05:57 +00:00
Writer : & cli . UiWriter { Ui : ui } ,
2015-08-16 20:54:49 +00:00
}
2020-07-17 15:05:57 +00:00
logFilter := LevelFilter ( )
logFilter . MinLevel = logutils . LogLevel ( strings . ToUpper ( config . LogLevel ) )
logFilter . Writer = logGate
if ! ValidateLevelFilter ( logFilter . MinLevel , logFilter ) {
ui . Error ( fmt . Sprintf (
2015-08-16 20:54:49 +00:00
"Invalid log level: %s. Valid log levels are: %v" ,
2020-07-17 15:05:57 +00:00
logFilter . MinLevel , logFilter . Levels ) )
return nil , nil , nil
2015-08-16 20:54:49 +00:00
}
2019-10-07 12:39:26 +00:00
// Create a log writer, and wrap a logOutput around it
2020-07-17 15:05:57 +00:00
writers := [ ] io . Writer { logFilter }
2019-10-07 12:39:26 +00:00
2015-08-16 20:54:49 +00:00
// Check if syslog is enabled
if config . EnableSyslog {
2015-08-16 21:34:38 +00:00
l , err := gsyslog . NewLogger ( gsyslog . LOG_NOTICE , config . SyslogFacility , "nomad" )
2015-08-16 20:54:49 +00:00
if err != nil {
2020-07-17 15:05:57 +00:00
ui . Error ( fmt . Sprintf ( "Syslog setup failed: %v" , err ) )
return nil , nil , nil
2015-08-16 20:54:49 +00:00
}
2020-07-17 15:05:57 +00:00
writers = append ( writers , & SyslogWrapper { l , logFilter } )
2015-08-16 20:54:49 +00:00
}
2019-10-07 12:39:26 +00:00
// Check if file logging is enabled
if config . LogFile != "" {
dir , fileName := filepath . Split ( config . LogFile )
// if a path is provided, but has no filename, then a default is used.
if fileName == "" {
fileName = "nomad.log"
}
// Try to enter the user specified log rotation duration first
var logRotateDuration time . Duration
if config . LogRotateDuration != "" {
duration , err := time . ParseDuration ( config . LogRotateDuration )
if err != nil {
2020-07-17 15:05:57 +00:00
ui . Error ( fmt . Sprintf ( "Failed to parse log rotation duration: %v" , err ) )
return nil , nil , nil
2019-10-07 12:39:26 +00:00
}
logRotateDuration = duration
} else {
// Default to 24 hrs if no rotation period is specified
logRotateDuration = 24 * time . Hour
}
logFile := & logFile {
2020-07-17 15:05:57 +00:00
logFilter : logFilter ,
2019-10-07 12:39:26 +00:00
fileName : fileName ,
logPath : dir ,
duration : logRotateDuration ,
MaxBytes : config . LogRotateBytes ,
MaxFiles : config . LogRotateMaxFiles ,
}
writers = append ( writers , logFile )
2015-08-16 20:54:49 +00:00
}
2019-10-07 12:39:26 +00:00
2020-07-17 15:05:57 +00:00
logOutput := io . MultiWriter ( writers ... )
return logFilter , logGate , logOutput
2015-08-16 20:54:49 +00:00
}
2015-08-16 21:34:38 +00:00
// setupAgent is used to start the agent and various interfaces
2019-10-15 19:14:25 +00:00
func ( c * Command ) setupAgent ( config * Config , logger hclog . InterceptLogger , logOutput io . Writer , inmem * metrics . InmemSink ) error {
2015-08-16 21:34:38 +00:00
c . Ui . Output ( "Starting Nomad agent..." )
2022-04-06 17:17:05 +00:00
2019-10-09 15:17:29 +00:00
agent , err := NewAgent ( config , logger , logOutput , inmem )
2015-08-16 21:34:38 +00:00
if err != nil {
2021-10-27 17:41:17 +00:00
// log the error as well, so it appears at the end
logger . Error ( "error starting agent" , "error" , err )
2015-08-16 21:34:38 +00:00
c . Ui . Error ( fmt . Sprintf ( "Error starting agent: %s" , err ) )
return err
}
c . agent = agent
2015-09-14 22:33:08 +00:00
// Setup the HTTP server
2022-01-03 14:33:53 +00:00
httpServers , err := NewHTTPServers ( agent , config )
2015-08-24 00:50:33 +00:00
if err != nil {
agent . Shutdown ( )
c . Ui . Error ( fmt . Sprintf ( "Error starting http server: %s" , err ) )
return err
}
2022-01-03 14:33:53 +00:00
c . httpServers = httpServers
2015-08-24 00:50:33 +00:00
2018-08-10 17:08:13 +00:00
// If DisableUpdateCheck is not enabled, set up update checking
// (DisableUpdateCheck is false by default)
if config . DisableUpdateCheck != nil && ! * config . DisableUpdateCheck {
2017-08-16 22:42:15 +00:00
version := config . Version . Version
if config . Version . VersionPrerelease != "" {
version += fmt . Sprintf ( "-%s" , config . Version . VersionPrerelease )
2015-08-16 21:34:38 +00:00
}
updateParams := & checkpoint . CheckParams {
Product : "nomad" ,
Version : version ,
}
if ! config . DisableAnonymousSignature {
updateParams . SignatureFile = filepath . Join ( config . DataDir , "checkpoint-signature" )
}
// Schedule a periodic check with expected interval of 24 hours
checkpoint . CheckInterval ( updateParams , 24 * time . Hour , c . checkpointResults )
// Do an immediate check within the next 30 seconds
go func ( ) {
2022-04-09 11:22:44 +00:00
time . Sleep ( helper . RandomStagger ( 30 * time . Second ) )
2015-08-16 21:34:38 +00:00
c . checkpointResults ( checkpoint . Check ( updateParams ) )
} ( )
}
2018-03-22 20:53:08 +00:00
2015-08-16 21:34:38 +00:00
return nil
}
// checkpointResults is used to handler periodic results from our update checker
func ( c * Command ) checkpointResults ( results * checkpoint . CheckResponse , err error ) {
if err != nil {
c . Ui . Error ( fmt . Sprintf ( "Failed to check for updates: %v" , err ) )
return
}
if results . Outdated {
2017-08-16 22:42:15 +00:00
c . Ui . Error ( fmt . Sprintf ( "Newer Nomad version available: %s (currently running: %s)" , results . CurrentVersion , c . Version . VersionNumber ( ) ) )
2015-08-16 21:34:38 +00:00
}
for _ , alert := range results . Alerts {
switch alert . Level {
case "info" :
c . Ui . Info ( fmt . Sprintf ( "Bulletin [%s]: %s (%s)" , alert . Level , alert . Message , alert . URL ) )
default :
c . Ui . Error ( fmt . Sprintf ( "Bulletin [%s]: %s (%s)" , alert . Level , alert . Message , alert . URL ) )
}
}
}
2017-07-19 18:28:16 +00:00
func ( c * Command ) AutocompleteFlags ( ) complete . Flags {
2017-07-19 18:51:01 +00:00
configFilePredictor := complete . PredictOr (
complete . PredictFiles ( "*.json" ) ,
complete . PredictFiles ( "*.hcl" ) )
return map [ string ] complete . Predictor {
2018-09-28 17:09:01 +00:00
"-dev" : complete . PredictNothing ,
2019-08-30 12:15:50 +00:00
"-dev-connect" : complete . PredictNothing ,
2018-09-28 17:09:01 +00:00
"-server" : complete . PredictNothing ,
"-client" : complete . PredictNothing ,
"-bootstrap-expect" : complete . PredictAnything ,
"-encrypt" : complete . PredictAnything ,
"-raft-protocol" : complete . PredictAnything ,
"-rejoin" : complete . PredictNothing ,
"-join" : complete . PredictAnything ,
"-retry-join" : complete . PredictAnything ,
"-retry-max" : complete . PredictAnything ,
"-state-dir" : complete . PredictDirs ( "*" ) ,
"-alloc-dir" : complete . PredictDirs ( "*" ) ,
"-node-class" : complete . PredictAnything ,
"-servers" : complete . PredictAnything ,
"-meta" : complete . PredictAnything ,
"-config" : configFilePredictor ,
"-bind" : complete . PredictAnything ,
"-region" : complete . PredictAnything ,
"-data-dir" : complete . PredictDirs ( "*" ) ,
"-plugin-dir" : complete . PredictDirs ( "*" ) ,
"-dc" : complete . PredictAnything ,
"-log-level" : complete . PredictAnything ,
2019-01-11 19:36:26 +00:00
"-json-logs" : complete . PredictNothing ,
2018-09-28 17:09:01 +00:00
"-node" : complete . PredictAnything ,
"-consul-auth" : complete . PredictAnything ,
"-consul-auto-advertise" : complete . PredictNothing ,
"-consul-ca-file" : complete . PredictAnything ,
"-consul-cert-file" : complete . PredictAnything ,
"-consul-key-file" : complete . PredictAnything ,
"-consul-checks-use-advertise" : complete . PredictNothing ,
"-consul-client-auto-join" : complete . PredictNothing ,
"-consul-client-service-name" : complete . PredictAnything ,
"-consul-client-http-check-name" : complete . PredictAnything ,
"-consul-server-service-name" : complete . PredictAnything ,
"-consul-server-http-check-name" : complete . PredictAnything ,
"-consul-server-serf-check-name" : complete . PredictAnything ,
"-consul-server-rpc-check-name" : complete . PredictAnything ,
"-consul-server-auto-join" : complete . PredictNothing ,
"-consul-ssl" : complete . PredictNothing ,
"-consul-verify-ssl" : complete . PredictNothing ,
"-consul-address" : complete . PredictAnything ,
2019-11-18 21:05:06 +00:00
"-consul-token" : complete . PredictAnything ,
2018-09-28 17:09:01 +00:00
"-vault-enabled" : complete . PredictNothing ,
"-vault-allow-unauthenticated" : complete . PredictNothing ,
"-vault-token" : complete . PredictAnything ,
"-vault-address" : complete . PredictAnything ,
"-vault-create-from-role" : complete . PredictAnything ,
"-vault-ca-file" : complete . PredictAnything ,
"-vault-ca-path" : complete . PredictAnything ,
"-vault-cert-file" : complete . PredictAnything ,
"-vault-key-file" : complete . PredictAnything ,
"-vault-tls-skip-verify" : complete . PredictNothing ,
"-vault-tls-server-name" : complete . PredictAnything ,
"-acl-enabled" : complete . PredictNothing ,
"-acl-replication-token" : complete . PredictAnything ,
2017-07-19 18:51:01 +00:00
}
2017-07-19 18:28:16 +00:00
}
func ( c * Command ) AutocompleteArgs ( ) complete . Predictor {
2017-07-19 18:51:01 +00:00
return nil
2017-07-19 18:28:16 +00:00
}
2015-08-16 01:34:47 +00:00
func ( c * Command ) Run ( args [ ] string ) int {
c . Ui = & cli . PrefixedUi {
OutputPrefix : "==> " ,
InfoPrefix : " " ,
ErrorPrefix : "==> " ,
Ui : c . Ui ,
}
2015-08-16 01:54:41 +00:00
// Parse our configs
c . args = args
config := c . readConfig ( )
if config == nil {
return 1
}
2021-01-20 14:09:31 +00:00
// reset UI to prevent prefixed json output
if config . LogJson {
c . Ui = & cli . BasicUi {
Reader : os . Stdin ,
Writer : os . Stdout ,
ErrorWriter : os . Stderr ,
}
}
2015-08-16 20:54:49 +00:00
// Setup the log outputs
2020-07-17 15:05:57 +00:00
logFilter , logGate , logOutput := SetupLoggers ( c . Ui , config )
c . logFilter = logFilter
c . logOutput = logOutput
2015-08-16 20:54:49 +00:00
if logGate == nil {
return 1
}
2019-02-21 18:41:54 +00:00
// Create logger
2019-10-15 19:14:25 +00:00
logger := hclog . NewInterceptLogger ( & hclog . LoggerOptions {
2019-02-21 18:41:54 +00:00
Name : "agent" ,
Level : hclog . LevelFromString ( config . LogLevel ) ,
Output : logOutput ,
JSONFormat : config . LogJson ,
} )
2021-10-12 18:38:44 +00:00
// Wrap log messages emitted with the 'log' package.
// These usually come from external dependencies.
2022-01-13 14:56:35 +00:00
log . SetOutput ( logger . StandardWriter ( & hclog . StandardLoggerOptions {
InferLevels : true ,
InferLevelsWithTimestamp : true ,
} ) )
2021-10-12 18:38:44 +00:00
log . SetPrefix ( "" )
log . SetFlags ( 0 )
2019-02-21 18:41:54 +00:00
// Swap out UI implementation if json logging is enabled
if config . LogJson {
c . Ui = & logging . HcLogUI { Log : logger }
2022-05-19 19:40:30 +00:00
// Don't buffer json logs because they aren't reordered anyway.
logGate . Flush ( )
2019-02-21 18:41:54 +00:00
}
2015-12-05 00:52:17 +00:00
// Log config files
2015-12-15 01:46:52 +00:00
if len ( config . Files ) > 0 {
2018-03-30 18:42:11 +00:00
c . Ui . Output ( fmt . Sprintf ( "Loaded configuration from %s" , strings . Join ( config . Files , ", " ) ) )
2015-12-09 23:50:18 +00:00
} else {
2018-03-30 18:42:11 +00:00
c . Ui . Output ( "No configuration files loaded" )
2015-12-09 23:50:18 +00:00
}
2015-12-05 00:52:17 +00:00
2015-08-16 01:54:41 +00:00
// Initialize the telemetry
2017-09-04 03:50:05 +00:00
inmem , err := c . setupTelemetry ( config )
if err != nil {
2015-08-16 01:54:41 +00:00
c . Ui . Error ( fmt . Sprintf ( "Error initializing telemetry: %s" , err ) )
return 1
}
2015-08-16 21:34:38 +00:00
// Create the agent
2019-10-09 15:17:29 +00:00
if err := c . setupAgent ( config , logger , logOutput , inmem ) ; err != nil {
2017-03-14 21:48:39 +00:00
logGate . Flush ( )
2015-08-16 21:34:38 +00:00
return 1
}
2015-09-14 22:33:08 +00:00
defer func ( ) {
2020-04-13 14:32:34 +00:00
c . agent . Shutdown ( )
// Shutdown the http server at the end, to ease debugging if
// the agent takes long to shutdown
2022-01-03 14:33:53 +00:00
if len ( c . httpServers ) > 0 {
for _ , srv := range c . httpServers {
srv . Shutdown ( )
}
2015-09-14 22:33:08 +00:00
}
} ( )
2015-12-02 19:55:29 +00:00
// Join startup nodes if specified
if err := c . startupJoin ( config ) ; err != nil {
c . Ui . Error ( err . Error ( ) )
return 1
}
2015-08-16 21:34:38 +00:00
// Compile agent information for output later
info := make ( map [ string ] string )
2017-08-16 22:42:15 +00:00
info [ "version" ] = config . Version . VersionNumber ( )
2015-08-16 21:34:38 +00:00
info [ "client" ] = strconv . FormatBool ( config . Client . Enabled )
info [ "log level" ] = config . LogLevel
info [ "server" ] = strconv . FormatBool ( config . Server . Enabled )
2015-09-21 20:53:49 +00:00
info [ "region" ] = fmt . Sprintf ( "%s (DC: %s)" , config . Region , config . Datacenter )
2018-05-22 22:14:01 +00:00
info [ "bind addrs" ] = c . getBindAddrSynopsis ( )
info [ "advertise addrs" ] = c . getAdvertiseAddrSynopsis ( )
2015-08-16 21:34:38 +00:00
// Sort the keys for output
infoKeys := make ( [ ] string , 0 , len ( info ) )
for key := range info {
infoKeys = append ( infoKeys , key )
}
sort . Strings ( infoKeys )
// Agent configuration output
padding := 18
c . Ui . Output ( "Nomad agent configuration:\n" )
for _ , k := range infoKeys {
2018-03-30 23:27:18 +00:00
c . Ui . Info ( fmt . Sprintf (
2015-08-16 21:34:38 +00:00
"%s%s: %s" ,
strings . Repeat ( " " , padding - len ( k ) ) ,
strings . Title ( k ) ,
info [ k ] ) )
}
c . Ui . Output ( "" )
// Output the header that the server has started
c . Ui . Output ( "Nomad agent started! Log data will stream in below:\n" )
2015-08-16 20:54:49 +00:00
// Enable log streaming
logGate . Flush ( )
2015-12-02 19:55:29 +00:00
// Start retry join process
2018-05-30 23:51:55 +00:00
if err := c . handleRetryJoin ( config ) ; err != nil {
c . Ui . Error ( err . Error ( ) )
return 1
}
// Wait for exit
return c . handleSignals ( )
}
// handleRetryJoin is used to start retry joining if it is configured.
func ( c * Command ) handleRetryJoin ( config * Config ) error {
2015-12-09 15:57:54 +00:00
c . retryJoinErrCh = make ( chan struct { } )
2018-04-18 16:18:18 +00:00
2018-05-11 19:52:05 +00:00
if config . Server . Enabled && len ( config . Server . RetryJoin ) != 0 {
joiner := retryJoiner {
discover : & discover . Discover { } ,
errCh : c . retryJoinErrCh ,
2018-09-13 17:43:40 +00:00
logger : c . agent . logger . Named ( "joiner" ) ,
2018-05-11 19:52:05 +00:00
serverJoin : c . agent . server . Join ,
serverEnabled : true ,
}
2018-05-22 18:14:41 +00:00
if err := joiner . Validate ( config ) ; err != nil {
2018-05-30 23:51:55 +00:00
return err
2018-05-22 18:14:41 +00:00
}
2018-05-30 23:51:55 +00:00
// Remove the duplicate fields
if len ( config . Server . RetryJoin ) != 0 {
config . Server . ServerJoin . RetryJoin = config . Server . RetryJoin
config . Server . RetryJoin = nil
}
if config . Server . RetryMaxAttempts != 0 {
config . Server . ServerJoin . RetryMaxAttempts = config . Server . RetryMaxAttempts
config . Server . RetryMaxAttempts = 0
2018-05-11 19:52:05 +00:00
}
2018-05-30 23:51:55 +00:00
if config . Server . RetryInterval != 0 {
config . Server . ServerJoin . RetryInterval = config . Server . RetryInterval
config . Server . RetryInterval = 0
}
2018-09-13 17:43:40 +00:00
c . agent . logger . Warn ( "using deprecated retry_join fields. Upgrade configuration to use server_join" )
2018-05-11 19:52:05 +00:00
}
2018-05-30 23:51:55 +00:00
if config . Server . Enabled &&
config . Server . ServerJoin != nil &&
len ( config . Server . ServerJoin . RetryJoin ) != 0 {
2018-05-11 19:52:05 +00:00
joiner := retryJoiner {
discover : & discover . Discover { } ,
errCh : c . retryJoinErrCh ,
2018-09-13 17:43:40 +00:00
logger : c . agent . logger . Named ( "joiner" ) ,
2018-05-11 19:52:05 +00:00
serverJoin : c . agent . server . Join ,
serverEnabled : true ,
}
2018-05-22 20:54:05 +00:00
if err := joiner . Validate ( config ) ; err != nil {
2018-05-30 23:51:55 +00:00
return err
2018-05-22 20:54:05 +00:00
}
2018-05-11 19:52:05 +00:00
go joiner . RetryJoin ( config . Server . ServerJoin )
}
2018-05-30 23:51:55 +00:00
if config . Client . Enabled &&
config . Client . ServerJoin != nil &&
len ( config . Client . ServerJoin . RetryJoin ) != 0 {
2018-05-11 19:52:05 +00:00
joiner := retryJoiner {
discover : & discover . Discover { } ,
errCh : c . retryJoinErrCh ,
2018-09-13 17:43:40 +00:00
logger : c . agent . logger . Named ( "joiner" ) ,
2018-05-11 19:52:05 +00:00
clientJoin : c . agent . client . SetServers ,
clientEnabled : true ,
}
2018-05-22 20:54:05 +00:00
if err := joiner . Validate ( config ) ; err != nil {
2018-05-30 23:51:55 +00:00
return err
2018-05-22 20:54:05 +00:00
}
2018-05-11 19:52:05 +00:00
go joiner . RetryJoin ( config . Client . ServerJoin )
2018-04-18 16:18:18 +00:00
}
2015-12-02 19:55:29 +00:00
2018-05-30 23:51:55 +00:00
return nil
2015-08-16 20:54:49 +00:00
}
// handleSignals blocks until we get an exit-causing signal
2017-11-15 01:53:23 +00:00
func ( c * Command ) handleSignals ( ) int {
2015-08-16 20:54:49 +00:00
signalCh := make ( chan os . Signal , 4 )
2016-10-10 22:12:56 +00:00
signal . Notify ( signalCh , os . Interrupt , syscall . SIGTERM , syscall . SIGHUP , syscall . SIGPIPE )
2015-08-16 20:54:49 +00:00
// Wait for a signal
WAIT :
var sig os . Signal
select {
case s := <- signalCh :
sig = s
2019-11-11 20:16:07 +00:00
case <- winsvc . ShutdownChannel ( ) :
sig = os . Interrupt
2015-08-16 20:54:49 +00:00
case <- c . ShutdownCh :
sig = os . Interrupt
2015-12-09 15:57:54 +00:00
case <- c . retryJoinErrCh :
2015-12-02 19:55:29 +00:00
return 1
2015-08-16 20:54:49 +00:00
}
2017-11-15 16:01:41 +00:00
// Skip any SIGPIPE signal and don't try to log it (See issues #1798, #3554)
2016-10-10 22:12:56 +00:00
if sig == syscall . SIGPIPE {
goto WAIT
}
2017-11-15 16:01:41 +00:00
c . Ui . Output ( fmt . Sprintf ( "Caught signal: %v" , sig ) )
2015-08-16 20:54:49 +00:00
// Check if this is a SIGHUP
if sig == syscall . SIGHUP {
2017-11-15 01:53:23 +00:00
c . handleReload ( )
2015-08-16 20:54:49 +00:00
goto WAIT
}
// Check if we should do a graceful leave
graceful := false
2017-11-15 01:53:23 +00:00
if sig == os . Interrupt && c . agent . GetConfig ( ) . LeaveOnInt {
2015-08-16 20:54:49 +00:00
graceful = true
2017-11-15 01:53:23 +00:00
} else if sig == syscall . SIGTERM && c . agent . GetConfig ( ) . LeaveOnTerm {
2015-08-16 20:54:49 +00:00
graceful = true
}
// Bail fast if not doing a graceful leave
if ! graceful {
return 1
}
// Attempt a graceful leave
gracefulCh := make ( chan struct { } )
c . Ui . Output ( "Gracefully shutting down agent..." )
go func ( ) {
if err := c . agent . Leave ( ) ; err != nil {
c . Ui . Error ( fmt . Sprintf ( "Error: %s" , err ) )
return
}
close ( gracefulCh )
} ( )
// Wait for leave or another signal
select {
case <- signalCh :
return 1
case <- time . After ( gracefulTimeout ) :
return 1
case <- gracefulCh :
return 0
}
}
2018-01-19 10:12:14 +00:00
// reloadHTTPServer shuts down the existing HTTP server and restarts it. This
// is helpful when reloading the agent configuration.
2018-01-16 19:16:35 +00:00
func ( c * Command ) reloadHTTPServer ( ) error {
2018-09-13 17:43:40 +00:00
c . agent . logger . Info ( "reloading HTTP server with new TLS configuration" )
2017-11-30 15:50:43 +00:00
2022-01-03 14:33:53 +00:00
for _ , srv := range c . httpServers {
srv . Shutdown ( )
}
2017-11-20 15:38:46 +00:00
2022-01-03 14:33:53 +00:00
httpServers , err := NewHTTPServers ( c . agent , c . agent . config )
2017-11-20 15:38:46 +00:00
if err != nil {
return err
}
2022-01-03 14:33:53 +00:00
c . httpServers = httpServers
2017-11-20 15:38:46 +00:00
return nil
}
2015-08-16 20:54:49 +00:00
// handleReload is invoked when we should reload our configs, e.g. SIGHUP
2017-11-15 01:53:23 +00:00
func ( c * Command ) handleReload ( ) {
2015-08-16 20:54:49 +00:00
c . Ui . Output ( "Reloading configuration..." )
newConf := c . readConfig ( )
if newConf == nil {
2020-12-09 19:05:18 +00:00
c . Ui . Error ( "Failed to reload configs" )
2017-11-15 01:53:23 +00:00
return
2015-08-16 20:54:49 +00:00
}
// Change the log level
minLevel := logutils . LogLevel ( strings . ToUpper ( newConf . LogLevel ) )
if ValidateLevelFilter ( minLevel , c . logFilter ) {
c . logFilter . SetMinLevel ( minLevel )
} else {
c . Ui . Error ( fmt . Sprintf (
"Invalid log level: %s. Valid log levels are: %v" ,
minLevel , c . logFilter . Levels ) )
// Keep the current log level
2017-11-15 01:53:23 +00:00
newConf . LogLevel = c . agent . GetConfig ( ) . LogLevel
}
2018-06-08 17:14:40 +00:00
shouldReloadAgent , shouldReloadHTTP := c . agent . ShouldReload ( newConf )
2018-01-16 12:34:39 +00:00
if shouldReloadAgent {
2018-09-13 17:43:40 +00:00
c . agent . logger . Debug ( "starting reload of agent config" )
2017-11-30 15:50:43 +00:00
err := c . agent . Reload ( newConf )
2017-11-20 15:38:46 +00:00
if err != nil {
2018-09-13 17:43:40 +00:00
c . agent . logger . Error ( "failed to reload the config" , "error" , err )
2017-11-20 15:38:46 +00:00
return
}
2018-03-21 16:46:05 +00:00
}
2017-02-01 22:20:14 +00:00
2018-06-07 20:22:31 +00:00
if s := c . agent . Server ( ) ; s != nil {
2018-09-13 17:43:40 +00:00
c . agent . logger . Debug ( "starting reload of server config" )
2018-09-27 01:14:36 +00:00
sconf , err := convertServerConfig ( newConf )
2018-06-07 20:22:31 +00:00
if err != nil {
2018-09-13 17:43:40 +00:00
c . agent . logger . Error ( "failed to convert server config" , "error" , err )
2018-06-07 20:22:31 +00:00
return
2018-09-27 01:14:36 +00:00
}
// Finalize the config to get the agent objects injected in
c . agent . finalizeServerConfig ( sconf )
// Reload the config
if err := s . Reload ( sconf ) ; err != nil {
c . agent . logger . Error ( "reloading server config failed" , "error" , err )
return
2017-02-01 22:20:14 +00:00
}
2018-06-07 19:34:18 +00:00
}
2018-06-08 17:14:40 +00:00
if s := c . agent . Client ( ) ; s != nil {
2018-09-13 17:43:40 +00:00
c . agent . logger . Debug ( "starting reload of client config" )
2018-09-27 01:14:36 +00:00
clientConfig , err := convertClientConfig ( newConf )
2018-06-08 17:14:40 +00:00
if err != nil {
2018-09-27 01:14:36 +00:00
c . agent . logger . Error ( "failed to convert client config" , "error" , err )
2018-06-08 17:14:40 +00:00
return
}
2018-09-27 01:14:36 +00:00
// Finalize the config to get the agent objects injected in
if err := c . agent . finalizeClientConfig ( clientConfig ) ; err != nil {
c . agent . logger . Error ( "failed to finalize client config" , "error" , err )
return
}
2018-06-08 17:14:40 +00:00
if err := c . agent . Client ( ) . Reload ( clientConfig ) ; err != nil {
2018-09-13 17:43:40 +00:00
c . agent . logger . Error ( "reloading client config failed" , "error" , err )
2018-06-08 17:14:40 +00:00
return
2017-12-05 00:29:43 +00:00
}
}
// reload HTTP server after we have reloaded both client and server, in case
// we error in either of the above cases. For example, reloading the http
// server to a TLS connection could succeed, while reloading the server's rpc
// connections could fail.
2018-03-21 16:46:05 +00:00
if shouldReloadHTTP {
2018-01-16 19:16:35 +00:00
err := c . reloadHTTPServer ( )
2017-12-05 00:29:43 +00:00
if err != nil {
2018-09-13 17:43:40 +00:00
c . agent . httpLogger . Error ( "reloading config failed" , "error" , err )
2017-12-05 00:29:43 +00:00
return
}
}
2015-08-16 01:34:47 +00:00
}
2016-09-11 07:46:05 +00:00
// setupTelemetry is used ot setup the telemetry sub-systems
2017-09-04 03:50:05 +00:00
func ( c * Command ) setupTelemetry ( config * Config ) ( * metrics . InmemSink , error ) {
2015-08-16 01:54:41 +00:00
/ * Setup telemetry
Aggregate on 10 second intervals for 1 minute . Expose the
metrics over stderr when there is a SIGUSR1 received .
* /
inm := metrics . NewInmemSink ( 10 * time . Second , time . Minute )
metrics . DefaultInmemSignal ( inm )
var telConfig * Telemetry
if config . Telemetry == nil {
telConfig = & Telemetry { }
} else {
telConfig = config . Telemetry
}
2015-08-16 20:54:49 +00:00
metricsConf := metrics . DefaultConfig ( "nomad" )
2015-08-16 01:54:41 +00:00
metricsConf . EnableHostname = ! telConfig . DisableHostname
2017-12-04 18:42:31 +00:00
// Prefer the hostname as a label.
2020-10-13 19:56:54 +00:00
metricsConf . EnableHostnameLabel = ! telConfig . DisableHostname
2017-12-04 18:42:31 +00:00
2016-12-13 21:10:39 +00:00
if telConfig . UseNodeName {
metricsConf . HostName = config . NodeName
metricsConf . EnableHostname = true
}
2015-08-16 01:54:41 +00:00
2018-11-14 16:13:52 +00:00
allowedPrefixes , blockedPrefixes , err := telConfig . PrefixFilters ( )
if err != nil {
return inm , err
}
metricsConf . AllowedPrefixes = allowedPrefixes
metricsConf . BlockedPrefixes = blockedPrefixes
2018-11-14 20:02:49 +00:00
if telConfig . FilterDefault != nil {
metricsConf . FilterDefault = * telConfig . FilterDefault
}
2015-08-16 01:54:41 +00:00
// Configure the statsite sink
var fanout metrics . FanoutSink
if telConfig . StatsiteAddr != "" {
sink , err := metrics . NewStatsiteSink ( telConfig . StatsiteAddr )
if err != nil {
2017-09-04 03:50:05 +00:00
return inm , err
2015-08-16 01:54:41 +00:00
}
fanout = append ( fanout , sink )
}
// Configure the statsd sink
if telConfig . StatsdAddr != "" {
sink , err := metrics . NewStatsdSink ( telConfig . StatsdAddr )
if err != nil {
2017-09-04 03:50:05 +00:00
return inm , err
2015-08-16 01:54:41 +00:00
}
fanout = append ( fanout , sink )
}
2017-09-08 18:50:07 +00:00
// Configure the prometheus sink
2017-09-11 13:47:44 +00:00
if telConfig . PrometheusMetrics {
promSink , err := prometheus . NewPrometheusSink ( )
if err != nil {
2017-09-12 21:25:45 +00:00
return inm , err
2017-09-11 13:47:44 +00:00
}
2017-09-12 21:25:45 +00:00
fanout = append ( fanout , promSink )
2017-09-08 18:50:07 +00:00
}
2016-10-15 08:49:33 +00:00
// Configure the datadog sink
if telConfig . DataDogAddr != "" {
sink , err := datadog . NewDogStatsdSink ( telConfig . DataDogAddr , config . NodeName )
if err != nil {
2017-09-04 03:50:05 +00:00
return inm , err
2016-10-15 08:49:33 +00:00
}
2018-02-06 16:21:15 +00:00
sink . SetTags ( telConfig . DataDogTags )
2016-10-15 08:49:33 +00:00
fanout = append ( fanout , sink )
}
2016-07-22 16:33:10 +00:00
// Configure the Circonus sink
if telConfig . CirconusAPIToken != "" || telConfig . CirconusCheckSubmissionURL != "" {
cfg := & circonus . Config { }
cfg . Interval = telConfig . CirconusSubmissionInterval
cfg . CheckManager . API . TokenKey = telConfig . CirconusAPIToken
cfg . CheckManager . API . TokenApp = telConfig . CirconusAPIApp
cfg . CheckManager . API . URL = telConfig . CirconusAPIURL
cfg . CheckManager . Check . SubmissionURL = telConfig . CirconusCheckSubmissionURL
cfg . CheckManager . Check . ID = telConfig . CirconusCheckID
cfg . CheckManager . Check . ForceMetricActivation = telConfig . CirconusCheckForceMetricActivation
cfg . CheckManager . Check . InstanceID = telConfig . CirconusCheckInstanceID
cfg . CheckManager . Check . SearchTag = telConfig . CirconusCheckSearchTag
2016-11-09 20:12:30 +00:00
cfg . CheckManager . Check . Tags = telConfig . CirconusCheckTags
cfg . CheckManager . Check . DisplayName = telConfig . CirconusCheckDisplayName
2016-07-22 16:33:10 +00:00
cfg . CheckManager . Broker . ID = telConfig . CirconusBrokerID
cfg . CheckManager . Broker . SelectTag = telConfig . CirconusBrokerSelectTag
2017-02-24 22:57:49 +00:00
if cfg . CheckManager . Check . DisplayName == "" {
cfg . CheckManager . Check . DisplayName = "Nomad"
}
2016-07-22 16:33:10 +00:00
if cfg . CheckManager . API . TokenApp == "" {
cfg . CheckManager . API . TokenApp = "nomad"
}
if cfg . CheckManager . Check . SearchTag == "" {
cfg . CheckManager . Check . SearchTag = "service:nomad"
}
sink , err := circonus . NewCirconusSink ( cfg )
if err != nil {
2017-09-04 03:50:05 +00:00
return inm , err
2016-07-22 16:33:10 +00:00
}
sink . Start ( )
fanout = append ( fanout , sink )
}
2015-08-16 01:54:41 +00:00
// Initialize the global sink
if len ( fanout ) > 0 {
fanout = append ( fanout , inm )
metrics . NewGlobal ( metricsConf , fanout )
} else {
metricsConf . EnableHostname = false
metrics . NewGlobal ( metricsConf , inm )
}
2018-11-14 16:13:52 +00:00
2017-09-04 03:50:05 +00:00
return inm , nil
2015-08-16 01:54:41 +00:00
}
2015-12-02 19:55:29 +00:00
func ( c * Command ) startupJoin ( config * Config ) error {
2018-05-30 23:51:55 +00:00
// Nothing to do
if ! config . Server . Enabled {
2015-12-02 19:55:29 +00:00
return nil
}
2018-05-30 23:51:55 +00:00
// Validate both old and new aren't being set
old := len ( config . Server . StartJoin )
var new int
if config . Server . ServerJoin != nil {
new = len ( config . Server . ServerJoin . StartJoin )
}
if old != 0 && new != 0 {
return fmt . Errorf ( "server_join and start_join cannot both be defined; prefer setting the server_join stanza" )
}
// Nothing to do
if old + new == 0 {
return nil
}
// Combine the lists and join
joining := config . Server . StartJoin
if new != 0 {
joining = append ( joining , config . Server . ServerJoin . StartJoin ... )
}
2015-12-02 19:55:29 +00:00
c . Ui . Output ( "Joining cluster..." )
2018-05-30 23:51:55 +00:00
n , err := c . agent . server . Join ( joining )
2015-12-02 19:55:29 +00:00
if err != nil {
return err
}
2018-03-30 18:42:11 +00:00
c . Ui . Output ( fmt . Sprintf ( "Join completed. Synced with %d initial agents" , n ) )
2015-12-02 19:55:29 +00:00
return nil
}
2018-05-22 22:14:01 +00:00
// getBindAddrSynopsis returns a string that describes the addresses the agent
// is bound to.
func ( c * Command ) getBindAddrSynopsis ( ) string {
if c == nil || c . agent == nil || c . agent . config == nil || c . agent . config . normalizedAddrs == nil {
return ""
}
b := new ( strings . Builder )
fmt . Fprintf ( b , "HTTP: %s" , c . agent . config . normalizedAddrs . HTTP )
if c . agent . server != nil {
if c . agent . config . normalizedAddrs . RPC != "" {
fmt . Fprintf ( b , "; RPC: %s" , c . agent . config . normalizedAddrs . RPC )
}
if c . agent . config . normalizedAddrs . Serf != "" {
fmt . Fprintf ( b , "; Serf: %s" , c . agent . config . normalizedAddrs . Serf )
}
}
return b . String ( )
}
// getAdvertiseAddrSynopsis returns a string that describes the addresses the agent
// is advertising.
func ( c * Command ) getAdvertiseAddrSynopsis ( ) string {
if c == nil || c . agent == nil || c . agent . config == nil || c . agent . config . AdvertiseAddrs == nil {
return ""
}
b := new ( strings . Builder )
fmt . Fprintf ( b , "HTTP: %s" , c . agent . config . AdvertiseAddrs . HTTP )
if c . agent . server != nil {
if c . agent . config . AdvertiseAddrs . RPC != "" {
fmt . Fprintf ( b , "; RPC: %s" , c . agent . config . AdvertiseAddrs . RPC )
}
if c . agent . config . AdvertiseAddrs . Serf != "" {
fmt . Fprintf ( b , "; Serf: %s" , c . agent . config . AdvertiseAddrs . Serf )
}
}
return b . String ( )
}
2015-08-16 01:34:47 +00:00
func ( c * Command ) Synopsis ( ) string {
return "Runs a Nomad agent"
}
func ( c * Command ) Help ( ) string {
helpText := `
Usage : nomad agent [ options ]
Starts the Nomad agent and runs until an interrupt is received .
The agent may be a client and / or server .
2015-09-21 21:02:36 +00:00
The Nomad agent ' s configuration primarily comes from the config
files used , but a subset of the options may also be passed directly
as CLI arguments , listed below .
2015-09-22 04:37:55 +00:00
General Options ( clients and servers ) :
2015-08-16 01:34:47 +00:00
2015-09-22 04:37:55 +00:00
- bind = < addr >
The address the agent will bind to for all of its various network
services . The individual services that run bind to individual
ports on this address . Defaults to the loopback 127.0 .0 .1 .
2015-09-22 03:38:43 +00:00
2015-09-21 21:02:36 +00:00
- config = < path >
The path to either a single config file or a directory of config
files to use for configuring the Nomad agent . This option may be
specified multiple times . If multiple config files are used , the
values from each will be merged together . During merging , values
from files found later in the list are merged over values from
previously parsed files .
2015-09-22 03:38:43 +00:00
- data - dir = < path >
The data directory used to store state and other persistent data .
On client machines this is used to house allocation data such as
downloaded artifacts used by drivers . On server nodes , the data
dir is also used to store the replicated log .
2015-09-21 21:02:36 +00:00
2018-08-30 20:43:09 +00:00
- plugin - dir = < path >
The plugin directory is used to discover Nomad plugins . If not specified ,
the plugin directory defaults to be that of < data - dir > / plugins / .
2015-09-22 08:59:17 +00:00
- dc = < datacenter >
2015-09-22 04:37:55 +00:00
The name of the datacenter this Nomad agent is a member of . By
default this is set to "dc1" .
- log - level = < level >
Specify the verbosity level of Nomad ' s logs . Valid values include
DEBUG , INFO , and WARN , in decreasing order of verbosity . The
default is INFO .
2019-01-11 19:36:26 +00:00
- log - json
Output logs in a JSON format . The default is false .
2015-09-22 04:37:55 +00:00
- node = < name >
The name of the local agent . This name is used to identify the node
in the cluster . The name must be unique per region . The default is
the current hostname of the machine .
- region = < region >
Name of the region the Nomad agent will be a member of . By default
this value is set to "global" .
2015-09-21 21:02:36 +00:00
- dev
Start the agent in development mode . This enables a pre - configured
dual - role agent ( client + server ) which is useful for developing
or testing Nomad . No other configuration is required to start the
2019-08-14 19:29:37 +00:00
agent in this mode , but you may pass an optional comma - separated
list of mode configurations :
2019-08-30 12:15:50 +00:00
- dev - connect
2020-03-18 11:25:20 +00:00
Start the agent in development mode , but bind to a public network
interface rather than localhost for using Consul Connect . This
mode is supported only on Linux as root .
2015-09-22 03:38:43 +00:00
2015-09-22 19:40:21 +00:00
Server Options :
2015-09-22 03:38:43 +00:00
- server
Enable server mode for the agent . Agents in server mode are
clustered together and handle the additional responsibility of
leader election , data replication , and scheduling work onto
eligible client nodes .
2015-09-22 04:37:55 +00:00
2015-09-22 19:40:21 +00:00
- bootstrap - expect = < num >
Configures the expected number of servers nodes to wait for before
2018-03-11 17:56:50 +00:00
bootstrapping the cluster . Once < num > servers have joined each other ,
2015-09-22 19:40:21 +00:00
Nomad initiates the bootstrap process .
2016-10-17 17:48:04 +00:00
- encrypt = < key >
Provides the gossip encryption key
2015-12-02 19:55:29 +00:00
- join = < address >
Address of an agent to join at start time . Can be specified
multiple times .
2017-11-22 00:29:11 +00:00
- raft - protocol = < num >
The Raft protocol version to use . Used for enabling certain Autopilot
2019-09-24 18:29:38 +00:00
features . Defaults to 2.
2017-11-22 00:29:11 +00:00
2015-12-02 19:55:29 +00:00
- retry - join = < address >
Address of an agent to join at start time with retries enabled .
Can be specified multiple times .
- retry - max = < num >
Maximum number of join attempts . Defaults to 0 , which will retry
indefinitely .
- retry - interval = < dur >
Time to wait between join attempts .
- rejoin
Ignore a previous leave and attempts to rejoin the cluster .
2015-09-22 19:40:21 +00:00
Client Options :
- client
2015-10-01 23:46:42 +00:00
Enable client mode for the agent . Client mode enables a given node to be
evaluated for allocations . If client mode is not enabled , no work will be
scheduled to the agent .
- state - dir
The directory used to store state and other persistent data . If not
specified a subdirectory under the "-data-dir" will be used .
- alloc - dir
2018-03-11 17:41:02 +00:00
The directory used to store allocation data such as downloaded artifacts as
2015-10-01 23:46:42 +00:00
well as data produced by tasks . If not specified , a subdirectory under the
"-data-dir" will be used .
- servers
A list of known server addresses to connect to given as "host:port" and
delimited by commas .
- node - class
Mark this node as a member of a node - class . This can be used to label
2016-03-15 18:28:31 +00:00
similar node types .
2015-10-01 23:46:42 +00:00
- meta
User specified metadata to associated with the node . Each instance of - meta
parses a single KEY = VALUE pair . Repeat the meta flag for each key / value pair
to be added .
2015-09-22 19:40:21 +00:00
2015-10-03 00:32:11 +00:00
- network - interface
Forces the network fingerprinter to use the specified network interface .
- network - speed
The default speed for network interfaces in MBits if the link speed can not
be determined dynamically .
2017-10-12 20:51:39 +00:00
ACL Options :
- acl - enabled
Specifies whether the agent should enable ACLs .
- acl - replication - token
2017-10-12 21:07:52 +00:00
The replication token for servers to use when replicating from the
2018-03-11 17:42:05 +00:00
authoritative region . The token must be a valid management token from the
authoritative region .
2017-10-12 20:51:39 +00:00
2017-10-05 13:26:26 +00:00
Consul Options :
- consul - address = < addr >
Specifies the address to the local Consul agent , given in the format host : port .
Supports Unix sockets with the format : unix : ///tmp/consul/consul.sock
- consul - auth = < auth >
Specifies the HTTP Basic Authentication information to use for access to the
Consul Agent , given in the format username : password .
- consul - auto - advertise
Specifies if Nomad should advertise its services in Consul . The services
are named according to server_service_name and client_service_name . Nomad
servers and clients advertise their respective services , each tagged
appropriately with either http or rpc tag . Nomad servers also advertise a
serf tagged service .
- consul - ca - file = < path >
Specifies an optional path to the CA certificate used for Consul communication .
This defaults to the system bundle if unspecified .
- consul - cert - file = < path >
Specifies the path to the certificate used for Consul communication . If this
is set then you need to also set key_file .
- consul - checks - use - advertise
Specifies if Consul heath checks should bind to the advertise address . By
default , this is the bind address .
- consul - client - auto - join
Specifies if the Nomad clients should automatically discover servers in the
same region by searching for the Consul service name defined in the
server_service_name option .
- consul - client - service - name = < name >
Specifies the name of the service in Consul for the Nomad clients .
2018-03-20 15:03:58 +00:00
- consul - client - http - check - name = < name >
2018-03-19 18:37:56 +00:00
Specifies the HTTP health check name in Consul for the Nomad clients .
2017-10-05 13:26:26 +00:00
- consul - key - file = < path >
Specifies the path to the private key used for Consul communication . If this
is set then you need to also set cert_file .
- consul - server - service - name = < name >
Specifies the name of the service in Consul for the Nomad servers .
2018-03-20 15:03:58 +00:00
- consul - server - http - check - name = < name >
2018-03-19 18:37:56 +00:00
Specifies the HTTP health check name in Consul for the Nomad servers .
2018-03-20 15:03:58 +00:00
- consul - server - serf - check - name = < name >
2018-03-19 18:37:56 +00:00
Specifies the Serf health check name in Consul for the Nomad servers .
2018-03-20 15:03:58 +00:00
- consul - server - rpc - check - name = < name >
2018-03-19 18:37:56 +00:00
Specifies the RPC health check name in Consul for the Nomad servers .
2017-10-05 13:26:26 +00:00
- consul - server - auto - join
Specifies if the Nomad servers should automatically discover and join other
Nomad servers by searching for the Consul service name defined in the
server_service_name option . This search only happens if the server does not
have a leader .
- consul - ssl
Specifies if the transport scheme should use HTTPS to communicate with the
Consul agent .
- consul - token = < token >
Specifies the token used to provide a per - request ACL token .
- consul - verify - ssl
Specifies if SSL peer verification should be used when communicating to the
Consul API client over HTTPS .
2016-09-01 22:41:07 +00:00
Vault Options :
- vault - enabled
Whether to enable or disable Vault integration .
- vault - address = < addr >
The address to communicate with Vault . This should be provided with the http : //
or https : // prefix.
- vault - token = < token >
The Vault token used to derive tokens from Vault on behalf of clients .
2016-11-01 19:37:27 +00:00
This only needs to be set on Servers . Overrides the Vault token read from
the VAULT_TOKEN environment variable .
2016-09-01 22:41:07 +00:00
2017-01-27 22:11:34 +00:00
- vault - create - from - role = < role >
The role name to create tokens for tasks from .
2016-09-01 22:41:07 +00:00
- vault - allow - unauthenticated
2018-03-11 18:59:27 +00:00
Whether to allow jobs to be submitted that request Vault Tokens but do not
2016-09-01 22:41:07 +00:00
authentication . The flag only applies to Servers .
2016-10-25 00:23:48 +00:00
- vault - ca - file = < path >
The path to a PEM - encoded CA cert file to use to verify the Vault server SSL
certificate .
- vault - ca - path = < path >
The path to a directory of PEM - encoded CA cert files to verify the Vault server
certificate .
- vault - cert - file = < token >
The path to the certificate for Vault communication .
- vault - key - file = < addr >
The path to the private key for Vault communication .
- vault - tls - skip - verify = < token >
Enables or disables SSL certificate verification .
- vault - tls - server - name = < token >
Used to set the SNI host when connecting over TLS .
2015-08-16 01:34:47 +00:00
`
return strings . TrimSpace ( helpText )
}