diff --git a/client/consul_template.go b/client/consul_template.go new file mode 100644 index 000000000..ea70cb403 --- /dev/null +++ b/client/consul_template.go @@ -0,0 +1,446 @@ +package client + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + ctconf "github.com/hashicorp/consul-template/config" + "github.com/hashicorp/consul-template/manager" + "github.com/hashicorp/consul-template/signals" + "github.com/hashicorp/consul-template/watch" + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/client/driver/env" + "github.com/hashicorp/nomad/nomad/structs" +) + +var ( + // testRetryRate is used to speed up tests by setting consul-templates retry + // rate to something low + testRetryRate time.Duration = 0 +) + +// TaskHooks is an interface which provides hooks into the tasks life-cycle +type TaskHooks interface { + // Restart is used to restart the task + Restart() + + // Signal is used to signal the task + Signal(os.Signal) + + // UnblockStart is used to unblock the starting of the task. This should be + // called after prestart work is completed + UnblockStart() + + // Kill is used to kill the task because of the passed error. + Kill(error) +} + +// TaskTemplateManager is used to run a set of templates for a given task +type TaskTemplateManager struct { + // templates is the set of templates we are managing + templates []*structs.Template + + // lookup allows looking up the set of Nomad templates by their consul-template ID + lookup map[string][]*structs.Template + + // allRendered marks whether all the templates have been rendered + allRendered bool + + // hooks is used to signal/restart the task as templates are rendered + hook TaskHooks + + // runner is the consul-template runner + runner *manager.Runner + + // signals is a lookup map from the string representation of a signal to its + // actual signal + signals map[string]os.Signal + + // shutdownCh is used to signal and started goroutine to shutdown + shutdownCh chan struct{} + + // shutdown marks whether the manager has been shutdown + shutdown bool + shutdownLock sync.Mutex +} + +func NewTaskTemplateManager(hook TaskHooks, tmpls []*structs.Template, + allRendered bool, config *config.Config, vaultToken, taskDir string, + taskEnv *env.TaskEnvironment) (*TaskTemplateManager, error) { + + // Check pre-conditions + if hook == nil { + return nil, fmt.Errorf("Invalid task hook given") + } else if config == nil { + return nil, fmt.Errorf("Invalid config given") + } else if taskDir == "" { + return nil, fmt.Errorf("Invalid task directory given") + } else if taskEnv == nil { + return nil, fmt.Errorf("Invalid task environment given") + } + + tm := &TaskTemplateManager{ + templates: tmpls, + allRendered: allRendered, + hook: hook, + shutdownCh: make(chan struct{}), + } + + // Parse the signals that we need + for _, tmpl := range tmpls { + if tmpl.ChangeSignal == "" { + continue + } + + sig, err := signals.Parse(tmpl.ChangeSignal) + if err != nil { + return nil, fmt.Errorf("Failed to parse signal %q", tmpl.ChangeSignal) + } + + if tm.signals == nil { + tm.signals = make(map[string]os.Signal) + } + + tm.signals[tmpl.ChangeSignal] = sig + } + + // Build the consul-template runner + runner, lookup, err := templateRunner(tmpls, config, vaultToken, taskDir, taskEnv) + if err != nil { + return nil, err + } + tm.runner = runner + tm.lookup = lookup + + go tm.run() + return tm, nil +} + +// Stop is used to stop the consul-template runner +func (tm *TaskTemplateManager) Stop() { + tm.shutdownLock.Lock() + defer tm.shutdownLock.Unlock() + + if tm.shutdown { + return + } + + close(tm.shutdownCh) + tm.shutdown = true + + // Stop the consul-template runner + if tm.runner != nil { + tm.runner.Stop() + } +} + +// run is the long lived loop that handles errors and templates being rendered +func (tm *TaskTemplateManager) run() { + if tm.runner == nil { + return + } + + // Start the runner + go tm.runner.Start() + + // Track when they have all been rendered so we don't signal the task for + // any render event before hand + var allRenderedTime time.Time + + // Handle the first rendering + if !tm.allRendered { + // Wait till all the templates have been rendered + WAIT: + for { + select { + case <-tm.shutdownCh: + return + case err, ok := <-tm.runner.ErrCh: + if !ok { + continue + } + + tm.hook.Kill(err) + case <-tm.runner.TemplateRenderedCh(): + // A template has been rendered, figure out what to do + events := tm.runner.RenderEvents() + + // Not all templates have been rendered yet + if len(events) < len(tm.lookup) { + continue + } + + for _, event := range events { + // This template hasn't been rendered + if event.LastDidRender.IsZero() { + continue WAIT + } + } + + break WAIT + } + } + + allRenderedTime = time.Now() + tm.hook.UnblockStart() + } + + // If all our templates are change mode no-op, then we can exit here + if tm.allTemplatesNoop() { + return + } + + // A lookup for the last time the template was handled + numTemplates := len(tm.templates) + handledRenders := make(map[string]time.Time, numTemplates) + + for { + select { + case <-tm.shutdownCh: + return + case err, ok := <-tm.runner.ErrCh: + if !ok { + continue + } + + tm.hook.Kill(err) + case <-tm.runner.TemplateRenderedCh(): + // A template has been rendered, figure out what to do + var handling []string + signals := make(map[string]struct{}) + restart := false + var splay time.Duration + + now := time.Now() + for id, event := range tm.runner.RenderEvents() { + + // First time through + if allRenderedTime.After(event.LastDidRender) { + handledRenders[id] = now + continue + } + + // We have already handled this one + if htime := handledRenders[id]; htime.After(event.LastDidRender) { + continue + } + + // Lookup the template and determine what to do + tmpls, ok := tm.lookup[id] + if !ok { + tm.hook.Kill(fmt.Errorf("consul-template runner returned unknown template id %q", id)) + return + } + + for _, tmpl := range tmpls { + switch tmpl.ChangeMode { + case structs.TemplateChangeModeSignal: + signals[tmpl.ChangeSignal] = struct{}{} + case structs.TemplateChangeModeRestart: + restart = true + case structs.TemplateChangeModeNoop: + continue + } + + if tmpl.Splay > splay { + splay = tmpl.Splay + } + } + + handling = append(handling, id) + } + + if restart || len(signals) != 0 { + if splay != 0 { + select { + case <-time.After(time.Duration(splay)): + case <-tm.shutdownCh: + return + } + } + + // Update handle time + now = time.Now() + for _, id := range handling { + handledRenders[id] = now + } + + if restart { + tm.hook.Restart() + } else if len(signals) != 0 { + for signal := range signals { + tm.hook.Signal(tm.signals[signal]) + } + } + } + } + } +} + +// allTemplatesNoop returns whether all the managed templates have change mode noop. +func (tm *TaskTemplateManager) allTemplatesNoop() bool { + for _, tmpl := range tm.templates { + if tmpl.ChangeMode != structs.TemplateChangeModeNoop { + return false + } + } + + return true +} + +// templateRunner returns a consul-template runner for the given templates and a +// lookup by destination to the template. If no templates are given, a nil +// template runner and lookup is returned. +func templateRunner(tmpls []*structs.Template, config *config.Config, + vaultToken, taskDir string, taskEnv *env.TaskEnvironment) ( + *manager.Runner, map[string][]*structs.Template, error) { + + if len(tmpls) == 0 { + return nil, nil, nil + } + + runnerConfig, err := runnerConfig(config, vaultToken) + if err != nil { + return nil, nil, err + } + + // Parse the templates + ctmplMapping := parseTemplateConfigs(tmpls, taskDir, taskEnv) + + // Set the config + flat := make([]*ctconf.ConfigTemplate, 0, len(ctmplMapping)) + for ctmpl := range ctmplMapping { + local := ctmpl + flat = append(flat, &local) + } + runnerConfig.ConfigTemplates = flat + + runner, err := manager.NewRunner(runnerConfig, false, false) + if err != nil { + return nil, nil, err + } + + // Build the lookup + idMap := runner.ConfigTemplateMapping() + lookup := make(map[string][]*structs.Template, len(idMap)) + for id, ctmpls := range idMap { + for _, ctmpl := range ctmpls { + templates := lookup[id] + templates = append(templates, ctmplMapping[ctmpl]) + lookup[id] = templates + } + } + + return runner, lookup, nil +} + +// parseTemplateConfigs converts the tasks templates into consul-templates +func parseTemplateConfigs(tmpls []*structs.Template, taskDir string, taskEnv *env.TaskEnvironment) map[ctconf.ConfigTemplate]*structs.Template { + // Build the task environment + // TODO Should be able to inject the Nomad env vars into Consul-template for + // rendering + taskEnv.Build() + + ctmpls := make(map[ctconf.ConfigTemplate]*structs.Template, len(tmpls)) + for _, tmpl := range tmpls { + var src, dest string + if tmpl.SourcePath != "" { + src = filepath.Join(taskDir, taskEnv.ReplaceEnv(tmpl.SourcePath)) + } + if tmpl.DestPath != "" { + dest = filepath.Join(taskDir, taskEnv.ReplaceEnv(tmpl.DestPath)) + } + + ct := &ctconf.ConfigTemplate{ + Source: src, + Destination: dest, + EmbeddedTemplate: tmpl.EmbeddedTmpl, + Perms: ctconf.DefaultFilePerms, + Wait: &watch.Wait{}, + } + + ctmpls[*ct] = tmpl + } + + return ctmpls +} + +// runnerConfig returns a consul-template runner configuration, setting the +// Vault and Consul configurations based on the clients configs. +func runnerConfig(config *config.Config, vaultToken string) (*ctconf.Config, error) { + conf := &ctconf.Config{} + + set := func(keys []string) { + for _, k := range keys { + conf.Set(k) + } + } + + if testRetryRate != 0 { + conf.Retry = testRetryRate + conf.Set("retry") + } + + // Setup the Consul config + if config.ConsulConfig != nil { + conf.Consul = config.ConsulConfig.Addr + conf.Token = config.ConsulConfig.Token + set([]string{"consul", "token"}) + + if config.ConsulConfig.EnableSSL { + conf.SSL = &ctconf.SSLConfig{ + Enabled: true, + Verify: config.ConsulConfig.VerifySSL, + Cert: config.ConsulConfig.CertFile, + Key: config.ConsulConfig.KeyFile, + CaCert: config.ConsulConfig.CAFile, + } + set([]string{"ssl", "ssl.enabled", "ssl.verify", "ssl.cert", "ssl.key", "ssl.ca_cert"}) + } + + if config.ConsulConfig.Auth != "" { + parts := strings.SplitN(config.ConsulConfig.Auth, ":", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("Failed to parse Consul Auth config") + } + + conf.Auth = &ctconf.AuthConfig{ + Enabled: true, + Username: parts[0], + Password: parts[1], + } + + set([]string{"auth", "auth.username", "auth.password", "auth.enabled"}) + } + } + + // Setup the Vault config + if config.VaultConfig != nil && config.VaultConfig.Enabled { + conf.Vault = &ctconf.VaultConfig{ + Address: config.VaultConfig.Addr, + Token: vaultToken, + RenewToken: false, + } + set([]string{"vault", "vault.address", "vault.token", "vault.renew_token"}) + + if strings.HasPrefix(config.VaultConfig.Addr, "https") || config.VaultConfig.TLSCertFile != "" { + conf.Vault.SSL = &ctconf.SSLConfig{ + Enabled: true, + Verify: !config.VaultConfig.TLSSkipVerify, + Cert: config.VaultConfig.TLSCertFile, + Key: config.VaultConfig.TLSKeyFile, + CaCert: config.VaultConfig.TLSCaFile, + // TODO need to add this to consul-template: CaPath: config.VaultConfig.TLSCaPath, + } + + set([]string{"vault.ssl", "vault.ssl.enabled", "vault.ssl.verify", + "vault.ssl.cert", "vault.ssl.key", "vault.ssl.ca_cert"}) + } + } + + return conf, nil +} diff --git a/client/consul_template_test.go b/client/consul_template_test.go new file mode 100644 index 000000000..18c601f3a --- /dev/null +++ b/client/consul_template_test.go @@ -0,0 +1,665 @@ +package client + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + "time" + + ctestutil "github.com/hashicorp/consul/testutil" + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/client/driver/env" + "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" + sconfig "github.com/hashicorp/nomad/nomad/structs/config" + "github.com/hashicorp/nomad/testutil" +) + +// MockTaskHooks is a mock of the TaskHooks interface useful for testing +type MockTaskHooks struct { + Restarts int + Signals []os.Signal + Unblocked bool + KillError error +} + +func NewMockTaskHooks() *MockTaskHooks { return &MockTaskHooks{} } +func (m *MockTaskHooks) Restart() { m.Restarts++ } +func (m *MockTaskHooks) Signal(s os.Signal) { m.Signals = append(m.Signals, s) } +func (m *MockTaskHooks) UnblockStart() { m.Unblocked = true } +func (m *MockTaskHooks) Kill(e error) { m.KillError = e } + +// testHarness is used to test the TaskTemplateManager by spinning up +// Consul/Vault as needed +type testHarness struct { + manager *TaskTemplateManager + mockHooks *MockTaskHooks + templates []*structs.Template + taskEnv *env.TaskEnvironment + node *structs.Node + config *config.Config + vaultToken string + taskDir string + vault *testutil.TestVault + consul *ctestutil.TestServer +} + +// newTestHarness returns a harness starting a dev consul and vault server, +// building the appropriate config and creating a TaskTemplateManager +func newTestHarness(t *testing.T, templates []*structs.Template, allRendered, consul, vault bool) *testHarness { + harness := &testHarness{ + mockHooks: NewMockTaskHooks(), + templates: templates, + node: mock.Node(), + config: &config.Config{}, + } + + // Build the task environment + harness.taskEnv = env.NewTaskEnvironment(harness.node) + + // Make a tempdir + d, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Failed to make tmpdir: %v", err) + } + harness.taskDir = d + + if consul { + harness.consul = ctestutil.NewTestServer(t) + harness.config.ConsulConfig = &sconfig.ConsulConfig{ + Addr: harness.consul.HTTPAddr, + } + } + + if vault { + harness.vault = testutil.NewTestVault(t).Start() + harness.config.VaultConfig = harness.vault.Config + harness.vaultToken = harness.vault.RootToken + } + + manager, err := NewTaskTemplateManager(harness.mockHooks, templates, allRendered, + harness.config, harness.vaultToken, harness.taskDir, harness.taskEnv) + if err != nil { + t.Fatalf("failed to build task template manager: %v", err) + } + + harness.manager = manager + return harness +} + +// stop is used to stop any running Vault or Consul server plus the task manager +func (h *testHarness) stop() { + if h.vault != nil { + h.vault.Stop() + } + if h.consul != nil { + h.consul.Stop() + } + if h.manager != nil { + h.manager.Stop() + } + if h.taskDir != "" { + os.RemoveAll(h.taskDir) + } +} + +func TestTaskTemplateManager_Invalid(t *testing.T) { + hooks := NewMockTaskHooks() + var tmpls []*structs.Template + config := &config.Config{} + taskDir := "foo" + vaultToken := "" + taskEnv := env.NewTaskEnvironment(mock.Node()) + + _, err := NewTaskTemplateManager(nil, nil, false, nil, "", "", nil) + if err == nil { + t.Fatalf("Expected error") + } + + _, err = NewTaskTemplateManager(nil, tmpls, false, config, vaultToken, taskDir, taskEnv) + if err == nil || !strings.Contains(err.Error(), "task hook") { + t.Fatalf("Expected invalid task hook error: %v", err) + } + + _, err = NewTaskTemplateManager(hooks, tmpls, false, nil, vaultToken, taskDir, taskEnv) + if err == nil || !strings.Contains(err.Error(), "config") { + t.Fatalf("Expected invalid config error: %v", err) + } + + _, err = NewTaskTemplateManager(hooks, tmpls, false, config, vaultToken, "", taskEnv) + if err == nil || !strings.Contains(err.Error(), "task directory") { + t.Fatalf("Expected invalid task dir error: %v", err) + } + + _, err = NewTaskTemplateManager(hooks, tmpls, false, config, vaultToken, taskDir, nil) + if err == nil || !strings.Contains(err.Error(), "task environment") { + t.Fatalf("Expected invalid task environment error: %v", err) + } + + tm, err := NewTaskTemplateManager(hooks, tmpls, false, config, vaultToken, taskDir, taskEnv) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } else if tm == nil { + t.Fatalf("Bad %v", tm) + } + + // Build a template with a bad signal + tmpl := &structs.Template{ + DestPath: "foo", + EmbeddedTmpl: "hello, world", + ChangeMode: structs.TemplateChangeModeSignal, + ChangeSignal: "foobarbaz", + } + + tmpls = append(tmpls, tmpl) + tm, err = NewTaskTemplateManager(hooks, tmpls, false, config, vaultToken, taskDir, taskEnv) + if err == nil || !strings.Contains(err.Error(), "Failed to parse signal") { + t.Fatalf("Expected signal parsing error: %v", err) + } +} + +func TestTaskTemplateManager_Unblock_Static(t *testing.T) { + // Make a template that will render immediately + content := "hello, world!" + file := "my.tmpl" + template := &structs.Template{ + EmbeddedTmpl: content, + DestPath: file, + ChangeMode: structs.TemplateChangeModeNoop, + } + + harness := newTestHarness(t, []*structs.Template{template}, false, false, false) + defer harness.stop() + + // Wait a little while + time.Sleep(time.Duration(testutil.TestMultiplier()*100) * time.Millisecond) + + // Ensure we have been unblocked + if !harness.mockHooks.Unblocked { + t.Fatalf("Task unblock should have been called") + } + + // Check the file is there + path := filepath.Join(harness.taskDir, file) + raw, err := ioutil.ReadFile(path) + if err != nil { + t.Fatalf("Failed to read rendered template from %q: %v", path, err) + } + + if s := string(raw); s != content { + t.Fatalf("Unexpected template data; got %q, want %q", s, content) + } +} + +func TestTaskTemplateManager_Unblock_Consul(t *testing.T) { + // Make a template that will render based on a key in Consul + key := "foo" + content := "barbaz" + embedded := fmt.Sprintf(`{{key "%s"}}`, key) + file := "my.tmpl" + template := &structs.Template{ + EmbeddedTmpl: embedded, + DestPath: file, + ChangeMode: structs.TemplateChangeModeNoop, + } + + // Drop the retry rate + testRetryRate = 10 * time.Millisecond + + harness := newTestHarness(t, []*structs.Template{template}, false, true, false) + defer harness.stop() + + // Wait a little while + time.Sleep(time.Duration(testutil.TestMultiplier()*100) * time.Millisecond) + + // Ensure we have not been unblocked + if harness.mockHooks.Unblocked { + t.Fatalf("Task unblock should not have been called") + } + + // Write the key to Consul + harness.consul.SetKV(key, []byte(content)) + + // Wait a little while + time.Sleep(time.Duration(200*testutil.TestMultiplier()) * time.Millisecond) + + // Ensure we have been unblocked + if !harness.mockHooks.Unblocked { + t.Fatalf("Task unblock should have been called") + } + + // Check the file is there + path := filepath.Join(harness.taskDir, file) + raw, err := ioutil.ReadFile(path) + if err != nil { + t.Fatalf("Failed to read rendered template from %q: %v", path, err) + } + + if s := string(raw); s != content { + t.Fatalf("Unexpected template data; got %q, want %q", s, content) + } +} + +func TestTaskTemplateManager_Unblock_Vault(t *testing.T) { + // Make a template that will render based on a key in Vault + vaultPath := "secret/password" + key := "password" + content := "barbaz" + embedded := fmt.Sprintf(`{{with secret "%s"}}{{.Data.%s}}{{end}}`, vaultPath, key) + file := "my.tmpl" + template := &structs.Template{ + EmbeddedTmpl: embedded, + DestPath: file, + ChangeMode: structs.TemplateChangeModeNoop, + } + + // Drop the retry rate + testRetryRate = 10 * time.Millisecond + + harness := newTestHarness(t, []*structs.Template{template}, false, false, true) + defer harness.stop() + + // Wait a little while + time.Sleep(time.Duration(testutil.TestMultiplier()*100) * time.Millisecond) + + // Ensure we have not been unblocked + if harness.mockHooks.Unblocked { + t.Fatalf("Task unblock should not have been called") + } + + // Write the secret to Vault + logical := harness.vault.Client.Logical() + logical.Write(vaultPath, map[string]interface{}{key: content}) + + // Wait a little while + time.Sleep(time.Duration(200*testutil.TestMultiplier()) * time.Millisecond) + + // Ensure we have been unblocked + if !harness.mockHooks.Unblocked { + t.Fatalf("Task unblock should have been called") + } + + // Check the file is there + path := filepath.Join(harness.taskDir, file) + raw, err := ioutil.ReadFile(path) + if err != nil { + t.Fatalf("Failed to read rendered template from %q: %v", path, err) + } + + if s := string(raw); s != content { + t.Fatalf("Unexpected template data; got %q, want %q", s, content) + } +} + +func TestTaskTemplateManager_Unblock_Multi_Template(t *testing.T) { + // Make a template that will render immediately + staticContent := "hello, world!" + staticFile := "my.tmpl" + template := &structs.Template{ + EmbeddedTmpl: staticContent, + DestPath: staticFile, + ChangeMode: structs.TemplateChangeModeNoop, + } + + // Make a template that will render based on a key in Consul + consulKey := "foo" + consulContent := "barbaz" + consulEmbedded := fmt.Sprintf(`{{key "%s"}}`, consulKey) + consulFile := "consul.tmpl" + template2 := &structs.Template{ + EmbeddedTmpl: consulEmbedded, + DestPath: consulFile, + ChangeMode: structs.TemplateChangeModeNoop, + } + + // Drop the retry rate + testRetryRate = 10 * time.Millisecond + + harness := newTestHarness(t, []*structs.Template{template, template2}, false, true, false) + defer harness.stop() + + // Wait a little while + time.Sleep(time.Duration(testutil.TestMultiplier()*100) * time.Millisecond) + + // Ensure we have not been unblocked + if harness.mockHooks.Unblocked { + t.Fatalf("Task unblock should not have been called") + } + + // Check that the static file has been rendered + path := filepath.Join(harness.taskDir, staticFile) + raw, err := ioutil.ReadFile(path) + if err != nil { + t.Fatalf("Failed to read rendered template from %q: %v", path, err) + } + + if s := string(raw); s != staticContent { + t.Fatalf("Unexpected template data; got %q, want %q", s, staticContent) + } + + // Write the key to Consul + harness.consul.SetKV(consulKey, []byte(consulContent)) + + // Wait a little while + time.Sleep(time.Duration(200*testutil.TestMultiplier()) * time.Millisecond) + + // Ensure we have been unblocked + if !harness.mockHooks.Unblocked { + t.Fatalf("Task unblock should have been called") + } + + // Check the consul file is there + path = filepath.Join(harness.taskDir, consulFile) + raw, err = ioutil.ReadFile(path) + if err != nil { + t.Fatalf("Failed to read rendered template from %q: %v", path, err) + } + + if s := string(raw); s != consulContent { + t.Fatalf("Unexpected template data; got %q, want %q", s, consulContent) + } +} + +func TestTaskTemplateManager_Rerender_Noop(t *testing.T) { + // Make a template that will render based on a key in Consul + key := "foo" + content1 := "bar" + content2 := "baz" + embedded := fmt.Sprintf(`{{key "%s"}}`, key) + file := "my.tmpl" + template := &structs.Template{ + EmbeddedTmpl: embedded, + DestPath: file, + ChangeMode: structs.TemplateChangeModeNoop, + } + + // Drop the retry rate + testRetryRate = 10 * time.Millisecond + + harness := newTestHarness(t, []*structs.Template{template}, false, true, false) + defer harness.stop() + + // Wait a little while + time.Sleep(time.Duration(testutil.TestMultiplier()*100) * time.Millisecond) + + // Ensure we have not been unblocked + if harness.mockHooks.Unblocked { + t.Fatalf("Task unblock should not have been called") + } + + // Write the key to Consul + harness.consul.SetKV(key, []byte(content1)) + + // Wait a little while + time.Sleep(time.Duration(200*testutil.TestMultiplier()) * time.Millisecond) + + // Ensure we have been unblocked + if !harness.mockHooks.Unblocked { + t.Fatalf("Task unblock should have been called") + } + + // Check the file is there + path := filepath.Join(harness.taskDir, file) + raw, err := ioutil.ReadFile(path) + if err != nil { + t.Fatalf("Failed to read rendered template from %q: %v", path, err) + } + + if s := string(raw); s != content1 { + t.Fatalf("Unexpected template data; got %q, want %q", s, content1) + } + + // Update the key in Consul + harness.consul.SetKV(key, []byte(content2)) + + // Wait a little while + time.Sleep(time.Duration(200*testutil.TestMultiplier()) * time.Millisecond) + + // Ensure we haven't been signaled/restarted + if harness.mockHooks.Restarts != 0 || len(harness.mockHooks.Signals) != 0 { + t.Fatalf("Noop ignored: %+v", harness.mockHooks) + } + + // Check the file has been updated + path = filepath.Join(harness.taskDir, file) + raw, err = ioutil.ReadFile(path) + if err != nil { + t.Fatalf("Failed to read rendered template from %q: %v", path, err) + } + + if s := string(raw); s != content2 { + t.Fatalf("Unexpected template data; got %q, want %q", s, content2) + } +} + +func TestTaskTemplateManager_Rerender_Signal(t *testing.T) { + // Make a template that renders based on a key in Consul and sends SIGALRM + key1 := "foo" + content1_1 := "bar" + content1_2 := "baz" + embedded1 := fmt.Sprintf(`{{key "%s"}}`, key1) + file1 := "my.tmpl" + template := &structs.Template{ + EmbeddedTmpl: embedded1, + DestPath: file1, + ChangeMode: structs.TemplateChangeModeSignal, + ChangeSignal: "SIGALRM", + } + + // Make a template that renders based on a key in Consul and sends SIGBUS + key2 := "bam" + content2_1 := "cat" + content2_2 := "dog" + embedded2 := fmt.Sprintf(`{{key "%s"}}`, key2) + file2 := "my-second.tmpl" + template2 := &structs.Template{ + EmbeddedTmpl: embedded2, + DestPath: file2, + ChangeMode: structs.TemplateChangeModeSignal, + ChangeSignal: "SIGBUS", + } + + // Drop the retry rate + testRetryRate = 10 * time.Millisecond + + harness := newTestHarness(t, []*structs.Template{template, template2}, false, true, false) + defer harness.stop() + + // Wait a little while + time.Sleep(time.Duration(testutil.TestMultiplier()*100) * time.Millisecond) + + // Ensure we have not been unblocked + if harness.mockHooks.Unblocked { + t.Fatalf("Task unblock should not have been called") + } + + // Write the key to Consul + harness.consul.SetKV(key1, []byte(content1_1)) + harness.consul.SetKV(key2, []byte(content2_1)) + + // Wait a little while + time.Sleep(time.Duration(200*testutil.TestMultiplier()) * time.Millisecond) + + // Ensure we have been unblocked + if !harness.mockHooks.Unblocked { + t.Fatalf("Task unblock should have been called") + } + + // Update the keys in Consul + harness.consul.SetKV(key1, []byte(content1_2)) + harness.consul.SetKV(key2, []byte(content2_2)) + + // Wait a little while + time.Sleep(time.Duration(200*testutil.TestMultiplier()) * time.Millisecond) + + // Ensure we have been signaled and notrestarted + if harness.mockHooks.Restarts != 0 { + t.Fatalf("Should not have been restarted: %+v", harness.mockHooks) + } + + if len(harness.mockHooks.Signals) != 2 { + t.Fatalf("Should have received two signals: %+v", harness.mockHooks) + } + + // Check the files have been updated + path := filepath.Join(harness.taskDir, file1) + raw, err := ioutil.ReadFile(path) + if err != nil { + t.Fatalf("Failed to read rendered template from %q: %v", path, err) + } + + if s := string(raw); s != content1_2 { + t.Fatalf("Unexpected template data; got %q, want %q", s, content1_2) + } + + path = filepath.Join(harness.taskDir, file2) + raw, err = ioutil.ReadFile(path) + if err != nil { + t.Fatalf("Failed to read rendered template from %q: %v", path, err) + } + + if s := string(raw); s != content2_2 { + t.Fatalf("Unexpected template data; got %q, want %q", s, content2_2) + } +} + +func TestTaskTemplateManager_Rerender_Restart(t *testing.T) { + // Make a template that renders based on a key in Consul and sends restart + key1 := "bam" + content1_1 := "cat" + content1_2 := "dog" + embedded1 := fmt.Sprintf(`{{key "%s"}}`, key1) + file1 := "my.tmpl" + template := &structs.Template{ + EmbeddedTmpl: embedded1, + DestPath: file1, + ChangeMode: structs.TemplateChangeModeRestart, + } + + // Drop the retry rate + testRetryRate = 10 * time.Millisecond + + harness := newTestHarness(t, []*structs.Template{template}, false, true, false) + defer harness.stop() + + // Wait a little while + time.Sleep(time.Duration(testutil.TestMultiplier()*100) * time.Millisecond) + + // Ensure we have not been unblocked + if harness.mockHooks.Unblocked { + t.Fatalf("Task unblock should not have been called") + } + + // Write the key to Consul + harness.consul.SetKV(key1, []byte(content1_1)) + + // Wait a little while + time.Sleep(time.Duration(200*testutil.TestMultiplier()) * time.Millisecond) + + // Ensure we have been unblocked + if !harness.mockHooks.Unblocked { + t.Fatalf("Task unblock should have been called") + } + + // Update the keys in Consul + harness.consul.SetKV(key1, []byte(content1_2)) + + // Wait a little while + time.Sleep(time.Duration(200*testutil.TestMultiplier()) * time.Millisecond) + + if harness.mockHooks.Restarts != 1 { + t.Fatalf("Should have received a restart: %+v", harness.mockHooks) + } + + // Check the files have been updated + path := filepath.Join(harness.taskDir, file1) + raw, err := ioutil.ReadFile(path) + if err != nil { + t.Fatalf("Failed to read rendered template from %q: %v", path, err) + } + + if s := string(raw); s != content1_2 { + t.Fatalf("Unexpected template data; got %q, want %q", s, content1_2) + } +} + +func TestTaskTemplateManager_Interpolate_Destination(t *testing.T) { + // Make a template that will have its destination interpolated + content := "hello, world!" + file := "${node.unique.id}.tmpl" + template := &structs.Template{ + EmbeddedTmpl: content, + DestPath: file, + ChangeMode: structs.TemplateChangeModeNoop, + } + + harness := newTestHarness(t, []*structs.Template{template}, false, false, false) + defer harness.stop() + + // Wait a little while + time.Sleep(time.Duration(testutil.TestMultiplier()*100) * time.Millisecond) + + // Ensure we have been unblocked + if !harness.mockHooks.Unblocked { + t.Fatalf("Task unblock should have been called") + } + + // Check the file is there + actual := fmt.Sprintf("%s.tmpl", harness.node.ID) + path := filepath.Join(harness.taskDir, actual) + raw, err := ioutil.ReadFile(path) + if err != nil { + t.Fatalf("Failed to read rendered template from %q: %v", path, err) + } + + if s := string(raw); s != content { + t.Fatalf("Unexpected template data; got %q, want %q", s, content) + } +} + +func TestTaskTemplateManager_AllRendered_Signal(t *testing.T) { + // Make a template that renders based on a key in Consul and sends SIGALRM + key1 := "foo" + content1_1 := "bar" + embedded1 := fmt.Sprintf(`{{key "%s"}}`, key1) + file1 := "my.tmpl" + template := &structs.Template{ + EmbeddedTmpl: embedded1, + DestPath: file1, + ChangeMode: structs.TemplateChangeModeSignal, + ChangeSignal: "SIGALRM", + } + + // Drop the retry rate + testRetryRate = 10 * time.Millisecond + + harness := newTestHarness(t, []*structs.Template{template}, true, true, false) + defer harness.stop() + + // Wait a little while + time.Sleep(time.Duration(testutil.TestMultiplier()*100) * time.Millisecond) + + // Write the key to Consul + harness.consul.SetKV(key1, []byte(content1_1)) + + // Wait a little while + time.Sleep(time.Duration(200*testutil.TestMultiplier()) * time.Millisecond) + + if len(harness.mockHooks.Signals) != 1 { + t.Fatalf("Should have received two signals: %+v", harness.mockHooks) + } + + // Check the files have been updated + path := filepath.Join(harness.taskDir, file1) + raw, err := ioutil.ReadFile(path) + if err != nil { + t.Fatalf("Failed to read rendered template from %q: %v", path, err) + } + + if s := string(raw); s != content1_1 { + t.Fatalf("Unexpected template data; got %q, want %q", s, content1_1) + } +} diff --git a/jobspec/parse.go b/jobspec/parse.go index fd5a19264..964870603 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -810,7 +810,7 @@ func parseTemplates(result *[]*structs.Template, list *ast.ObjectList) error { "destination", "data", "change_mode", - "restart_signal", + "change_signal", "splay", "once", } diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 22ed87667..b2ade94f3 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -164,20 +164,18 @@ func TestParse(t *testing.T) { }, Templates: []*structs.Template{ { - SourcePath: "foo", - DestPath: "foo", - ChangeMode: "foo", - RestartSignal: "foo", - Splay: 10 * time.Second, - Once: true, + SourcePath: "foo", + DestPath: "foo", + ChangeMode: "foo", + ChangeSignal: "foo", + Splay: 10 * time.Second, }, { - SourcePath: "bar", - DestPath: "bar", - ChangeMode: structs.TemplateChangeModeRestart, - RestartSignal: "", - Splay: 5 * time.Second, - Once: false, + SourcePath: "bar", + DestPath: "bar", + ChangeMode: structs.TemplateChangeModeRestart, + ChangeSignal: "", + Splay: 5 * time.Second, }, }, }, diff --git a/jobspec/test-fixtures/basic.hcl b/jobspec/test-fixtures/basic.hcl index 8591b8ba0..1250f9a45 100644 --- a/jobspec/test-fixtures/basic.hcl +++ b/jobspec/test-fixtures/basic.hcl @@ -138,9 +138,8 @@ job "binstore-storagelocker" { source = "foo" destination = "foo" change_mode = "foo" - restart_signal = "foo" + change_signal = "foo" splay = "10s" - once = true } template { diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go index ce83f7094..57ee3ab1c 100644 --- a/nomad/structs/diff_test.go +++ b/nomad/structs/diff_test.go @@ -3208,44 +3208,40 @@ func TestTaskDiff(t *testing.T) { Old: &Task{ Templates: []*Template{ { - SourcePath: "foo", - DestPath: "bar", - EmbededTmpl: "baz", - ChangeMode: "bam", - RestartSignal: "SIGHUP", - Splay: 1, - Once: true, + SourcePath: "foo", + DestPath: "bar", + EmbeddedTmpl: "baz", + ChangeMode: "bam", + ChangeSignal: "SIGHUP", + Splay: 1, }, { - SourcePath: "foo2", - DestPath: "bar2", - EmbededTmpl: "baz2", - ChangeMode: "bam2", - RestartSignal: "SIGHUP2", - Splay: 2, - Once: false, + SourcePath: "foo2", + DestPath: "bar2", + EmbeddedTmpl: "baz2", + ChangeMode: "bam2", + ChangeSignal: "SIGHUP2", + Splay: 2, }, }, }, New: &Task{ Templates: []*Template{ { - SourcePath: "foo", - DestPath: "bar", - EmbededTmpl: "baz", - ChangeMode: "bam", - RestartSignal: "SIGHUP", - Splay: 1, - Once: true, + SourcePath: "foo", + DestPath: "bar", + EmbeddedTmpl: "baz", + ChangeMode: "bam", + ChangeSignal: "SIGHUP", + Splay: 1, }, { - SourcePath: "foo3", - DestPath: "bar3", - EmbededTmpl: "baz3", - ChangeMode: "bam3", - RestartSignal: "SIGHUP3", - Splay: 3, - Once: true, + SourcePath: "foo3", + DestPath: "bar3", + EmbeddedTmpl: "baz3", + ChangeMode: "bam3", + ChangeSignal: "SIGHUP3", + Splay: 3, }, }, }, diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index fa40f84a9..a3c98cd09 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -2222,24 +2222,21 @@ type Template struct { // DestPath is the path to where the template should be rendered DestPath string `mapstructure:"destination"` - // EmbededTmpl store the raw template. This is useful for smaller templates + // EmbeddedTmpl store the raw template. This is useful for smaller templates // where they are embeded in the job file rather than sent as an artificat - EmbededTmpl string `mapstructure:"data"` + EmbeddedTmpl string `mapstructure:"data"` // ChangeMode indicates what should be done if the template is re-rendered ChangeMode string `mapstructure:"change_mode"` - // RestartSignal is the signal that should be sent if the change mode + // ChangeSignal is the signal that should be sent if the change mode // requires it. - RestartSignal string `mapstructure:"restart_signal"` + ChangeSignal string `mapstructure:"change_signal"` // Splay is used to avoid coordinated restarts of processes by applying a // random wait between 0 and the given splay value before signalling the // application of a change Splay time.Duration `mapstructure:"splay"` - - // Once mode is used to indicate that template should be rendered only once - Once bool `mapstructure:"once"` } // DefaultTemplate returns a default template. @@ -2247,7 +2244,6 @@ func DefaultTemplate() *Template { return &Template{ ChangeMode: TemplateChangeModeRestart, Splay: 5 * time.Second, - Once: false, } } @@ -2264,7 +2260,7 @@ func (t *Template) Validate() error { var mErr multierror.Error // Verify we have something to render - if t.SourcePath == "" && t.EmbededTmpl == "" { + if t.SourcePath == "" && t.EmbeddedTmpl == "" { multierror.Append(&mErr, fmt.Errorf("Must specify a source path or have an embeded template")) } @@ -2285,7 +2281,7 @@ func (t *Template) Validate() error { switch t.ChangeMode { case TemplateChangeModeNoop, TemplateChangeModeRestart: case TemplateChangeModeSignal: - if t.RestartSignal == "" { + if t.ChangeSignal == "" { multierror.Append(&mErr, fmt.Errorf("Must specify signal value when change mode is signal")) } default: