diff --git a/vendor/github.com/hashicorp/consul-template/CHANGELOG.md b/vendor/github.com/hashicorp/consul-template/CHANGELOG.md index e04dcccdb..a64a51225 100644 --- a/vendor/github.com/hashicorp/consul-template/CHANGELOG.md +++ b/vendor/github.com/hashicorp/consul-template/CHANGELOG.md @@ -1,3 +1,49 @@ +## v0.22.1 (Nov 08, 2019) + +SECURITY: + +* curl is vulnerable in the latest alpine docker image [[GH-1302](https://github.com/hashicorp/consul-template/issues/1302)] + +## v0.22.0 (September 10, 2019) + +IMPROVEMENTS: + +* Add rate limiting to consul api calls [[GH-1279](https://github.com/hashicorp/consul-template/pull/1279)] +* Add `byMeta` function [[GH-1237](https://github.com/hashicorp/consul-template/pull/1237)] +* Add support for : and = in service tag values [[GH-1149](https://github.com/hashicorp/consul-template/pull/1149), [GH-1049](https://github.com/hashicorp/consul-template/issues/1049)] +* Add `explodeMap` function [[GH-1148](https://github.com/hashicorp/consul-template/pull/1148)] +* Don't wait for splay when stopping child runner [[GH-1141](https://github.com/hashicorp/consul-template/pull/1141)] +* Add `safels` and `safetree` functions [[GH-1132](https://github.com/hashicorp/consul-template/pull/1132)] +* Support Vault certificates with no lease [[GH-1106](https://github.com/hashicorp/consul-template/pull/1106)] +* Add wrapper function for go-sockaddr templating [[GH-1087](https://github.com/hashicorp/consul-template/pull/1087)] +* Build binaries for arm64 platform [[GH-1251](https://github.com/hashicorp/consul-template/pull/1251)] + +BUG FIXES: + +* Fix arm/arm64 builds by enabling CGO and restricting builds to Linux [workaround for [go/issues/32912](https://github.com/golang/go/issues/32912)] + +## v0.21.3 (September 05, 2019) + +BUG FIXES: + +* Fix regression in non-renewable sleep [[GH-1277](https://github.com/hashicorp/consul-template/pull/1277), [GH-1272](https://github.com/hashicorp/consul-template/issues/1272), [GH-1276](https://github.com/hashicorp/consul-template/issues/1276)] + +## v0.21.2 (August 31, 2019) + +BUG FIXES: + +* Fix regression in backup [[GH-1271](https://github.com/hashicorp/consul-template/pull/1271), [GH-1270](https://github.com/hashicorp/consul-template/issues/1270)] + +## v0.21.1 (August 30, 2019) + +BUG FIXES: + +* Fixed issue in Vault call retry logic [[GH-1269](https://github.com/hashicorp/consul-template/pull/1269), [GH-1224](https://github.com/hashicorp/consul-template/issues/1224)] +* Fixed race in backup [[GH-1265](https://github.com/hashicorp/consul-template/pull/1265), [GH-1264](https://github.com/hashicorp/consul-template/issues/1264)] +* Fixed issue when reading deleted secret [[GH-1260](https://github.com/hashicorp/consul-template/pull/1260), [GH-1198](https://github.com/hashicorp/consul-template/issues/1198)] +* Fix issue with Vault writes [[GH-1257](https://github.com/hashicorp/consul-template/pull/1257), [GH-1252](https://github.com/hashicorp/consul-template/issues/1252)] +* Fix loop to work with template set integers [[GH-1255](https://github.com/hashicorp/consul-template/pull/1255), [GH-1143](https://github.com/hashicorp/consul-template/issues/1143)] + ## v0.21.0 (August 05, 2019) IMPROVEMENTS: @@ -13,7 +59,7 @@ BUG FIXES: * Fixed issue with templates not rendering with `-once` [[GH-1227](https://github.com/hashicorp/consul-template/pull/1227), [GH-1196](https://github.com/hashicorp/consul-template/issues/1196), [GH-1207](https://github.com/hashicorp/consul-template/issues/1207)] * Fixed regression with ~/.vault-token and with vault_agent_token_file not respecting renew_token [[GH-1228](https://github.com/hashicorp/consul-template/issues/1228), [GH-1189](https://github.com/hashicorp/consul-template/issues/1189)] * CA certificates missing from docker 'light' image [[GH-1200](https://github.com/hashicorp/consul-template/issues/1200)] -* Fixed issue with dedup data garbage in Consul KV [[GH-1158](https://github.com/hashicorp/consul-template/issues/1158), [[GH-1168](https://github.com/hashicorp/consul-template/issues/1168)] +* Fixed issue with dedup data garbage in Consul KV [[GH-1158](https://github.com/hashicorp/consul-template/issues/1158), [GH-1168](https://github.com/hashicorp/consul-template/issues/1168)] * Fixed bad case in import path [[GH-1139](https://github.com/hashicorp/consul-template/issues/1139)] * Documented limits on using "." in service names [[GH-1205](https://github.com/hashicorp/consul-template/issues/1205)] diff --git a/vendor/github.com/hashicorp/consul-template/Makefile b/vendor/github.com/hashicorp/consul-template/Makefile index 0d5e40b50..7d97be30e 100644 --- a/vendor/github.com/hashicorp/consul-template/Makefile +++ b/vendor/github.com/hashicorp/consul-template/Makefile @@ -14,11 +14,11 @@ GOTAGS ?= GOMAXPROCS ?= 4 # Get the project metadata -GO_DOCKER_VERSION ?= 1.12 +GO_DOCKER_VERSION ?= 1.13 PROJECT := $(shell go list -m -mod=vendor) OWNER := "hashicorp" NAME := $(notdir $(PROJECT)) -GIT_COMMIT ?= $(shell git rev-parse --short HEAD) +GIT_COMMIT ?= $(shell git rev-parse --short HEAD || echo release) VERSION := $(shell awk -F\" '/Version/ { print $$2; exit }' "${CURRENT_DIR}/version/version.go") # Current system information @@ -27,8 +27,9 @@ GOARCH ?= $(shell go env GOARCH) # Default os-arch combination to build XC_OS ?= darwin freebsd linux netbsd openbsd solaris windows -XC_ARCH ?= 386 amd64 arm -XC_EXCLUDE ?= darwin/arm solaris/386 solaris/arm windows/arm +XC_ARCH ?= 386 amd64 arm arm64 +# XC_EXCLUDE "arm64" entries excludes both arm and arm64 +XC_EXCLUDE ?= darwin/arm64 freebsd/arm64 netbsd/arm64 openbsd/arm64 solaris/386 solaris/arm64 windows/arm64 # GPG Signing key (blank by default, means no GPG signing) GPG_KEY ?= @@ -52,8 +53,15 @@ define make-xc-target @printf "%s%20s %s\n" "-->" "${1}/${2}:" "${PROJECT} (excluded)" else @printf "%s%20s %s\n" "-->" "${1}/${2}:" "${PROJECT}" + case "$2" in \ + arm) export CGO_ENABLED="1" ; \ + export GOARM=5 \ + export CC="arm-linux-gnueabi-gcc" ;; \ + arm64) export CGO_ENABLED="1" ; \ + export CC="aarch64-linux-gnu-gcc" ;; \ + *) export CGO_ENABLED="0" ;; \ + esac ; \ env \ - CGO_ENABLED="0" \ GOOS="${1}" \ GOARCH="${2}" \ go build \ @@ -73,7 +81,16 @@ endef $(foreach goarch,$(XC_ARCH),$(foreach goos,$(XC_OS),$(eval $(call make-xc-target,$(goos),$(goarch),$(if $(findstring windows,$(goos)),.exe,))))) # Use docker to create pristine builds for release +# First build image w/ arm build requirements, then build all binaries pristine: + @docker build \ + --rm \ + --force-rm \ + --no-cache \ + --compress \ + --file="docker/pristine/Dockerfile" \ + --build-arg="GOVERSION=${GO_DOCKER_VERSION}" \ + --tag="pristine-builder" . @docker run \ --interactive \ --user $$(id -u):$$(id -g) \ @@ -82,9 +99,9 @@ pristine: --volume="${CURRENT_DIR}:/go/src/${PROJECT}" \ --volume="${GOPATH}/pkg/mod:/go/pkg/mod" \ --workdir="/go/src/${PROJECT}" \ - --env=CGO_ENABLED="0" \ --env=GO111MODULE=on \ - "golang:${GO_DOCKER_VERSION}" env GOCACHE=/tmp make -j4 build + "pristine-builder" \ + env GOCACHE=/tmp make -j4 build # dev builds and installs the project locally. dev: @@ -229,3 +246,8 @@ _sign: @echo "" @echo "And then upload the binaries in dist/!" .PHONY: _sign + +# Add/Update the "Table Of Contents" in the README.md +toc: + @./scripts/readme-toc.sh +.PHONY: toc diff --git a/vendor/github.com/hashicorp/consul-template/README.md b/vendor/github.com/hashicorp/consul-template/README.md index fb8f34dab..0fcd8e816 100644 --- a/vendor/github.com/hashicorp/consul-template/README.md +++ b/vendor/github.com/hashicorp/consul-template/README.md @@ -20,6 +20,103 @@ this functionality might prove useful. --- +## Table of Contents + +- [Community Support](#community-support) +- [Installation](#installation) +- [Quick Example](#quick-example) +- [Usage](#usage) + - [Command Line Flags](#command-line-flags) + - [Configuration File Format](#configuration-file-format) + - [Templating Language](#templating-language) + - [API Functions](#api-functions) + - [datacenters](#datacenters) + - [file](#file) + - [key](#key) + - [keyExists](#keyexists) + - [keyOrDefault](#keyordefault) + - [ls](#ls) + - [safeLs](#safels) + - [node](#node) + - [nodes](#nodes) + - [secret](#secret) + - [secrets](#secrets) + - [service](#service) + - [services](#services) + - [tree](#tree) + - [safeTree](#safetree) + - [Scratch](#scratch) + - [scratch.Key](#scratchkey) + - [scratch.Get](#scratchget) + - [scratch.Set](#scratchset) + - [scratch.SetX](#scratchsetx) + - [scratch.MapSet](#scratchmapset) + - [scratch.MapSetX](#scratchmapsetx) + - [scratch.MapValues](#scratchmapvalues) + - [Helper Functions](#helper-functions) + - [base64Decode](#base64decode) + - [base64Encode](#base64encode) + - [base64URLDecode](#base64urldecode) + - [base64URLEncode](#base64urlencode) + - [byKey](#bykey) + - [byTag](#bytag) + - [byMeta](#bymeta) + - [contains](#contains) + - [containsAll](#containsall) + - [containsAny](#containsany) + - [containsNone](#containsnone) + - [containsNotAll](#containsnotall) + - [env](#env) + - [executeTemplate](#executetemplate) + - [explode](#explode) + - [explodeMap](#explodemap) + - [indent](#indent) + - [in](#in) + - [loop](#loop) + - [join](#join) + - [trimSpace](#trimspace) + - [parseBool](#parsebool) + - [parseFloat](#parsefloat) + - [parseInt](#parseint) + - [parseJSON](#parsejson) + - [parseUint](#parseuint) + - [plugin](#plugin) + - [regexMatch](#regexmatch) + - [regexReplaceAll](#regexreplaceall) + - [replaceAll](#replaceall) + - [split](#split) + - [timestamp](#timestamp) + - [toJSON](#tojson) + - [toJSONPretty](#tojsonpretty) + - [toLower](#tolower) + - [toTitle](#totitle) + - [toTOML](#totoml) + - [toUpper](#toupper) + - [toYAML](#toyaml) + - [sockaddr](#sockaddr) + - [Math Functions](#math-functions) + - [add](#add) + - [subtract](#subtract) + - [multiply](#multiply) + - [divide](#divide) + - [modulo](#modulo) +- [Plugins](#plugins) + - [Authoring Plugins](#authoring-plugins) + - [Important Notes](#important-notes) +- [Caveats](#caveats) + - [Dots in Service Names](#dots-in-service-names) + - [Once Mode](#once-mode) + - [Exec Mode](#exec-mode) + - [De-Duplication Mode](#de-duplication-mode) + - [Termination on Error](#termination-on-error) + - [Command Environment](#command-environment) + - [Multi-phase Execution](#multi-phase-execution) +- [Running and Process Lifecycle](#running-and-process-lifecycle) +- [Debugging](#debugging) +- [FAQ](#faq) +- [Contributing](#contributing) + + ## Community Support If you have questions about how consul-template works, its capabilities or @@ -272,19 +369,6 @@ vault { # of the address is required. address = "https://vault.service.consul:8200" - # This is the grace period between lease renewal of periodic secrets and secret - # re-acquisition. When renewing a secret, if the remaining lease is less than or - # equal to the configured grace, Consul Template will request a new credential. - # This prevents Vault from revoking the credential at expiration and Consul - # Template having a stale credential. - # - # Note: If you set this to a value that is higher than your default TTL or - # max TTL (as set in vault), Consul Template will always read a new secret! - # - # This should also be less than or around 1/3 of your TTL for a predictable - # behaviour. See https://github.com/hashicorp/vault/issues/3414 - grace = "5m" - # This is a Vault Enterprise namespace to use for reading/writing secrets. # # This value can also be specified via the environment variable VAULT_NAMESPACE. @@ -719,6 +803,23 @@ maxconns:15 minconns:5 ``` +##### `safeLs` + +Same as [ls](#ls), but refuse to render template, if the KV prefix query return blank/empty data. + +This is especially useful, for rendering mission critical files, that are being populated by consul-template. + +For example: + +```text +/root/.ssh/authorized_keys +/etc/sysconfig/iptables +``` + +Using `safeLs` on empty prefixes will result in template output not being rendered at all. + +To learn how `safeLs` was born see [CT-1131](https://github.com/hashicorp/consul-template/issues/1131) [C-3975](https://github.com/hashicorp/consul/issues/3975) and [CR-82](https://github.com/hashicorp/consul-replicate/issues/82). + ##### `node` Query [Consul][consul] for a node in the catalog. @@ -859,18 +960,16 @@ secret in plain-text on disk. If an attacker is able to get access to the file, they will have access to plain-text secrets. Please note that Vault does not support blocking queries. As a result, Consul -Template will not immediately reload in the event a secret is changed as it does -with Consul's key-value store. Consul Template will fetch a new secret at half -the lease duration of the original secret. For example, most items in Vault's -generic secret backend have a default 30 day lease. This means Consul Template -will renew the secret every 15 days. As such, it is recommended that a smaller -lease duration be used when generating the initial secret to force Consul -Template to renew more often. +Template will not immediately reload in the event a secret is changed as it +does with Consul's key-value store. Consul Template will renew the secret with +Vault's [Renewer API](https://godoc.org/github.com/hashicorp/vault/api#Renewer). +The Renew API tries to use most of the time the secret is good, renewing at +around 90% of the lease time (as set by Vault). Also consider enabling `error_on_missing_key` when working with templates that will interact with Vault. By default, Consul Template uses Go's templating language. When accessing a struct field or map key that does not exist, it -defaults to printing "". This may not be the desired behavior, +defaults to printing ``. This may not be the desired behavior, especially when working with passwords or other data. As such, it is recommended you set: @@ -956,7 +1055,7 @@ The example above is querying Consul for healthy "web" services, in the "east-aw ```liquid {{ range service "web" }} -server {{ .Name }}{{ .Address }}:{{ .Port }}{{ end }} +server {{ .Name }} {{ .Address }}:{{ .Port }}{{ end }} ``` renders the IP addresses of all _healthy_ nodes with a logical service named @@ -1060,6 +1159,23 @@ nested/config/value "value" Unlike `ls`, `tree` returns **all** keys under the prefix, just like the Unix `tree` command. +##### `safeTree` + +Same as [tree](#tree), but refuse to render template, if the KV prefix query return blank/empty data. + +This is especially useful, for rendering mission critical files, that are being populated by consul-template. + +For example: + +```text +/root/.ssh/authorized_keys +/etc/sysconfig/iptables +``` + +Using `safeTree` on empty prefixes will result in template output not being rendered at all. + +To learn how `safeTree` was born see [CT-1131](https://github.com/hashicorp/consul-template/issues/1131) [C-3975](https://github.com/hashicorp/consul/issues/3975) and [CR-82](https://github.com/hashicorp/consul-replicate/issues/82). + --- #### Scratch @@ -1250,6 +1366,81 @@ Takes the list of services returned by the [`service`](#service) or {{ end }}{{ end }} ``` +##### `byMeta` + +Takes a list of services returned by the [`service`](#service) or +[`services`](#services) and returns a map that groups services by ServiceMeta values. +Multiple service meta keys can be passed as a comma separated string. `|int` can be added to +a meta key to convert numbers from service meta values to padded numbers in `printf "%05d" % value` +format (useful for sorting as Go Template sorts maps by keys). + +**Example**: + +If we have the following services registered in Consul: + +```json +{ + "Services": [ + { + "ID": "redis-dev-1", + "Name": "redis", + "ServiceMeta": { + "environment": "dev", + "shard_number": "1" + }, + ... + }, + { + "ID": "redis-prod-1", + "Name": "redis", + "ServiceMeta": { + "environment": "prod", + "shard_number": "1" + }, + ... + }, + { + "ID": "redis-prod-2", + "Name": "redis", + "ServiceMeta": { + "environment": "prod", + "shard_number": "2", + }, + ... + } + ] +} +``` + +```liquid +{{ service "redis|any" | byMeta "environment,shard_number|int" | toJSON }} +``` + +The code above will produce a map of services grouped by meta: + +```json +{ + "dev_00001": [ + { + "ID": "redis-dev-1", + ... + } + ], + "prod_00001": [ + { + "ID": "redis-prod-1", + ... + } + ], + "prod_00002": [ + { + "ID": "redis-prod-2", + ... + } + ] +} +``` + ##### `contains` Determines if a needle is within an iterable element. @@ -1260,7 +1451,7 @@ Determines if a needle is within an iterable element. {{ end }} ``` -#### `containsAll` +##### `containsAll` Returns `true` if all needles are within an iterable element, or `false` otherwise. Returns `true` if the list of needles is empty. @@ -1271,7 +1462,7 @@ otherwise. Returns `true` if the list of needles is empty. {{ end }} ``` -#### `containsAny` +##### `containsAny` Returns `true` if any needle is within an iterable element, or `false` otherwise. Returns `false` if the list of needles is empty. @@ -1282,7 +1473,7 @@ otherwise. Returns `false` if the list of needles is empty. {{ end }} ``` -#### `containsNone` +##### `containsNone` Returns `true` if no needles are within an iterable element, or `false` otherwise. Returns `true` if the list of needles is empty. @@ -1293,7 +1484,7 @@ otherwise. Returns `true` if the list of needles is empty. {{ end }} ``` -#### `containsNotAll` +##### `containsNotAll` Returns `true` if some needle is not within an iterable element, or `false` otherwise. Returns `false` if the list of needles is empty. @@ -1365,6 +1556,17 @@ You will need to have a reasonable format about your data in Consul. Please see [Go's text/template package][text-template] for more information. +##### `explodeMap` + +Takes the value of a map and converts it into a deeply-nested map for parsing/traversing, +using the same logic as `explode`. + +```liquid +{{ scratch.MapSet "example", "foo/bar", "a" }} +{{ scratch.MapSet "example", "foo/baz", "b" }} +{{ scratch.Get "example" | explodeMap | toYAML }} +``` + ##### `indent` Indents a block of text by prefixing N number of spaces per line. @@ -1705,6 +1907,18 @@ minconns: "2" Note: Consul stores all KV data as strings. Thus true is "true", 1 is "1", etc. +##### `sockaddr` + +Takes a quote-escaped template string as an argument and passes it on to +[hashicorp/go-sockaddr](https://github.com/hashicorp/go-sockaddr) templating engine. + +```liquid +{{ sockaddr "GetPrivateIP" }} +``` + +See [hashicorp/go-sockaddr documentation](https://godoc.org/github.com/hashicorp/go-sockaddr) +for more information. + --- #### Math Functions @@ -2208,12 +2422,7 @@ A: Configuration management tools are designed to be used in unison with Consul ## Contributing -To build and install Consul Template locally, you will need to install the -Docker engine: - -- [Docker for Mac](https://docs.docker.com/engine/installation/mac/) -- [Docker for Windows](https://docs.docker.com/engine/installation/windows/) -- [Docker for Linux](https://docs.docker.com/engine/installation/linux/ubuntulinux/) +To build and install Envconsul locally, you will need to [install Go][go]. Clone the repository: @@ -2255,3 +2464,4 @@ go test ./... -run SomeTestFunction_name [releases]: https://releases.hashicorp.com/consul-template "Consul Template Releases" [text-template]: https://golang.org/pkg/text/template/ "Go's text/template package" [vault]: https://www.vaultproject.io "Vault by HashiCorp" +[go]: https://golang.org "Go programming language" diff --git a/vendor/github.com/hashicorp/consul-template/child/child.go b/vendor/github.com/hashicorp/consul-template/child/child.go index ebee97b97..3c94816f5 100644 --- a/vendor/github.com/hashicorp/consul-template/child/child.go +++ b/vendor/github.com/hashicorp/consul-template/child/child.go @@ -197,7 +197,7 @@ func (c *Child) Reload() error { c.Lock() defer c.Unlock() - c.kill() + c.kill(false) return c.start() } @@ -223,7 +223,7 @@ func (c *Child) Kill() { log.Printf("[INFO] (child) killing process") c.Lock() defer c.Unlock() - c.kill() + c.kill(false) } // Stop behaves almost identical to Kill except it suppresses future processes @@ -231,6 +231,17 @@ func (c *Child) Kill() { // process from sending its value back up the exit channel. This is useful // when doing a graceful shutdown of an application. func (c *Child) Stop() { + c.internalStop(false) +} + +// StopImmediately behaves almost identical to Stop except it does not wait +// for any random splay if configured. This is used for performing a fast +// shutdown of consul-template and its children when a kill signal is received. +func (c *Child) StopImmediately() { + c.internalStop(true) +} + +func (c *Child) internalStop(immediately bool) { log.Printf("[INFO] (child) stopping process") c.Lock() @@ -242,7 +253,7 @@ func (c *Child) Stop() { log.Printf("[WARN] (child) already stopped") return } - c.kill() + c.kill(immediately) close(c.stopCh) c.stopped = true } @@ -354,7 +365,7 @@ func (c *Child) reload() error { return c.signal(c.reloadSignal) } -func (c *Child) kill() { +func (c *Child) kill(immediately bool) { if !c.running() { return } @@ -362,13 +373,15 @@ func (c *Child) kill() { exited := false process := c.cmd.Process - if c.cmd.ProcessState == nil { + if c.cmd.ProcessState != nil { + log.Printf("[DEBUG] (child) Kill() called but process dead; not waiting for splay.") + } else if immediately { + log.Printf("[DEBUG] (child) Kill() called but performing immediate shutdown; not waiting for splay.") + } else { select { case <-c.stopCh: case <-c.randomSplay(): } - } else { - log.Printf("[DEBUG] (runner) Kill() called but process dead; not waiting for splay.") } if c.killSignal != nil { diff --git a/vendor/github.com/hashicorp/consul-template/cli.go b/vendor/github.com/hashicorp/consul-template/cli.go index 2ce9e0883..52b99c27a 100644 --- a/vendor/github.com/hashicorp/consul-template/cli.go +++ b/vendor/github.com/hashicorp/consul-template/cli.go @@ -152,7 +152,7 @@ func (cli *CLI) Run(args []string) int { go runner.Start() case *config.KillSignal: fmt.Fprintf(cli.errStream, "Cleaning up...\n") - runner.Stop() + runner.StopImmediately() return ExitCodeInterrupt case signals.SignalLookup["SIGCHLD"]: // The SIGCHLD signal is sent to the parent of a child process when it @@ -731,11 +731,6 @@ Options: -vault-addr=
Sets the address of the Vault server - -vault-grace= - Sets the grace period between lease renewal and secret re-acquisition - if - the remaining lease duration is less than this value, Consul Template will - acquire a new secret from Vault - -vault-renew-token Periodically renew the provided Vault API token - this defaults to "true" and will renew the token at half of the lease duration diff --git a/vendor/github.com/hashicorp/consul-template/dependency/dependency.go b/vendor/github.com/hashicorp/consul-template/dependency/dependency.go index 04c16b1a1..c9161f82f 100644 --- a/vendor/github.com/hashicorp/consul-template/dependency/dependency.go +++ b/vendor/github.com/hashicorp/consul-template/dependency/dependency.go @@ -18,7 +18,7 @@ const ( nodeNameRe = `(?P[[:word:]\.\-\_]+)` nearRe = `(~(?P[[:word:]\.\-\_]+))?` prefixRe = `/?(?P[^@]+)` - tagRe = `((?P[[:word:]\.\-\_]+)\.)?` + tagRe = `((?P[[:word:]=:\.\-\_]+)\.)?` ) type Type int diff --git a/vendor/github.com/hashicorp/consul-template/dependency/vault_common.go b/vendor/github.com/hashicorp/consul-template/dependency/vault_common.go index 213c63293..6abe69cfd 100644 --- a/vendor/github.com/hashicorp/consul-template/dependency/vault_common.go +++ b/vendor/github.com/hashicorp/consul-template/dependency/vault_common.go @@ -7,6 +7,9 @@ import ( "strings" "time" + "crypto/x509" + "encoding/pem" + "github.com/hashicorp/vault/api" ) @@ -64,15 +67,80 @@ type SecretWrapInfo struct { WrappedAccessor string } -// vaultRenewDuration accepts a secret and returns the recommended amount of +// +type renewer interface { + Dependency + stopChan() chan struct{} + secrets() (*Secret, *api.Secret) +} + +func renewSecret(clients *ClientSet, d renewer) error { + log.Printf("[TRACE] %s: starting renewer", d) + + secret, vaultSecret := d.secrets() + renewer, err := clients.Vault().NewRenewer(&api.RenewerInput{ + Secret: vaultSecret, + }) + if err != nil { + return err + } + go renewer.Renew() + defer renewer.Stop() + + for { + select { + case err := <-renewer.DoneCh(): + if err != nil { + log.Printf("[WARN] %s: failed to renew: %s", d, err) + } + log.Printf("[WARN] %s: renewer done (maybe the lease expired)", d) + return nil + case renewal := <-renewer.RenewCh(): + log.Printf("[TRACE] %s: successfully renewed", d) + printVaultWarnings(d, renewal.Secret.Warnings) + updateSecret(secret, renewal.Secret) + case <-d.stopChan(): + return ErrStopped + } + } +} + +// durationFrom cert gets the duration of validity from cert data and +// returns that value as an integer number of seconds +func durationFromCert(certData string) int { + block, _ := pem.Decode([]byte(certData)) + if block == nil { + return -1 + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + log.Printf("[WARN] Unable to parse certificate data: %s", err) + return -1 + } + + return int(cert.NotAfter.Sub(cert.NotBefore).Seconds()) +} + +// leaseCheckWait accepts a secret and returns the recommended amount of // time to sleep. -func vaultRenewDuration(s *Secret) time.Duration { +func leaseCheckWait(s *Secret) time.Duration { // Handle whether this is an auth or a regular secret. base := s.LeaseDuration if s.Auth != nil && s.Auth.LeaseDuration > 0 { base = s.Auth.LeaseDuration } + // Handle if this is a certificate with no lease + if certInterface, ok := s.Data["certificate"]; ok && s.LeaseID == "" { + if certData, ok := certInterface.(string); ok { + newDuration := durationFromCert(certData) + if newDuration > 0 { + log.Printf("[DEBUG] Found certificate and set lease duration to %d seconds", newDuration) + base = newDuration + } + } + } + // Ensure we have a lease duration, since sometimes this can be zero. if base <= 0 { base = int(VaultDefaultLeaseDuration.Seconds()) diff --git a/vendor/github.com/hashicorp/consul-template/dependency/vault_read.go b/vendor/github.com/hashicorp/consul-template/dependency/vault_read.go index eae6fc916..00ebf27ec 100644 --- a/vendor/github.com/hashicorp/consul-template/dependency/vault_read.go +++ b/vendor/github.com/hashicorp/consul-template/dependency/vault_read.go @@ -18,7 +18,8 @@ var ( // VaultReadQuery is the dependency to Vault for a secret type VaultReadQuery struct { - stopCh chan struct{} + stopCh chan struct{} + sleepCh chan time.Duration rawPath string queryValues url.Values @@ -45,81 +46,70 @@ func NewVaultReadQuery(s string) (*VaultReadQuery, error) { return &VaultReadQuery{ stopCh: make(chan struct{}, 1), + sleepCh: make(chan time.Duration, 1), rawPath: secretURL.Path, queryValues: secretURL.Query(), }, nil } // Fetch queries the Vault API -func (d *VaultReadQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) { +func (d *VaultReadQuery) Fetch(clients *ClientSet, opts *QueryOptions, +) (interface{}, *ResponseMetadata, error) { select { case <-d.stopCh: return nil, nil, ErrStopped default: } + select { + case dur := <-d.sleepCh: + time.Sleep(dur) + default: + } - opts = opts.Merge(&QueryOptions{}) + firstRun := d.secret == nil - if d.secret != nil { - if vaultSecretRenewable(d.secret) { - log.Printf("[TRACE] %s: starting renewer", d) - - renewer, err := clients.Vault().NewRenewer(&api.RenewerInput{ - Grace: opts.VaultGrace, - Secret: d.vaultSecret, - }) - if err != nil { - return nil, nil, errors.Wrap(err, d.String()) - } - go renewer.Renew() - defer renewer.Stop() - - RENEW: - for { - select { - case err := <-renewer.DoneCh(): - if err != nil { - log.Printf("[WARN] %s: failed to renew: %s", d, err) - } - log.Printf("[WARN] %s: renewer returned (maybe the lease expired)", d) - break RENEW - case renewal := <-renewer.RenewCh(): - log.Printf("[TRACE] %s: successfully renewed", d) - printVaultWarnings(d, renewal.Secret.Warnings) - updateSecret(d.secret, renewal.Secret) - case <-d.stopCh: - return nil, nil, ErrStopped - } - } - } else { - // The secret isn't renewable, probably the generic secret backend. - dur := vaultRenewDuration(d.secret) - log.Printf("[TRACE] %s: secret is not renewable, sleeping for %s", d, dur) - select { - case <-time.After(dur): - // The lease is almost expired, it's time to request a new one. - case <-d.stopCh: - return nil, nil, ErrStopped - } + if !firstRun && vaultSecretRenewable(d.secret) { + err := renewSecret(clients, d) + if err != nil { + return nil, nil, errors.Wrap(err, d.String()) } } - // We don't have a secret, or the prior renewal failed - vaultSecret, err := d.readSecret(clients, opts) + err := d.fetchSecret(clients, opts) if err != nil { return nil, nil, errors.Wrap(err, d.String()) } - // Print any warnings - printVaultWarnings(d, vaultSecret.Warnings) - - // Create the cloned secret which will be exposed to the template. - d.vaultSecret = vaultSecret - d.secret = transformSecret(vaultSecret) + if !vaultSecretRenewable(d.secret) { + dur := leaseCheckWait(d.secret) + log.Printf("[TRACE] %s: non-renewable secret, set sleep for %s", d, dur) + d.sleepCh <- dur + } return respWithMetadata(d.secret) } +func (d *VaultReadQuery) fetchSecret(clients *ClientSet, opts *QueryOptions, +) error { + opts = opts.Merge(&QueryOptions{}) + vaultSecret, err := d.readSecret(clients, opts) + if err == nil { + printVaultWarnings(d, vaultSecret.Warnings) + d.vaultSecret = vaultSecret + // the cloned secret which will be exposed to the template + d.secret = transformSecret(vaultSecret) + } + return err +} + +func (d *VaultReadQuery) stopChan() chan struct{} { + return d.stopCh +} + +func (d *VaultReadQuery) secrets() (*Secret, *api.Secret) { + return d.secret, d.vaultSecret +} + // CanShare returns if this dependency is shareable. func (d *VaultReadQuery) CanShare() bool { return false @@ -147,7 +137,8 @@ func (d *VaultReadQuery) readSecret(clients *ClientSet, opts *QueryOptions) (*ap if d.isKVv2 == nil { mountPath, isKVv2, err := isKVv2(vaultClient, d.rawPath) if err != nil { - log.Printf("[WARN] %s: failed to check if %s is KVv2, assume not: %s", d, d.rawPath, err) + log.Printf("[WARN] %s: failed to check if %s is KVv2, "+ + "assume not: %s", d, d.rawPath, err) isKVv2 = false d.secretPath = d.rawPath } else if isKVv2 { @@ -163,12 +154,22 @@ func (d *VaultReadQuery) readSecret(clients *ClientSet, opts *QueryOptions) (*ap Path: "/v1/" + d.secretPath, RawQuery: queryString, }) - vaultSecret, err := vaultClient.Logical().ReadWithData(d.secretPath, d.queryValues) + vaultSecret, err := vaultClient.Logical().ReadWithData(d.secretPath, + d.queryValues) + if err != nil { return nil, errors.Wrap(err, d.String()) } - if vaultSecret == nil { + if vaultSecret == nil || deletedKVv2(vaultSecret) { return nil, fmt.Errorf("no secret exists at %s", d.secretPath) } return vaultSecret, nil } + +func deletedKVv2(s *api.Secret) bool { + switch md := s.Data["metadata"].(type) { + case map[string]interface{}: + return md["deletion_time"] != "" + } + return false +} diff --git a/vendor/github.com/hashicorp/consul-template/dependency/vault_token.go b/vendor/github.com/hashicorp/consul-template/dependency/vault_token.go index a3c7f4c87..93ad5984a 100644 --- a/vendor/github.com/hashicorp/consul-template/dependency/vault_token.go +++ b/vendor/github.com/hashicorp/consul-template/dependency/vault_token.go @@ -1,9 +1,6 @@ package dependency import ( - "log" - "time" - "github.com/hashicorp/vault/api" "github.com/pkg/errors" ) @@ -37,66 +34,33 @@ func NewVaultTokenQuery(token string) (*VaultTokenQuery, error) { } // Fetch queries the Vault API -func (d *VaultTokenQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) { +func (d *VaultTokenQuery) Fetch(clients *ClientSet, opts *QueryOptions, +) (interface{}, *ResponseMetadata, error) { select { case <-d.stopCh: return nil, nil, ErrStopped default: } - opts = opts.Merge(&QueryOptions{}) - if vaultSecretRenewable(d.secret) { - log.Printf("[TRACE] %s: starting renewer", d) - - renewer, err := clients.Vault().NewRenewer(&api.RenewerInput{ - Grace: opts.VaultGrace, - Secret: d.vaultSecret, - }) + err := renewSecret(clients, d) if err != nil { return nil, nil, errors.Wrap(err, d.String()) } - go renewer.Renew() - defer renewer.Stop() - - RENEW: - for { - select { - case err := <-renewer.DoneCh(): - if err != nil { - log.Printf("[WARN] %s: failed to renew: %s", d, err) - } - log.Printf("[WARN] %s: renewer returned (maybe the lease expired)", d) - break RENEW - case renewal := <-renewer.RenewCh(): - log.Printf("[TRACE] %s: successfully renewed", d) - printVaultWarnings(d, renewal.Secret.Warnings) - updateSecret(d.secret, renewal.Secret) - case <-d.stopCh: - return nil, nil, ErrStopped - } - } - } - - // The secret isn't renewable, probably the generic secret backend. - // TODO This is incorrect when given a non-renewable template. We should - // instead to a lookup self to determine the lease duration. - dur := vaultRenewDuration(d.secret) - if dur < opts.VaultGrace { - dur = opts.VaultGrace - } - - log.Printf("[TRACE] %s: token is not renewable, sleeping for %s", d, dur) - select { - case <-time.After(dur): - // The lease is almost expired, it's time to request a new one. - case <-d.stopCh: - return nil, nil, ErrStopped + renewSecret(clients, d) } return nil, nil, ErrLeaseExpired } +func (d *VaultTokenQuery) stopChan() chan struct{} { + return d.stopCh +} + +func (d *VaultTokenQuery) secrets() (*Secret, *api.Secret) { + return d.secret, d.vaultSecret +} + // CanShare returns if this dependency is shareable. func (d *VaultTokenQuery) CanShare() bool { return false diff --git a/vendor/github.com/hashicorp/consul-template/dependency/vault_write.go b/vendor/github.com/hashicorp/consul-template/dependency/vault_write.go index 22aad3aa2..c2841712f 100644 --- a/vendor/github.com/hashicorp/consul-template/dependency/vault_write.go +++ b/vendor/github.com/hashicorp/consul-template/dependency/vault_write.go @@ -21,7 +21,8 @@ var ( // VaultWriteQuery is the dependency to Vault for a secret type VaultWriteQuery struct { - stopCh chan struct{} + stopCh chan struct{} + sleepCh chan time.Duration path string data map[string]interface{} @@ -42,6 +43,7 @@ func NewVaultWriteQuery(s string, d map[string]interface{}) (*VaultWriteQuery, e return &VaultWriteQuery{ stopCh: make(chan struct{}, 1), + sleepCh: make(chan time.Duration, 1), path: s, data: d, dataHash: sha1Map(d), @@ -49,75 +51,62 @@ func NewVaultWriteQuery(s string, d map[string]interface{}) (*VaultWriteQuery, e } // Fetch queries the Vault API -func (d *VaultWriteQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) { +func (d *VaultWriteQuery) Fetch(clients *ClientSet, opts *QueryOptions, +) (interface{}, *ResponseMetadata, error) { select { case <-d.stopCh: return nil, nil, ErrStopped default: } + select { + case dur := <-d.sleepCh: + time.Sleep(dur) + default: + } - opts = opts.Merge(&QueryOptions{}) + firstRun := d.secret == nil - if d.secret != nil { - if vaultSecretRenewable(d.secret) { - log.Printf("[TRACE] %s: starting renewer", d) - - renewer, err := clients.Vault().NewRenewer(&api.RenewerInput{ - Grace: opts.VaultGrace, - Secret: d.vaultSecret, - }) - if err != nil { - return nil, nil, errors.Wrap(err, d.String()) - } - go renewer.Renew() - defer renewer.Stop() - - RENEW: - for { - select { - case err := <-renewer.DoneCh(): - if err != nil { - log.Printf("[WARN] %s: failed to renew: %s", d, err) - } - log.Printf("[WARN] %s: renewer returned (maybe the lease expired)", d) - break RENEW - case renewal := <-renewer.RenewCh(): - log.Printf("[TRACE] %s: successfully renewed", d) - printVaultWarnings(d, renewal.Secret.Warnings) - updateSecret(d.secret, renewal.Secret) - case <-d.stopCh: - return nil, nil, ErrStopped - } - } - } else { - // The secret isn't renewable, probably the generic secret backend. - dur := vaultRenewDuration(d.secret) - log.Printf("[TRACE] %s: secret is not renewable, sleeping for %s", d, dur) - select { - case <-time.After(dur): - // The lease is almost expired, it's time to request a new one. - case <-d.stopCh: - return nil, nil, ErrStopped - } + if !firstRun && vaultSecretRenewable(d.secret) { + err := renewSecret(clients, d) + if err != nil { + return nil, nil, errors.Wrap(err, d.String()) } } - // We don't have a secret, or the prior renewal failed + opts = opts.Merge(&QueryOptions{}) vaultSecret, err := d.writeSecret(clients, opts) if err != nil { return nil, nil, errors.Wrap(err, d.String()) } - // Print any warnings - printVaultWarnings(d, vaultSecret.Warnings) + // vaultSecret == nil when writing to KVv1 engines + if vaultSecret == nil { + return respWithMetadata(d.secret) + } - // Create the cloned secret which will be exposed to the template. + printVaultWarnings(d, vaultSecret.Warnings) d.vaultSecret = vaultSecret + // cloned secret which will be exposed to the template d.secret = transformSecret(vaultSecret) + if !vaultSecretRenewable(d.secret) { + dur := leaseCheckWait(d.secret) + log.Printf("[TRACE] %s: non-renewable secret, set sleep for %s", d, dur) + d.sleepCh <- dur + } + return respWithMetadata(d.secret) } +// meet renewer interface +func (d *VaultWriteQuery) stopChan() chan struct{} { + return d.stopCh +} + +func (d *VaultWriteQuery) secrets() (*Secret, *api.Secret) { + return d.secret, d.vaultSecret +} + // CanShare returns if this dependency is shareable. func (d *VaultWriteQuery) CanShare() bool { return false @@ -168,14 +157,20 @@ func (d *VaultWriteQuery) writeSecret(clients *ClientSet, opts *QueryOptions) (* RawQuery: opts.String(), }) - vaultSecret, err := clients.Vault().Logical().Write(d.path, d.data) + data := d.data + + _, isv2, _ := isKVv2(clients.Vault(), d.path) + if isv2 { + data = map[string]interface{}{"data": d.data} + } + + vaultSecret, err := clients.Vault().Logical().Write(d.path, data) if err != nil { return nil, errors.Wrap(err, d.String()) } - if vaultSecret == nil { - if _, isv2, _ := isKVv2(clients.Vault(), d.path); isv2 { - return nil, fmt.Errorf("no secret exists at %s", d.path) - } + // vaultSecret is always nil when KVv1 engine (isv2==false) + if isv2 && vaultSecret == nil { + return nil, fmt.Errorf("no secret exists at %s", d.path) } return vaultSecret, nil diff --git a/vendor/github.com/hashicorp/consul-template/go.mod b/vendor/github.com/hashicorp/consul-template/go.mod index 4698887d3..a449ad5ba 100644 --- a/vendor/github.com/hashicorp/consul-template/go.mod +++ b/vendor/github.com/hashicorp/consul-template/go.mod @@ -7,13 +7,14 @@ require ( github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect github.com/frankban/quicktest v1.4.0 // indirect github.com/google/btree v1.0.0 // indirect - github.com/hashicorp/consul/api v1.1.0 - github.com/hashicorp/consul/sdk v0.1.1 + github.com/hashicorp/consul/api v1.2.0 + github.com/hashicorp/consul/sdk v0.2.0 github.com/hashicorp/go-gatedio v0.5.0 github.com/hashicorp/go-immutable-radix v1.1.0 // indirect github.com/hashicorp/go-msgpack v0.5.5 // indirect github.com/hashicorp/go-multierror v1.0.0 github.com/hashicorp/go-rootcerts v1.0.1 + github.com/hashicorp/go-sockaddr v1.0.2 github.com/hashicorp/go-syslog v1.0.0 github.com/hashicorp/golang-lru v0.5.3 // indirect github.com/hashicorp/hcl v1.0.0 diff --git a/vendor/github.com/hashicorp/consul-template/go.sum b/vendor/github.com/hashicorp/consul-template/go.sum index 4fd78bc91..648fcc967 100644 --- a/vendor/github.com/hashicorp/consul-template/go.sum +++ b/vendor/github.com/hashicorp/consul-template/go.sum @@ -36,11 +36,10 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/hashicorp/consul v1.5.3 h1:EmTWRf/cuqZk6Ug9tgFUVE9xNgJPpmBvJwJMvm+agSk= -github.com/hashicorp/consul/api v1.1.0 h1:BNQPM9ytxj6jbjjdRPioQ94T6YXriSopn0i8COv6SRA= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1 h1:LnuDWGNsoajlhGyHJvuWW6FVqRl8JOTPqS6CPTsYjhY= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/api v1.2.0 h1:oPsuzLp2uk7I7rojPKuncWbZ+m5TMoD4Ivs+2Rkeh4Y= +github.com/hashicorp/consul/api v1.2.0/go.mod h1:1SIkFYi2ZTXUE5Kgt179+4hH33djo11+0Eo2XgTAtkw= +github.com/hashicorp/consul/sdk v0.2.0 h1:GWFYFmry/k4b1hEoy7kSkmU8e30GAyI4VZHk0fRxeL4= +github.com/hashicorp/consul/sdk v0.2.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= diff --git a/vendor/github.com/hashicorp/consul-template/manager/runner.go b/vendor/github.com/hashicorp/consul-template/manager/runner.go index 496cb05e0..877f4bf94 100644 --- a/vendor/github.com/hashicorp/consul-template/manager/runner.go +++ b/vendor/github.com/hashicorp/consul-template/manager/runner.go @@ -390,26 +390,13 @@ func (r *Runner) Start() { // Stop halts the execution of this runner and its subprocesses. func (r *Runner) Stop() { - r.stopLock.Lock() - defer r.stopLock.Unlock() + r.internalStop(false) +} - if r.stopped { - return - } - - log.Printf("[INFO] (runner) stopping") - r.stopDedup() - r.stopWatcher() - r.stopChild() - - if err := r.deletePid(); err != nil { - log.Printf("[WARN] (runner) could not remove pid at %v: %s", - r.config.PidFile, err) - } - - r.stopped = true - - close(r.DoneCh) +// StopImmediately behaves like Stop but won't wait for any splay on any child +// process it may be running. +func (r *Runner) StopImmediately() { + r.internalStop(true) } // TemplateRenderedCh returns a channel that will be triggered when one or more @@ -437,6 +424,29 @@ func (r *Runner) RenderEvents() map[string]*RenderEvent { return times } +func (r *Runner) internalStop(immediately bool) { + r.stopLock.Lock() + defer r.stopLock.Unlock() + + if r.stopped { + return + } + + log.Printf("[INFO] (runner) stopping") + r.stopDedup() + r.stopWatcher() + r.stopChild(immediately) + + if err := r.deletePid(); err != nil { + log.Printf("[WARN] (runner) could not remove pid at %q: %s", + *r.config.PidFile, err) + } + + r.stopped = true + + close(r.DoneCh) +} + func (r *Runner) stopDedup() { if r.dedup != nil { log.Printf("[DEBUG] (runner) stopping de-duplication manager") @@ -451,13 +461,18 @@ func (r *Runner) stopWatcher() { } } -func (r *Runner) stopChild() { +func (r *Runner) stopChild(immediately bool) { r.childLock.RLock() defer r.childLock.RUnlock() if r.child != nil { - log.Printf("[DEBUG] (runner) stopping child process") - r.child.Stop() + if immediately { + log.Printf("[DEBUG] (runner) stopping child process immediately") + r.child.StopImmediately() + } else { + log.Printf("[DEBUG] (runner) stopping child process") + r.child.Stop() + } } } diff --git a/vendor/github.com/hashicorp/consul-template/renderer/renderer.go b/vendor/github.com/hashicorp/consul-template/renderer/renderer.go index 34e68a2b1..59931c19e 100644 --- a/vendor/github.com/hashicorp/consul-template/renderer/renderer.go +++ b/vendor/github.com/hashicorp/consul-template/renderer/renderer.go @@ -163,12 +163,14 @@ func AtomicWrite(path string, createDestDirs bool, contents []byte, perms os.Fil } // If we got this far, it means we are about to save the file. Copy the - // current contents of the file onto disk (if it exists) so we have a backup. + // current file so we have a backup. Note that os.Link preserves the Mode. if backup { - if _, err := os.Stat(path); !os.IsNotExist(err) { - if err := copyFile(path, path+".bak"); err != nil { - return err - } + bak, old := path+".bak", path+".old.bak" + os.Rename(bak, old) // ignore error + if err := os.Link(path, bak); err != nil { + log.Printf("[WARN] (runner) could not backup %q: %v", path, err) + } else { + os.Remove(old) // ignore error } } @@ -178,33 +180,3 @@ func AtomicWrite(path string, createDestDirs bool, contents []byte, perms os.Fil return nil } - -// copyFile copies the file at src to the path at dst. Any errors that occur -// are returned. -func copyFile(src, dst string) error { - s, err := os.Open(src) - if err != nil { - return err - } - defer s.Close() - - stat, err := s.Stat() - if err != nil { - return err - } - - d, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, stat.Mode()) - if err != nil { - return err - } - if _, err := io.Copy(d, s); err != nil { - d.Close() - return err - } - if err := d.Close(); err != nil { - return err - } - - // io.Copy can restrict file permissions based on umask. - return os.Chmod(dst, stat.Mode()) -} diff --git a/vendor/github.com/hashicorp/consul-template/template/funcs.go b/vendor/github.com/hashicorp/consul-template/template/funcs.go index 2270f6fd2..a599034f7 100644 --- a/vendor/github.com/hashicorp/consul-template/template/funcs.go +++ b/vendor/github.com/hashicorp/consul-template/template/funcs.go @@ -11,6 +11,7 @@ import ( "path/filepath" "reflect" "regexp" + "sort" "strconv" "strings" "text/template" @@ -18,6 +19,7 @@ import ( "github.com/BurntSushi/toml" dep "github.com/hashicorp/consul-template/dependency" + socktmpl "github.com/hashicorp/go-sockaddr/template" "github.com/pkg/errors" yaml "gopkg.in/yaml.v2" ) @@ -208,8 +210,13 @@ func keyWithDefaultFunc(b *Brain, used, missing *dep.Set) func(string, string) ( } } +func safeLsFunc(b *Brain, used, missing *dep.Set) func(string) ([]*dep.KeyPair, error) { + // call lsFunc but explicitly mark that empty data set returned on monitored KV prefix is NOT safe + return lsFunc(b, used, missing, false) +} + // lsFunc returns or accumulates keyPrefix dependencies. -func lsFunc(b *Brain, used, missing *dep.Set) func(string) ([]*dep.KeyPair, error) { +func lsFunc(b *Brain, used, missing *dep.Set, emptyIsSafe bool) func(string) ([]*dep.KeyPair, error) { return func(s string) ([]*dep.KeyPair, error) { result := []*dep.KeyPair{} @@ -231,9 +238,23 @@ func lsFunc(b *Brain, used, missing *dep.Set) func(string) ([]*dep.KeyPair, erro result = append(result, pair) } } - return result, nil + + if len(result) == 0 { + if emptyIsSafe { + // Operator used potentially unsafe ls function in the template instead of the safeLs + return result, nil + } + } else { + // non empty result is good so we just return the data + return result, nil + } + + // If we reach this part of the code result is completely empty as value returned no KV pairs + // Operator selected to use safeLs on the specific KV prefix so we will refuse to render template + // by marking d as missing } + // b.Recall either returned an error or safeLs entered unsafe case missing.Add(d) return result, nil @@ -358,6 +379,51 @@ func secretsFunc(b *Brain, used, missing *dep.Set) func(string) ([]string, error } } +// byMeta returns Services grouped by one or many ServiceMeta fields. +func byMeta(meta string, services []*dep.HealthService) (groups map[string][]*dep.HealthService, err error) { + re := regexp.MustCompile("[^a-zA-Z0-9_-]") + normalize := func(x string) string { + return re.ReplaceAllString(x, "_") + } + getOrDefault := func(m map[string]string, key string) string { + realKey := strings.TrimSuffix(key, "|int") + if val, ok := m[realKey]; ok { + if val != "" { + return val + } + } + if strings.HasSuffix(key, "|int") { + return "0" + } + return fmt.Sprintf("_no_%s_", realKey) + } + + metas := strings.Split(meta, ",") + + groups = make(map[string][]*dep.HealthService) + + for _, s := range services { + sm := s.ServiceMeta + keyParts := []string{} + for _, meta := range metas { + value := getOrDefault(sm, meta) + if strings.HasSuffix(meta, "|int") { + value = getOrDefault(sm, meta) + i, err := strconv.Atoi(value) + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("cannot parse %v as number ", value)) + } + value = fmt.Sprintf("%05d", i) + } + keyParts = append(keyParts, normalize(value)) + } + key := strings.Join(keyParts, "_") + groups[key] = append(groups[key], s) + } + + return groups, nil +} + // serviceFunc returns or accumulates health service dependencies. func serviceFunc(b *Brain, used, missing *dep.Set) func(...string) ([]*dep.HealthService, error) { return func(s ...string) ([]*dep.HealthService, error) { @@ -406,8 +472,13 @@ func servicesFunc(b *Brain, used, missing *dep.Set) func(...string) ([]*dep.Cata } } +func safeTreeFunc(b *Brain, used, missing *dep.Set) func(string) ([]*dep.KeyPair, error) { + // call treeFunc but explicitly mark that empty data set returned on monitored KV prefix is NOT safe + return treeFunc(b, used, missing, false) +} + // treeFunc returns or accumulates keyPrefix dependencies. -func treeFunc(b *Brain, used, missing *dep.Set) func(string) ([]*dep.KeyPair, error) { +func treeFunc(b *Brain, used, missing *dep.Set, emptyIsSafe bool) func(string) ([]*dep.KeyPair, error) { return func(s string) ([]*dep.KeyPair, error) { result := []*dep.KeyPair{} @@ -430,9 +501,23 @@ func treeFunc(b *Brain, used, missing *dep.Set) func(string) ([]*dep.KeyPair, er result = append(result, pair) } } - return result, nil + + if len(result) == 0 { + if emptyIsSafe { + // Operator used potentially unsafe tree function in the template instead of the safeTree + return result, nil + } + } else { + // non empty result is good so we just return the data + return result, nil + } + + // If we reach this part of the code result is completely empty as value returned no KV pairs + // Operator selected to use safeTree on the specific KV prefix so we will refuse to render template + // by marking d as missing } + // b.Recall either returned an error or safeTree entered unsafe case missing.Add(d) return result, nil @@ -583,8 +668,8 @@ func explode(pairs []*dep.KeyPair) (map[string]interface{}, error) { return m, nil } -// explodeHelper is a recursive helper for explode. -func explodeHelper(m map[string]interface{}, k, v, p string) error { +// explodeHelper is a recursive helper for explode and explodeMap +func explodeHelper(m map[string]interface{}, k string, v interface{}, p string) error { if strings.Contains(k, "/") { parts := strings.Split(k, "/") top := parts[0] @@ -607,6 +692,24 @@ func explodeHelper(m map[string]interface{}, k, v, p string) error { return nil } +// explodeMap turns a single-level map into a deeply-nested hash. +func explodeMap(mapIn map[string]interface{}) (map[string]interface{}, error) { + mapOut := make(map[string]interface{}) + + var keys []string + for k := range mapIn { + keys = append(keys, k) + } + sort.Strings(keys) + + for i := range keys { + if err := explodeHelper(mapOut, keys[i], mapIn[keys[i]], keys[i]); err != nil { + return nil, errors.Wrap(err, "explodeMap") + } + } + return mapOut, nil +} + // in searches for a given value in a given interface. func in(l, v interface{}) (bool, error) { lv := reflect.ValueOf(l) @@ -697,16 +800,41 @@ func indent(spaces int, s string) (string, error) { // print(i) // } // -func loop(ints ...int64) (<-chan int64, error) { - var start, stop int64 - switch len(ints) { +func loop(ifaces ...interface{}) (<-chan int64, error) { + + to64 := func(i interface{}) (int64, error) { + v := reflect.ValueOf(i) + switch v.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, + reflect.Int64: + return int64(v.Int()), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, + reflect.Uint64: + return int64(v.Uint()), nil + case reflect.String: + return parseInt(v.String()) + } + return 0, fmt.Errorf("loop: bad argument type: %T", i) + } + + var i1, i2 interface{} + switch len(ifaces) { case 1: - start, stop = 0, ints[0] + i1, i2 = 0, ifaces[0] case 2: - start, stop = ints[0], ints[1] + i1, i2 = ifaces[0], ifaces[1] default: - return nil, fmt.Errorf("loop: wrong number of arguments, expected 1 or 2"+ - ", but got %d", len(ints)) + return nil, fmt.Errorf("loop: wrong number of arguments, expected "+ + "1 or 2, but got %d", len(ifaces)) + } + + start, err := to64(i1) + if err != nil { + return nil, err + } + stop, err := to64(i2) + if err != nil { + return nil, err } ch := make(chan int64) @@ -1184,3 +1312,13 @@ func pathInSandbox(sandbox, path string) error { } return nil } + +// sockaddr wraps go-sockaddr templating +func sockaddr(args ...string) (string, error) { + t := fmt.Sprintf("{{ %s }} ", strings.Join(args, " ")) + k, err := socktmpl.Parse(t) + if err != nil { + return "", err + } + return k, nil +} diff --git a/vendor/github.com/hashicorp/consul-template/template/template.go b/vendor/github.com/hashicorp/consul-template/template/template.go index ff3f36f21..36da55183 100644 --- a/vendor/github.com/hashicorp/consul-template/template/template.go +++ b/vendor/github.com/hashicorp/consul-template/template/template.go @@ -230,14 +230,16 @@ func funcMap(i *funcMapInput) template.FuncMap { "key": keyFunc(i.brain, i.used, i.missing), "keyExists": keyExistsFunc(i.brain, i.used, i.missing), "keyOrDefault": keyWithDefaultFunc(i.brain, i.used, i.missing), - "ls": lsFunc(i.brain, i.used, i.missing), + "ls": lsFunc(i.brain, i.used, i.missing, true), + "safeLs": safeLsFunc(i.brain, i.used, i.missing), "node": nodeFunc(i.brain, i.used, i.missing), "nodes": nodesFunc(i.brain, i.used, i.missing), "secret": secretFunc(i.brain, i.used, i.missing), "secrets": secretsFunc(i.brain, i.used, i.missing), "service": serviceFunc(i.brain, i.used, i.missing), "services": servicesFunc(i.brain, i.used, i.missing), - "tree": treeFunc(i.brain, i.used, i.missing), + "tree": treeFunc(i.brain, i.used, i.missing, true), + "safeTree": safeTreeFunc(i.brain, i.used, i.missing), // Scratch "scratch": func() *Scratch { return &scratch }, @@ -257,6 +259,7 @@ func funcMap(i *funcMapInput) template.FuncMap { "env": envFunc(i.env), "executeTemplate": executeTemplateFunc(i.t), "explode": explode, + "explodeMap": explodeMap, "in": in, "indent": indent, "loop": loop, @@ -280,7 +283,8 @@ func funcMap(i *funcMapInput) template.FuncMap { "toUpper": toUpper, "toYAML": toYAML, "split": split, - + "byMeta": byMeta, + "sockaddr": sockaddr, // Math functions "add": add, "subtract": subtract, diff --git a/vendor/github.com/hashicorp/consul-template/version/version.go b/vendor/github.com/hashicorp/consul-template/version/version.go index 6e12bfc06..077931edc 100644 --- a/vendor/github.com/hashicorp/consul-template/version/version.go +++ b/vendor/github.com/hashicorp/consul-template/version/version.go @@ -2,7 +2,7 @@ package version import "fmt" -const Version = "0.21.0" +const Version = "0.22.1" var ( Name string diff --git a/vendor/github.com/hashicorp/consul-template/watch/view.go b/vendor/github.com/hashicorp/consul-template/watch/view.go index 191c008c1..bcef6c0b2 100644 --- a/vendor/github.com/hashicorp/consul-template/watch/view.go +++ b/vendor/github.com/hashicorp/consul-template/watch/view.go @@ -3,6 +3,7 @@ package watch import ( "fmt" "log" + "math/rand" "reflect" "sync" "time" @@ -207,6 +208,8 @@ func (v *View) fetch(doneCh, successCh chan<- struct{}, errCh chan<- error) { default: } + start := time.Now() // for rateLimiter below + data, rm, err := v.dependency.Fetch(v.clients, &dep.QueryOptions{ AllowStale: allowStale, WaitTime: defaultWaitTime, @@ -247,6 +250,10 @@ func (v *View) fetch(doneCh, successCh chan<- struct{}, errCh chan<- error) { allowStale = true } + if dur := rateLimiter(start); dur > 1 { + time.Sleep(dur) + } + if rm.LastIndex == v.lastIndex { log.Printf("[TRACE] (view) %s no new data (index was the same)", v.dependency) continue @@ -282,6 +289,18 @@ func (v *View) fetch(doneCh, successCh chan<- struct{}, errCh chan<- error) { } } +const minDelayBetweenUpdates = time.Millisecond * 100 + +// return a duration to sleep to limit the frequency of upstream calls +func rateLimiter(start time.Time) time.Duration { + remaining := minDelayBetweenUpdates - time.Since(start) + if remaining > 0 { + dither := time.Duration(rand.Int63n(20000000)) // 0-20ms + return remaining + dither + } + return 0 +} + // stop halts polling of this view. func (v *View) stop() { v.dependency.Stop() diff --git a/vendor/vendor.json b/vendor/vendor.json index 9ac26780c..75b747301 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -174,17 +174,18 @@ {"path":"github.com/gorilla/context","checksumSHA1":"g/V4qrXjUGG9B+e3hB+4NAYJ5Gs=","revision":"08b5f424b9271eedf6f9f0ce86cb9396ed337a42","revisionTime":"2016-08-17T18:46:32Z"}, {"path":"github.com/gorilla/mux","checksumSHA1":"STQSdSj2FcpCf0NLfdsKhNutQT0=","revision":"e48e440e4c92e3251d812f8ce7858944dfa3331c","revisionTime":"2018-08-07T07:52:56Z"}, {"path":"github.com/gorilla/websocket","checksumSHA1":"gr0edNJuVv4+olNNZl5ZmwLgscA=","revision":"0ec3d1bd7fe50c503d6df98ee649d81f4857c564","revisionTime":"2019-03-06T00:42:57Z"}, - {"path":"github.com/hashicorp/consul-template","checksumSHA1":"237KekVBW1eZohSDylZzT+/0NQI=","revision":"9e45d493d7fffa8a61dd315b714c39d1103da051","revisionTime":"2019-08-12T18:34:47Z"}, - {"path":"github.com/hashicorp/consul-template/child","checksumSHA1":"AhDPiKa7wzh3SE6Gx0WrsDYwBHg=","revision":"9e45d493d7fffa8a61dd315b714c39d1103da051","revisionTime":"2019-08-12T18:34:47Z"}, - {"path":"github.com/hashicorp/consul-template/config","checksumSHA1":"hjsBe5Qnn0DCttJkSNjy9mreW5Q=","revision":"9e45d493d7fffa8a61dd315b714c39d1103da051","revisionTime":"2019-08-12T18:34:47Z"}, - {"path":"github.com/hashicorp/consul-template/dependency","checksumSHA1":"S2ktxYTJRgmUE1GC5Bv+ZAmYWgE=","revision":"9e45d493d7fffa8a61dd315b714c39d1103da051","revisionTime":"2019-08-12T18:34:47Z"}, - {"path":"github.com/hashicorp/consul-template/logging","checksumSHA1":"o5N7SV389Ej+3b1iRNmz1dx5e1M=","revision":"9e45d493d7fffa8a61dd315b714c39d1103da051","revisionTime":"2019-08-12T18:34:47Z"}, - {"path":"github.com/hashicorp/consul-template/manager","checksumSHA1":"Ozv8RPN8d//DoFpwR2mQ/xMWhcs=","revision":"9e45d493d7fffa8a61dd315b714c39d1103da051","revisionTime":"2019-08-12T18:34:47Z"}, - {"path":"github.com/hashicorp/consul-template/renderer","checksumSHA1":"DUHtghMoLyrgPhv4lexVniBuWYk=","revision":"9e45d493d7fffa8a61dd315b714c39d1103da051","revisionTime":"2019-08-12T18:34:47Z"}, - {"path":"github.com/hashicorp/consul-template/signals","checksumSHA1":"YSEUV/9/k85XciRKu0cngxdjZLE=","revision":"9e45d493d7fffa8a61dd315b714c39d1103da051","revisionTime":"2019-08-12T18:34:47Z"}, - {"path":"github.com/hashicorp/consul-template/template","checksumSHA1":"mmM6LpEgkbLjobfgabon11mz40M=","revision":"9e45d493d7fffa8a61dd315b714c39d1103da051","revisionTime":"2019-08-12T18:34:47Z"}, - {"path":"github.com/hashicorp/consul-template/version","checksumSHA1":"eWyAvppME/4vsmaazcrf3oEbzGo=","revision":"9e45d493d7fffa8a61dd315b714c39d1103da051","revisionTime":"2019-08-12T18:34:47Z"}, - {"path":"github.com/hashicorp/consul-template/watch","checksumSHA1":"cJxopvJKg7DBBb8tnDsfmBp5Q8I=","revision":"9e45d493d7fffa8a61dd315b714c39d1103da051","revisionTime":"2019-08-12T18:34:47Z"}, + {"path":"github.com/hashicorp/consul-template","checksumSHA1":"fmltp5DcXXO4cec5ZX19GcerHDw=","revision":"f04989c64e9bd4c49a7217ac4635732dd8e0bb26","revisionTime":"2019-11-08T20:12:44Z","version":"v0.22.1","versionExact":"v0.22.1"}, + {"path":"github.com/hashicorp/consul-template/child","checksumSHA1":"yQfiSUOpV5BvGeztDd4fcA7qsbw=","revision":"f04989c64e9bd4c49a7217ac4635732dd8e0bb26","revisionTime":"2019-11-08T20:12:44Z","version":"v0.22.1","versionExact":"v0.22.1"}, + {"path":"github.com/hashicorp/consul-template/config","checksumSHA1":"hjsBe5Qnn0DCttJkSNjy9mreW5Q=","revision":"f04989c64e9bd4c49a7217ac4635732dd8e0bb26","revisionTime":"2019-11-08T20:12:44Z","version":"v0.22.1","versionExact":"v0.22.1"}, + {"path":"github.com/hashicorp/consul-template/conrfig","revision":"","version":"v0.22.1","versionExact":"v0.22.1"}, + {"path":"github.com/hashicorp/consul-template/dependency","checksumSHA1":"6Tni+iVTu73EHriUDFaFJXyZzvM=","revision":"f04989c64e9bd4c49a7217ac4635732dd8e0bb26","revisionTime":"2019-11-08T20:12:44Z","version":"v0.22.1","versionExact":"v0.22.1"}, + {"path":"github.com/hashicorp/consul-template/logging","checksumSHA1":"o5N7SV389Ej+3b1iRNmz1dx5e1M=","revision":"f04989c64e9bd4c49a7217ac4635732dd8e0bb26","revisionTime":"2019-11-08T20:12:44Z","version":"v0.22.1","versionExact":"v0.22.1"}, + {"path":"github.com/hashicorp/consul-template/manager","checksumSHA1":"BFPu1t60WuMR7HyUSR0nI6IvbA0=","revision":"f04989c64e9bd4c49a7217ac4635732dd8e0bb26","revisionTime":"2019-11-08T20:12:44Z","version":"v0.22.1","versionExact":"v0.22.1"}, + {"path":"github.com/hashicorp/consul-template/renderer","checksumSHA1":"zgTxCql4T0tvDUIMM+EQD6R/tEg=","revision":"f04989c64e9bd4c49a7217ac4635732dd8e0bb26","revisionTime":"2019-11-08T20:12:44Z","version":"v0.22.1","versionExact":"v0.22.1"}, + {"path":"github.com/hashicorp/consul-template/signals","checksumSHA1":"YSEUV/9/k85XciRKu0cngxdjZLE=","revision":"f04989c64e9bd4c49a7217ac4635732dd8e0bb26","revisionTime":"2019-11-08T20:12:44Z","version":"v0.22.1","versionExact":"v0.22.1"}, + {"path":"github.com/hashicorp/consul-template/template","checksumSHA1":"/AjvyyxEZXksXgxm1gmdJdJoXkw=","revision":"f04989c64e9bd4c49a7217ac4635732dd8e0bb26","revisionTime":"2019-11-08T20:12:44Z","version":"v0.22.1","versionExact":"v0.22.1"}, + {"path":"github.com/hashicorp/consul-template/version","checksumSHA1":"CqEejkuDiTgPVrLg0xrMmAWvNwY=","revision":"f04989c64e9bd4c49a7217ac4635732dd8e0bb26","revisionTime":"2019-11-08T20:12:44Z","version":"v0.22.1","versionExact":"v0.22.1"}, + {"path":"github.com/hashicorp/consul-template/watch","checksumSHA1":"cBIJewG416sFREUenIUK9v3zrUk=","revision":"f04989c64e9bd4c49a7217ac4635732dd8e0bb26","revisionTime":"2019-11-08T20:12:44Z","version":"v0.22.1","versionExact":"v0.22.1"}, {"path":"github.com/hashicorp/consul/agent/consul/autopilot","checksumSHA1":"+I7fgoQlrnTUGW5krqNLadWwtjg=","revision":"fb848fc48818f58690db09d14640513aa6bf3c02","revisionTime":"2018-04-13T17:05:42Z"}, {"path":"github.com/hashicorp/consul/api","checksumSHA1":"7JPBtnIgLkdcJ0ldXMTEnVjNEjA=","revision":"40cec98468b829e5cdaacb0629b3e23a028db688","revisionTime":"2019-05-22T20:19:12Z"}, {"path":"github.com/hashicorp/consul/command/flags","checksumSHA1":"soNN4xaHTbeXFgNkZ7cX0gbFXQk=","revision":"fb848fc48818f58690db09d14640513aa6bf3c02","revisionTime":"2018-04-13T17:05:42Z"},