Update go-getter to v1.1.0

vendor/github.com/hashicorp/go-getter/README.md

@@ -97,7 +97,7 @@ would download the given HTTP URL using the Git protocol.
 
 Forced protocols will also override any detectors.
 
-In the absense of a forced protocol, detectors may be run on the URL, transforming
+In the absence of a forced protocol, detectors may be run on the URL, transforming
 the protocol anyways. The above example would've used the Git protocol either
 way since the Git detector would've detected it was a GitHub URL.
 
@@ -155,20 +155,44 @@ For file downloads of any protocol, go-getter can automatically verify
 a checksum for you. Note that checksumming only works for downloading files,
 not directories, but checksumming will work for any protocol.
 
-To checksum a file, append a `checksum` query parameter to the URL.
-The paramter value should be in the format of `type:value`, where
-type is "md5", "sha1", "sha256", or "sha512". The "value" should be
-the actual checksum value. go-getter will parse out this query parameter
-automatically and use it to verify the checksum. An example URL
-is shown below:
+To checksum a file, append a `checksum` query parameter to the URL. go-getter
+will parse out this query parameter automatically and use it to verify the
+checksum. The parameter value can be in the format of `type:value` or just
+`value`, where type is "md5", "sha1", "sha256", "sha512" or "file" . The
+"value" should be the actual checksum value or download URL for "file". When
+`type` part is omitted, type will be guessed based on the length of the
+checksum string. Examples:
 
 ```
 ./foo.txt?checksum=md5:b7d96c89d09d9e204f5fedc4d5d55b21
 ```
 
+```
+./foo.txt?checksum=b7d96c89d09d9e204f5fedc4d5d55b21
+```
+
+```
+./foo.txt?checksum=file:./foo.txt.sha256sum
+```
+ 
+When checksumming from a file - ex: with `checksum=file:url` - go-getter will
+get the file linked in the URL after `file:` using the same configuration. For
+example, in `file:http://releases.ubuntu.com/cosmic/MD5SUMS` go-getter will
+download a checksum file under the aforementioned url using the http protocol.
+All protocols supported by go-getter can be used. The checksum file will be
+downloaded in a temporary file then parsed. The destination of the temporary
+file can be changed by setting system specific environment variables: `TMPDIR`
+for unix; `TMP`, `TEMP` or `USERPROFILE` on windows. Read godoc of
+[os.TempDir](https://golang.org/pkg/os/#TempDir) for more information on the
+temporary directory selection. Content of files are expected to be BSD or GNU
+style. Once go-getter is done with the checksum file; it is deleted.
+
 The checksum query parameter is never sent to the backend protocol
 implementation. It is used at a higher level by go-getter itself.
 
+If the destination file exists and the checksums match: download
+will be skipped.
+
 ### Unarchiving
 
 go-getter will automatically unarchive files into a file or directory
@@ -215,11 +239,12 @@ from the URL before going to the final protocol downloader.
 
 ## Protocol-Specific Options
 
-This section documents the protocol-specific options that can be specified
-for go-getter. These options should be appended to the input as normal query
-parameters. Depending on the usage of go-getter, applications may provide
-alternate ways of inputting options. For example, [Nomad](https://www.nomadproject.io)
-provides a nice options block for specifying options rather than in the URL.
+This section documents the protocol-specific options that can be specified for
+go-getter. These options should be appended to the input as normal query
+parameters ([HTTP headers](#headers) are an exception to this, however).
+Depending on the usage of go-getter, applications may provide alternate ways of
+inputting options. For example, [Nomad](https://www.nomadproject.io) provides a
+nice options block for specifying options rather than in the URL.
 
 ## General (All Protocols)
 
@@ -250,6 +275,9 @@ None
     from a private key file on disk, you would run `base64 -w0 <file>`.
 
     **Note**: Git 2.3+ is required to use this feature.
+  
+  * `depth` - The Git clone depth. The provided number specifies the last `n`
+    revisions to clone from the repository.
 
 ### Mercurial (`hg`)
 
@@ -263,6 +291,13 @@ To use HTTP basic authentication with go-getter, simply prepend `username:passwo
 hostname in the URL such as `https://Aladdin:OpenSesame@www.example.com/index.html`. All special
 characters, including the username and password, must be URL encoded.
 
+#### Headers
+
+Optional request headers can be added by supplying them in a custom
+[`HttpGetter`](https://godoc.org/github.com/hashicorp/go-getter#HttpGetter)
+(_not_ as query parameters like most other options). These headers will be sent
+out on every request the getter in question makes.
+
 ### S3 (`s3`)
 
 S3 takes various access configurations in the URL. Note that it will also

vendor/github.com/hashicorp/go-getter/appveyor.yml

@@ -13,4 +13,4 @@ install:
 
     go get -d -v -t ./...
 build_script:
-- cmd: go test -v ./...
+- cmd: go test ./...

vendor/github.com/hashicorp/go-getter/checksum.go

@@ -0,0 +1,291 @@
+package getter
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/md5"
+	"crypto/sha1"
+	"crypto/sha256"
+	"crypto/sha512"
+	"encoding/hex"
+	"fmt"
+	"hash"
+	"io"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	urlhelper "github.com/hashicorp/go-getter/helper/url"
+)
+
+// fileChecksum helps verifying the checksum for a file.
+type fileChecksum struct {
+	Type     string
+	Hash     hash.Hash
+	Value    []byte
+	Filename string
+}
+
+// checksum is a simple method to compute the checksum of a source file
+// and compare it to the given expected value.
+func (c *fileChecksum) checksum(source string) error {
+	f, err := os.Open(source)
+	if err != nil {
+		return fmt.Errorf("Failed to open file for checksum: %s", err)
+	}
+	defer f.Close()
+
+	c.Hash.Reset()
+	if _, err := io.Copy(c.Hash, f); err != nil {
+		return fmt.Errorf("Failed to hash: %s", err)
+	}
+
+	if actual := c.Hash.Sum(nil); !bytes.Equal(actual, c.Value) {
+		return fmt.Errorf(
+			"Checksums did not match.\nExpected: %s\nGot: %s",
+			hex.EncodeToString(c.Value),
+			hex.EncodeToString(actual))
+	}
+
+	return nil
+}
+
+// extractChecksum will return a fileChecksum based on the 'checksum'
+// parameter of u.
+// ex:
+//  http://hashicorp.com/terraform?checksum=<checksumValue>
+//  http://hashicorp.com/terraform?checksum=<checksumType>:<checksumValue>
+//  http://hashicorp.com/terraform?checksum=file:<checksum_url>
+// when checksumming from a file, extractChecksum will go get checksum_url
+// in a temporary directory, parse the content of the file then delete it.
+// Content of files are expected to be BSD style or GNU style.
+//
+// BSD-style checksum:
+//  MD5 (file1) = <checksum>
+//  MD5 (file2) = <checksum>
+//
+// GNU-style:
+//  <checksum>  file1
+//  <checksum> *file2
+//
+// see parseChecksumLine for more detail on checksum file parsing
+func (c *Client) extractChecksum(u *url.URL) (*fileChecksum, error) {
+	q := u.Query()
+	v := q.Get("checksum")
+
+	if v == "" {
+		return nil, nil
+	}
+
+	vs := strings.SplitN(v, ":", 2)
+	switch len(vs) {
+	case 2:
+		break // good
+	default:
+		// here, we try to guess the checksum from it's length
+		// if the type was not passed
+		return newChecksumFromValue(v, filepath.Base(u.EscapedPath()))
+	}
+
+	checksumType, checksumValue := vs[0], vs[1]
+
+	switch checksumType {
+	case "file":
+		return c.checksumFromFile(checksumValue, u)
+	default:
+		return newChecksumFromType(checksumType, checksumValue, filepath.Base(u.EscapedPath()))
+	}
+}
+
+func newChecksum(checksumValue, filename string) (*fileChecksum, error) {
+	c := &fileChecksum{
+		Filename: filename,
+	}
+	var err error
+	c.Value, err = hex.DecodeString(checksumValue)
+	if err != nil {
+		return nil, fmt.Errorf("invalid checksum: %s", err)
+	}
+	return c, nil
+}
+
+func newChecksumFromType(checksumType, checksumValue, filename string) (*fileChecksum, error) {
+	c, err := newChecksum(checksumValue, filename)
+	if err != nil {
+		return nil, err
+	}
+
+	c.Type = strings.ToLower(checksumType)
+	switch c.Type {
+	case "md5":
+		c.Hash = md5.New()
+	case "sha1":
+		c.Hash = sha1.New()
+	case "sha256":
+		c.Hash = sha256.New()
+	case "sha512":
+		c.Hash = sha512.New()
+	default:
+		return nil, fmt.Errorf(
+			"unsupported checksum type: %s", checksumType)
+	}
+
+	return c, nil
+}
+
+func newChecksumFromValue(checksumValue, filename string) (*fileChecksum, error) {
+	c, err := newChecksum(checksumValue, filename)
+	if err != nil {
+		return nil, err
+	}
+
+	switch len(c.Value) {
+	case md5.Size:
+		c.Hash = md5.New()
+		c.Type = "md5"
+	case sha1.Size:
+		c.Hash = sha1.New()
+		c.Type = "sha1"
+	case sha256.Size:
+		c.Hash = sha256.New()
+		c.Type = "sha256"
+	case sha512.Size:
+		c.Hash = sha512.New()
+		c.Type = "sha512"
+	default:
+		return nil, fmt.Errorf("Unknown type for checksum %s", checksumValue)
+	}
+
+	return c, nil
+}
+
+// checksumsFromFile will return all the fileChecksums found in file
+//
+// checksumsFromFile will try to guess the hashing algorithm based on content
+// of checksum file
+//
+// checksumsFromFile will only return checksums for files that match file
+// behind src
+func (c *Client) checksumFromFile(checksumFile string, src *url.URL) (*fileChecksum, error) {
+	checksumFileURL, err := urlhelper.Parse(checksumFile)
+	if err != nil {
+		return nil, err
+	}
+
+	tempfile, err := tmpFile("", filepath.Base(checksumFileURL.Path))
+	if err != nil {
+		return nil, err
+	}
+	defer os.Remove(tempfile)
+
+	c2 := &Client{
+		Ctx:              c.Ctx,
+		Getters:          c.Getters,
+		Decompressors:    c.Decompressors,
+		Detectors:        c.Detectors,
+		Pwd:              c.Pwd,
+		Dir:              false,
+		Src:              checksumFile,
+		Dst:              tempfile,
+		ProgressListener: c.ProgressListener,
+	}
+	if err = c2.Get(); err != nil {
+		return nil, fmt.Errorf(
+			"Error downloading checksum file: %s", err)
+	}
+
+	filename := filepath.Base(src.Path)
+	absPath, err := filepath.Abs(src.Path)
+	if err != nil {
+		return nil, err
+	}
+	checksumFileDir := filepath.Dir(checksumFileURL.Path)
+	relpath, err := filepath.Rel(checksumFileDir, absPath)
+	switch {
+	case err == nil ||
+		err.Error() == "Rel: can't make "+absPath+" relative to "+checksumFileDir:
+		// ex: on windows C:\gopath\...\content.txt cannot be relative to \
+		// which is okay, may be another expected path will work.
+		break
+	default:
+		return nil, err
+	}
+
+	// possible file identifiers:
+	options := []string{
+		filename,       // ubuntu-14.04.1-server-amd64.iso
+		"*" + filename, // *ubuntu-14.04.1-server-amd64.iso  Standard checksum
+		"?" + filename, // ?ubuntu-14.04.1-server-amd64.iso  shasum -p
+		relpath,        // dir/ubuntu-14.04.1-server-amd64.iso
+		"./" + relpath, // ./dir/ubuntu-14.04.1-server-amd64.iso
+		absPath,        // fullpath; set if local
+	}
+
+	f, err := os.Open(tempfile)
+	if err != nil {
+		return nil, fmt.Errorf(
+			"Error opening downloaded file: %s", err)
+	}
+	defer f.Close()
+	rd := bufio.NewReader(f)
+	for {
+		line, err := rd.ReadString('\n')
+		if err != nil {
+			if err != io.EOF {
+				return nil, fmt.Errorf(
+					"Error reading checksum file: %s", err)
+			}
+			break
+		}
+		checksum, err := parseChecksumLine(line)
+		if err != nil || checksum == nil {
+			continue
+		}
+		if checksum.Filename == "" {
+			// filename not sure, let's try
+			return checksum, nil
+		}
+		// make sure the checksum is for the right file
+		for _, option := range options {
+			if option != "" && checksum.Filename == option {
+				// any checksum will work so we return the first one
+				return checksum, nil
+			}
+		}
+	}
+	return nil, fmt.Errorf("no checksum found in: %s", checksumFile)
+}
+
+// parseChecksumLine takes a line from a checksum file and returns
+// checksumType, checksumValue and filename parseChecksumLine guesses the style
+// of the checksum BSD vs GNU by splitting the line and by counting the parts.
+// of a line.
+// for BSD type sums parseChecksumLine guesses the hashing algorithm
+// by checking the length of the checksum.
+func parseChecksumLine(line string) (*fileChecksum, error) {
+	parts := strings.Fields(line)
+
+	switch len(parts) {
+	case 4:
+		// BSD-style checksum:
+		//  MD5 (file1) = <checksum>
+		//  MD5 (file2) = <checksum>
+		if len(parts[1]) <= 2 ||
+			parts[1][0] != '(' || parts[1][len(parts[1])-1] != ')' {
+			return nil, fmt.Errorf(
+				"Unexpected BSD-style-checksum filename format: %s", line)
+		}
+		filename := parts[1][1 : len(parts[1])-1]
+		return newChecksumFromType(parts[0], parts[3], filename)
+	case 2:
+		// GNU-style:
+		//  <checksum>  file1
+		//  <checksum> *file2
+		return newChecksumFromValue(parts[0], parts[1])
+	case 0:
+		return nil, nil // empty line
+	default:
+		return newChecksumFromValue(parts[0], "")
+	}
+}

vendor/github.com/hashicorp/go-getter/client.go

@@ -1,15 +1,8 @@
 package getter
 
 import (
-	"bytes"
-	"crypto/md5"
-	"crypto/sha1"
-	"crypto/sha256"
-	"crypto/sha512"
-	"encoding/hex"
+	"context"
 	"fmt"
-	"hash"
-	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -17,7 +10,7 @@ import (
 	"strings"
 
 	urlhelper "github.com/hashicorp/go-getter/helper/url"
-	"github.com/hashicorp/go-safetemp"
+	safetemp "github.com/hashicorp/go-safetemp"
 )
 
 // Client is a client for downloading things.
@@ -26,6 +19,9 @@ import (
 // Using a client directly allows more fine-grained control over how downloading
 // is done, as well as customizing the protocols supported.
 type Client struct {
+ 	// Ctx for cancellation
+	Ctx context.Context
+
 	// Src is the source URL to get.
 	//
 	// Dst is the path to save the downloaded thing as. If Dir is set to
@@ -62,10 +58,20 @@ type Client struct {
 	//
 	// WARNING: deprecated. If Mode is set, that will take precedence.
 	Dir bool
+
+	// ProgressListener allows to track file downloads.
+	// By default a no op progress listener is used.
+	ProgressListener ProgressTracker
+
+	Options []ClientOption
 }
 
 // Get downloads the configured source to the destination.
 func (c *Client) Get() error {
+	if err := c.Configure(c.Options...); err != nil {
+		return err
+	}
+
 	// Store this locally since there are cases we swap this
 	mode := c.Mode
 	if mode == ClientModeInvalid {
@@ -76,18 +82,7 @@ func (c *Client) Get() error {
 		}
 	}
 
-	// Default decompressor value
-	decompressors := c.Decompressors
-	if decompressors == nil {
-		decompressors = Decompressors
-	}
-
-	// Detect the URL. This is safe if it is already detected.
-	detectors := c.Detectors
-	if detectors == nil {
-		detectors = Detectors
-	}
-	src, err := Detect(c.Src, c.Pwd, detectors)
+	src, err := Detect(c.Src, c.Pwd, c.Detectors)
 	if err != nil {
 		return err
 	}
@@ -119,12 +114,7 @@ func (c *Client) Get() error {
 		force = u.Scheme
 	}
 
-	getters := c.Getters
-	if getters == nil {
-		getters = Getters
-	}
-
-	g, ok := getters[force]
+	g, ok := c.Getters[force]
 	if !ok {
 		return fmt.Errorf(
 			"download not supported for scheme '%s'", force)
@@ -150,7 +140,7 @@ func (c *Client) Get() error {
 	if archiveV == "" {
 		// We don't appear to... but is it part of the filename?
 		matchingLen := 0
-		for k, _ := range decompressors {
+		for k := range c.Decompressors {
 			if strings.HasSuffix(u.Path, "."+k) && len(k) > matchingLen {
 				archiveV = k
 				matchingLen = len(k)
@@ -163,7 +153,7 @@ func (c *Client) Get() error {
 	// real path.
 	var decompressDst string
 	var decompressDir bool
-	decompressor := decompressors[archiveV]
+	decompressor := c.Decompressors[archiveV]
 	if decompressor != nil {
 		// Create a temporary directory to store our archive. We delete
 		// this at the end of everything.
@@ -182,44 +172,16 @@ func (c *Client) Get() error {
 		mode = ClientModeFile
 	}
 
-	// Determine if we have a checksum
-	var checksumHash hash.Hash
-	var checksumValue []byte
-	if v := q.Get("checksum"); v != "" {
-		// Delete the query parameter if we have it.
-		q.Del("checksum")
-		u.RawQuery = q.Encode()
-
-		// Determine the checksum hash type
-		checksumType := ""
-		idx := strings.Index(v, ":")
-		if idx > -1 {
-			checksumType = v[:idx]
-		}
-		switch checksumType {
-		case "md5":
-			checksumHash = md5.New()
-		case "sha1":
-			checksumHash = sha1.New()
-		case "sha256":
-			checksumHash = sha256.New()
-		case "sha512":
-			checksumHash = sha512.New()
-		default:
-			return fmt.Errorf(
-				"unsupported checksum type: %s", checksumType)
-		}
-
-		// Get the remainder of the value and parse it into bytes
-		b, err := hex.DecodeString(v[idx+1:])
-		if err != nil {
-			return fmt.Errorf("invalid checksum: %s", err)
-		}
-
-		// Set our value
-		checksumValue = b
+	// Determine checksum if we have one
+	checksum, err := c.extractChecksum(u)
+	if err != nil {
+		return fmt.Errorf("invalid checksum: %s", err)
 	}
 
+	// Delete the query parameter if we have it.
+	q.Del("checksum")
+	u.RawQuery = q.Encode()
+
 	if mode == ClientModeAny {
 		// Ask the getter which client mode to use
 		mode, err = g.ClientMode(u)
@@ -248,15 +210,24 @@ func (c *Client) Get() error {
 	// If we're not downloading a directory, then just download the file
 	// and return.
 	if mode == ClientModeFile {
-		err := g.GetFile(dst, u)
-		if err != nil {
-			return err
+		getFile := true
+		if checksum != nil {
+			if err := checksum.checksum(dst); err == nil {
+				// don't get the file if the checksum of dst is correct
+				getFile = false
+			}
 		}
-
-		if checksumHash != nil {
-			if err := checksum(dst, checksumHash, checksumValue); err != nil {
+		if getFile {
+			err := g.GetFile(dst, u)
+			if err != nil {
 				return err
 			}
+
+			if checksum != nil {
+				if err := checksum.checksum(dst); err != nil {
+					return err
+				}
+			}
 		}
 
 		if decompressor != nil {
@@ -291,7 +262,7 @@ func (c *Client) Get() error {
 	if decompressor == nil {
 		// If we're getting a directory, then this is an error. You cannot
 		// checksum a directory. TODO: test
-		if checksumHash != nil {
+		if checksum != nil {
 			return fmt.Errorf(
 				"checksum cannot be specified for directory download")
 		}
@@ -320,30 +291,7 @@ func (c *Client) Get() error {
 			return err
 		}
 
-		return copyDir(realDst, subDir, false)
-	}
-
-	return nil
-}
-
-// checksum is a simple method to compute the checksum of a source file
-// and compare it to the given expected value.
-func checksum(source string, h hash.Hash, v []byte) error {
-	f, err := os.Open(source)
-	if err != nil {
-		return fmt.Errorf("Failed to open file for checksum: %s", err)
-	}
-	defer f.Close()
-
-	if _, err := io.Copy(h, f); err != nil {
-		return fmt.Errorf("Failed to hash: %s", err)
-	}
-
-	if actual := h.Sum(nil); !bytes.Equal(actual, v) {
-		return fmt.Errorf(
-			"Checksums did not match.\nExpected: %s\nGot: %s",
-			hex.EncodeToString(v),
-			hex.EncodeToString(actual))
+		return copyDir(c.Ctx, realDst, subDir, false)
 	}
 
 	return nil

vendor/github.com/hashicorp/go-getter/client_option.go

@@ -0,0 +1,46 @@
+package getter
+
+import "context"
+
+// A ClientOption allows to configure a client
+type ClientOption func(*Client) error
+
+// Configure configures a client with options.
+func (c *Client) Configure(opts ...ClientOption) error {
+	if c.Ctx == nil {
+		c.Ctx = context.Background()
+	}
+	c.Options = opts
+	for _, opt := range opts {
+		err := opt(c)
+		if err != nil {
+			return err
+		}
+	}
+	// Default decompressor values
+	if c.Decompressors == nil {
+		c.Decompressors = Decompressors
+	}
+	// Default detector values
+	if c.Detectors == nil {
+		c.Detectors = Detectors
+	}
+	// Default getter values
+	if c.Getters == nil {
+		c.Getters = Getters
+	}
+
+	for _, getter := range c.Getters {
+		getter.SetClient(c)
+	}
+	return nil
+}
+
+// WithContext allows to pass a context to operation
+// in order to be able to cancel a download in progress.
+func WithContext(ctx context.Context) func(*Client) error {
+	return func(c *Client) error {
+		c.Ctx = ctx
+		return nil
+	}
+}

vendor/github.com/hashicorp/go-getter/client_option_progress.go

@@ -0,0 +1,38 @@
+package getter
+
+import (
+	"io"
+)
+
+// WithProgress allows for a user to track
+// the progress of a download.
+// For example by displaying a progress bar with
+// current download.
+// Not all getters have progress support yet.
+func WithProgress(pl ProgressTracker) func(*Client) error {
+	return func(c *Client) error {
+		c.ProgressListener = pl
+		return nil
+	}
+}
+
+// ProgressTracker allows to track the progress of downloads.
+type ProgressTracker interface {
+	// TrackProgress should be called when
+	// a new object is being downloaded.
+	// src is the location the file is
+	// downloaded from.
+	// currentSize is the current size of
+	// the file in case it is a partial
+	// download.
+	// totalSize is the total size in bytes,
+	// size can be zero if the file size
+	// is not known.
+	// stream is the file being downloaded, every
+	// written byte will add up to processed size.
+	//
+	// TrackProgress returns a ReadCloser that wraps the
+	// download in progress ( stream ).
+	// When the download is finished, body shall be closed.
+	TrackProgress(src string, currentSize, totalSize int64, stream io.ReadCloser) (body io.ReadCloser)
+}

vendor/github.com/hashicorp/go-getter/common.go

@@ -0,0 +1,14 @@
+package getter
+
+import (
+	"io/ioutil"
+)
+
+func tmpFile(dir, pattern string) (string, error) {
+	f, err := ioutil.TempFile(dir, pattern)
+	if err != nil {
+		return "", err
+	}
+	f.Close()
+	return f.Name(), nil
+}

vendor/github.com/hashicorp/go-getter/copy_dir.go

@@ -1,7 +1,7 @@
 package getter
 
 import (
-	"io"
+	"context"
 	"os"
 	"path/filepath"
 	"strings"
@@ -11,7 +11,7 @@ import (
 // should already exist.
 //
 // If ignoreDot is set to true, then dot-prefixed files/folders are ignored.
-func copyDir(dst string, src string, ignoreDot bool) error {
+func copyDir(ctx context.Context, dst string, src string, ignoreDot bool) error {
 	src, err := filepath.EvalSymlinks(src)
 	if err != nil {
 		return err
@@ -66,7 +66,7 @@ func copyDir(dst string, src string, ignoreDot bool) error {
 		}
 		defer dstF.Close()
 
-		if _, err := io.Copy(dstF, srcF); err != nil {
+		if _, err := Copy(ctx, dstF, srcF); err != nil {
 			return err
 		}
 

vendor/github.com/hashicorp/go-getter/decompress_testing.go

@@ -18,16 +18,18 @@ import (
 
 // TestDecompressCase is a single test case for testing decompressors
 type TestDecompressCase struct {
-	Input   string   // Input is the complete path to the input file
-	Dir     bool     // Dir is whether or not we're testing directory mode
-	Err     bool     // Err is whether we expect an error or not
-	DirList []string // DirList is the list of files for Dir mode
-	FileMD5 string   // FileMD5 is the expected MD5 for a single file
-	Mtime *time.Time // Mtime is the optionally expected mtime for a single file (or all files if in Dir mode)
+	Input   string     // Input is the complete path to the input file
+	Dir     bool       // Dir is whether or not we're testing directory mode
+	Err     bool       // Err is whether we expect an error or not
+	DirList []string   // DirList is the list of files for Dir mode
+	FileMD5 string     // FileMD5 is the expected MD5 for a single file
+	Mtime   *time.Time // Mtime is the optionally expected mtime for a single file (or all files if in Dir mode)
 }
 
 // TestDecompressor is a helper function for testing generic decompressors.
 func TestDecompressor(t testing.T, d Decompressor, cases []TestDecompressCase) {
+	t.Helper()
+
 	for _, tc := range cases {
 		t.Logf("Testing: %s", tc.Input)
 

vendor/github.com/hashicorp/go-getter/detect.go

@@ -23,6 +23,7 @@ var Detectors []Detector
 func init() {
 	Detectors = []Detector{
 		new(GitHubDetector),
+		new(GitDetector),
 		new(BitBucketDetector),
 		new(S3Detector),
 		new(FileDetector),

vendor/github.com/hashicorp/go-getter/detect_git.go

@@ -0,0 +1,26 @@
+package getter
+
+// GitDetector implements Detector to detect Git SSH URLs such as
+// git@host.com:dir1/dir2 and converts them to proper URLs.
+type GitDetector struct{}
+
+func (d *GitDetector) Detect(src, _ string) (string, bool, error) {
+	if len(src) == 0 {
+		return "", false, nil
+	}
+
+	u, err := detectSSH(src)
+	if err != nil {
+		return "", true, err
+	}
+	if u == nil {
+		return "", false, nil
+	}
+
+	// We require the username to be "git" to assume that this is a Git URL
+	if u.User.Username() != "git" {
+		return "", false, nil
+	}
+
+	return "git::" + u.String(), true, nil
+}

vendor/github.com/hashicorp/go-getter/detect_github.go

@@ -17,8 +17,6 @@ func (d *GitHubDetector) Detect(src, _ string) (string, bool, error) {
 
 	if strings.HasPrefix(src, "github.com/") {
 		return d.detectHTTP(src)
-	} else if strings.HasPrefix(src, "git@github.com:") {
-		return d.detectSSH(src)
 	}
 
 	return "", false, nil
@@ -47,27 +45,3 @@ func (d *GitHubDetector) detectHTTP(src string) (string, bool, error) {
 
 	return "git::" + url.String(), true, nil
 }
-
-func (d *GitHubDetector) detectSSH(src string) (string, bool, error) {
-	idx := strings.Index(src, ":")
-	qidx := strings.Index(src, "?")
-	if qidx == -1 {
-		qidx = len(src)
-	}
-
-	var u url.URL
-	u.Scheme = "ssh"
-	u.User = url.User("git")
-	u.Host = "github.com"
-	u.Path = src[idx+1 : qidx]
-	if qidx < len(src) {
-		q, err := url.ParseQuery(src[qidx+1:])
-		if err != nil {
-			return "", true, fmt.Errorf("error parsing GitHub SSH URL: %s", err)
-		}
-
-		u.RawQuery = q.Encode()
-	}
-
-	return "git::" + u.String(), true, nil
-}

vendor/github.com/hashicorp/go-getter/detect_ssh.go

@@ -0,0 +1,49 @@
+package getter
+
+import (
+	"fmt"
+	"net/url"
+	"regexp"
+	"strings"
+)
+
+// Note that we do not have an SSH-getter currently so this file serves
+// only to hold the detectSSH helper that is used by other detectors.
+
+// sshPattern matches SCP-like SSH patterns (user@host:path)
+var sshPattern = regexp.MustCompile("^(?:([^@]+)@)?([^:]+):/?(.+)$")
+
+// detectSSH determines if the src string matches an SSH-like URL and
+// converts it into a net.URL compatible string. This returns nil if the
+// string doesn't match the SSH pattern.
+//
+// This function is tested indirectly via detect_git_test.go
+func detectSSH(src string) (*url.URL, error) {
+	matched := sshPattern.FindStringSubmatch(src)
+	if matched == nil {
+		return nil, nil
+	}
+
+	user := matched[1]
+	host := matched[2]
+	path := matched[3]
+	qidx := strings.Index(path, "?")
+	if qidx == -1 {
+		qidx = len(path)
+	}
+
+	var u url.URL
+	u.Scheme = "ssh"
+	u.User = url.User(user)
+	u.Host = host
+	u.Path = path[0:qidx]
+	if qidx < len(path) {
+		q, err := url.ParseQuery(path[qidx+1:])
+		if err != nil {
+			return nil, fmt.Errorf("error parsing GitHub SSH URL: %s", err)
+		}
+		u.RawQuery = q.Encode()
+	}
+
+	return &u, nil
+}

vendor/github.com/hashicorp/go-getter/get.go

@@ -41,6 +41,11 @@ type Getter interface {
 	// ClientMode returns the mode based on the given URL. This is used to
 	// allow clients to let the getters decide which mode to use.
 	ClientMode(*url.URL) (ClientMode, error)
+
+	// SetClient allows a getter to know it's client
+	// in order to access client's Get functions or
+	// progress tracking.
+	SetClient(*Client)
 }
 
 // Getters is the mapping of scheme to the Getter implementation that will
@@ -74,12 +79,12 @@ func init() {
 //
 // src is a URL, whereas dst is always just a file path to a folder. This
 // folder doesn't need to exist. It will be created if it doesn't exist.
-func Get(dst, src string) error {
+func Get(dst, src string, opts ...ClientOption) error {
 	return (&Client{
 		Src:     src,
 		Dst:     dst,
 		Dir:     true,
-		Getters: Getters,
+		Options: opts,
 	}).Get()
 }
 
@@ -89,23 +94,23 @@ func Get(dst, src string) error {
 // dst must be a directory. If src is a file, it will be downloaded
 // into dst with the basename of the URL. If src is a directory or
 // archive, it will be unpacked directly into dst.
-func GetAny(dst, src string) error {
+func GetAny(dst, src string, opts ...ClientOption) error {
 	return (&Client{
 		Src:     src,
 		Dst:     dst,
 		Mode:    ClientModeAny,
-		Getters: Getters,
+		Options: opts,
 	}).Get()
 }
 
 // GetFile downloads the file specified by src into the path specified by
 // dst.
-func GetFile(dst, src string) error {
+func GetFile(dst, src string, opts ...ClientOption) error {
 	return (&Client{
 		Src:     src,
 		Dst:     dst,
 		Dir:     false,
-		Getters: Getters,
+		Options: opts,
 	}).Get()
 }
 

vendor/github.com/hashicorp/go-getter/get_base.go

@@ -0,0 +1,20 @@
+package getter
+
+import "context"
+
+// getter is our base getter; it regroups
+// fields all getters have in common.
+type getter struct {
+	client *Client
+}
+
+func (g *getter) SetClient(c *Client) { g.client = c }
+
+// Context tries to returns the Contex from the getter's
+// client. otherwise context.Background() is returned.
+func (g *getter) Context() context.Context {
+	if g == nil || g.client == nil {
+		return context.Background()
+	}
+	return g.client.Ctx
+}

vendor/github.com/hashicorp/go-getter/get_file.go

@@ -8,7 +8,11 @@ import (
 // FileGetter is a Getter implementation that will download a module from
 // a file scheme.
 type FileGetter struct {
-	// Copy, if set to true, will copy data instead of using a symlink
+	getter
+
+	// Copy, if set to true, will copy data instead of using a symlink. If
+	// false, attempts to symlink to speed up the operation and to lower the
+	// disk space usage. If the symlink fails, may attempt to copy on windows.
 	Copy bool
 }
 

vendor/github.com/hashicorp/go-getter/get_file_copy.go

@@ -0,0 +1,29 @@
+package getter
+
+import (
+	"context"
+	"io"
+)
+
+// readerFunc is syntactic sugar for read interface.
+type readerFunc func(p []byte) (n int, err error)
+
+func (rf readerFunc) Read(p []byte) (n int, err error) { return rf(p) }
+
+// Copy is a io.Copy cancellable by context
+func Copy(ctx context.Context, dst io.Writer, src io.Reader) (int64, error) {
+	// Copy will call the Reader and Writer interface multiple time, in order
+	// to copy by chunk (avoiding loading the whole file in memory).
+	return io.Copy(dst, readerFunc(func(p []byte) (int, error) {
+
+		select {
+		case <-ctx.Done():
+			// context has been canceled
+			// stop process and propagate "context canceled" error
+			return 0, ctx.Err()
+		default:
+			// otherwise just run default io.Reader implementation
+			return src.Read(p)
+		}
+	}))
+}

vendor/github.com/hashicorp/go-getter/get_file_unix.go

@@ -4,7 +4,6 @@ package getter
 
 import (
 	"fmt"
-	"io"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -50,6 +49,7 @@ func (g *FileGetter) Get(dst string, u *url.URL) error {
 }
 
 func (g *FileGetter) GetFile(dst string, u *url.URL) error {
+	ctx := g.Context()
 	path := u.Path
 	if u.RawPath != "" {
 		path = u.RawPath
@@ -98,6 +98,6 @@ func (g *FileGetter) GetFile(dst string, u *url.URL) error {
 	}
 	defer dstF.Close()
 
-	_, err = io.Copy(dstF, srcF)
+	_, err = Copy(ctx, dstF, srcF)
 	return err
 }

vendor/github.com/hashicorp/go-getter/get_file_windows.go

@@ -4,15 +4,16 @@ package getter
 
 import (
 	"fmt"
-	"io"
 	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"syscall"
 )
 
 func (g *FileGetter) Get(dst string, u *url.URL) error {
+	ctx := g.Context()
 	path := u.Path
 	if u.RawPath != "" {
 		path = u.RawPath
@@ -51,7 +52,7 @@ func (g *FileGetter) Get(dst string, u *url.URL) error {
 	sourcePath := toBackslash(path)
 
 	// Use mklink to create a junction point
-	output, err := exec.Command("cmd", "/c", "mklink", "/J", dst, sourcePath).CombinedOutput()
+	output, err := exec.CommandContext(ctx, "cmd", "/c", "mklink", "/J", dst, sourcePath).CombinedOutput()
 	if err != nil {
 		return fmt.Errorf("failed to run mklink %v %v: %v %q", dst, sourcePath, err, output)
 	}
@@ -60,6 +61,7 @@ func (g *FileGetter) Get(dst string, u *url.URL) error {
 }
 
 func (g *FileGetter) GetFile(dst string, u *url.URL) error {
+	ctx := g.Context()
 	path := u.Path
 	if u.RawPath != "" {
 		path = u.RawPath
@@ -92,7 +94,21 @@ func (g *FileGetter) GetFile(dst string, u *url.URL) error {
 
 	// If we're not copying, just symlink and we're done
 	if !g.Copy {
-		return os.Symlink(path, dst)
+		if err = os.Symlink(path, dst); err == nil {
+			return err
+		}
+		lerr, ok := err.(*os.LinkError)
+		if !ok {
+			return err
+		}
+		switch lerr.Err {
+		case syscall.ERROR_PRIVILEGE_NOT_HELD:
+			// no symlink privilege, let's
+			// fallback to a copy to avoid an error.
+			break
+		default:
+			return err
+		}
 	}
 
 	// Copy
@@ -108,7 +124,7 @@ func (g *FileGetter) GetFile(dst string, u *url.URL) error {
 	}
 	defer dstF.Close()
 
-	_, err = io.Copy(dstF, srcF)
+	_, err = Copy(ctx, dstF, srcF)
 	return err
 }
 

vendor/github.com/hashicorp/go-getter/get_git.go

@@ -1,6 +1,7 @@
 package getter
 
 import (
+	"context"
 	"encoding/base64"
 	"fmt"
 	"io/ioutil"
@@ -8,28 +9,34 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
+	"strconv"
 	"strings"
 
 	urlhelper "github.com/hashicorp/go-getter/helper/url"
-	"github.com/hashicorp/go-safetemp"
-	"github.com/hashicorp/go-version"
+	safetemp "github.com/hashicorp/go-safetemp"
+	version "github.com/hashicorp/go-version"
 )
 
 // GitGetter is a Getter implementation that will download a module from
 // a git repository.
-type GitGetter struct{}
+type GitGetter struct {
+	getter
+}
 
 func (g *GitGetter) ClientMode(_ *url.URL) (ClientMode, error) {
 	return ClientModeDir, nil
 }
 
 func (g *GitGetter) Get(dst string, u *url.URL) error {
+	ctx := g.Context()
 	if _, err := exec.LookPath("git"); err != nil {
 		return fmt.Errorf("git must be available and on the PATH")
 	}
 
 	// Extract some query parameters we use
 	var ref, sshKey string
+	var depth int
 	q := u.Query()
 	if len(q) > 0 {
 		ref = q.Get("ref")
@@ -38,6 +45,11 @@ func (g *GitGetter) Get(dst string, u *url.URL) error {
 		sshKey = q.Get("sshkey")
 		q.Del("sshkey")
 
+		if n, err := strconv.Atoi(q.Get("depth")); err == nil {
+			depth = n
+		}
+		q.Del("depth")
+
 		// Copy the URL
 		var newU url.URL = *u
 		u = &newU
@@ -78,15 +90,35 @@ func (g *GitGetter) Get(dst string, u *url.URL) error {
 		}
 	}
 
+	// For SSH-style URLs, if they use the SCP syntax of host:path, then
+	// the URL will be mangled. We detect that here and correct the path.
+	// Example: host:path/bar will turn into host/path/bar
+	if u.Scheme == "ssh" {
+		if idx := strings.Index(u.Host, ":"); idx > -1 {
+			// Copy the URL so we don't modify the input
+			var newU url.URL = *u
+			u = &newU
+
+			// Path includes the part after the ':'.
+			u.Path = u.Host[idx+1:] + u.Path
+			if u.Path[0] != '/' {
+				u.Path = "/" + u.Path
+			}
+
+			// Host trims up to the :
+			u.Host = u.Host[:idx]
+		}
+	}
+
 	// Clone or update the repository
 	_, err := os.Stat(dst)
 	if err != nil && !os.IsNotExist(err) {
 		return err
 	}
 	if err == nil {
-		err = g.update(dst, sshKeyFile, ref)
+		err = g.update(ctx, dst, sshKeyFile, ref, depth)
 	} else {
-		err = g.clone(dst, sshKeyFile, u)
+		err = g.clone(ctx, dst, sshKeyFile, u, depth)
 	}
 	if err != nil {
 		return err
@@ -100,7 +132,7 @@ func (g *GitGetter) Get(dst string, u *url.URL) error {
 	}
 
 	// Lastly, download any/all submodules.
-	return g.fetchSubmodules(dst, sshKeyFile)
+	return g.fetchSubmodules(ctx, dst, sshKeyFile, depth)
 }
 
 // GetFile for Git doesn't support updating at this time. It will download
@@ -138,16 +170,23 @@ func (g *GitGetter) checkout(dst string, ref string) error {
 	return getRunCommand(cmd)
 }
 
-func (g *GitGetter) clone(dst, sshKeyFile string, u *url.URL) error {
-	cmd := exec.Command("git", "clone", u.String(), dst)
+func (g *GitGetter) clone(ctx context.Context, dst, sshKeyFile string, u *url.URL, depth int) error {
+	args := []string{"clone"}
+
+	if depth > 0 {
+		args = append(args, "--depth", strconv.Itoa(depth))
+	}
+
+	args = append(args, u.String(), dst)
+	cmd := exec.CommandContext(ctx, "git", args...)
 	setupGitEnv(cmd, sshKeyFile)
 	return getRunCommand(cmd)
 }
 
-func (g *GitGetter) update(dst, sshKeyFile, ref string) error {
+func (g *GitGetter) update(ctx context.Context, dst, sshKeyFile, ref string, depth int) error {
 	// Determine if we're a branch. If we're NOT a branch, then we just
 	// switch to master prior to checking out
-	cmd := exec.Command("git", "show-ref", "-q", "--verify", "refs/heads/"+ref)
+	cmd := exec.CommandContext(ctx, "git", "show-ref", "-q", "--verify", "refs/heads/"+ref)
 	cmd.Dir = dst
 
 	if getRunCommand(cmd) != nil {
@@ -162,15 +201,24 @@ func (g *GitGetter) update(dst, sshKeyFile, ref string) error {
 		return err
 	}
 
-	cmd = exec.Command("git", "pull", "--ff-only")
+	if depth > 0 {
+		cmd = exec.Command("git", "pull", "--depth", strconv.Itoa(depth), "--ff-only")
+	} else {
+		cmd = exec.Command("git", "pull", "--ff-only")
+	}
+
 	cmd.Dir = dst
 	setupGitEnv(cmd, sshKeyFile)
 	return getRunCommand(cmd)
 }
 
 // fetchSubmodules downloads any configured submodules recursively.
-func (g *GitGetter) fetchSubmodules(dst, sshKeyFile string) error {
-	cmd := exec.Command("git", "submodule", "update", "--init", "--recursive")
+func (g *GitGetter) fetchSubmodules(ctx context.Context, dst, sshKeyFile string, depth int) error {
+	args := []string{"submodule", "update", "--init", "--recursive"}
+	if depth > 0 {
+		args = append(args, "--depth", strconv.Itoa(depth))
+	}
+	cmd := exec.CommandContext(ctx, "git", args...)
 	cmd.Dir = dst
 	setupGitEnv(cmd, sshKeyFile)
 	return getRunCommand(cmd)
@@ -187,7 +235,7 @@ func setupGitEnv(cmd *exec.Cmd, sshKeyFile string) {
 	// with versions of Go < 1.9.
 	env := os.Environ()
 	for i, v := range env {
-		if strings.HasPrefix(v, gitSSHCommand) {
+		if strings.HasPrefix(v, gitSSHCommand) && len(v) > len(gitSSHCommand) {
 			sshCmd = []string{v}
 
 			env[i], env[len(env)-1] = env[len(env)-1], env[i]
@@ -202,6 +250,9 @@ func setupGitEnv(cmd *exec.Cmd, sshKeyFile string) {
 
 	if sshKeyFile != "" {
 		// We have an SSH key temp file configured, tell ssh about this.
+		if runtime.GOOS == "windows" {
+			sshKeyFile = strings.Replace(sshKeyFile, `\`, `/`, -1)
+		}
 		sshCmd = append(sshCmd, "-i", sshKeyFile)
 	}
 
@@ -224,11 +275,20 @@ func checkGitVersion(min string) error {
 	}
 
 	fields := strings.Fields(string(out))
-	if len(fields) != 3 {
+	if len(fields) < 3 {
 		return fmt.Errorf("Unexpected 'git version' output: %q", string(out))
 	}
+	v := fields[2]
+	if runtime.GOOS == "windows" && strings.Contains(v, ".windows.") {
+		// on windows, git version will return for example:
+		// git version 2.20.1.windows.1
+		// Which does not follow the semantic versionning specs
+		// https://semver.org. We remove that part in order for
+		// go-version to not error.
+		v = v[:strings.Index(v, ".windows.")]
+	}
 
-	have, err := version.NewVersion(fields[2])
+	have, err := version.NewVersion(v)
 	if err != nil {
 		return err
 	}

vendor/github.com/hashicorp/go-getter/get_hg.go

@@ -1,6 +1,7 @@
 package getter
 
 import (
+	"context"
 	"fmt"
 	"net/url"
 	"os"
@@ -9,18 +10,21 @@ import (
 	"runtime"
 
 	urlhelper "github.com/hashicorp/go-getter/helper/url"
-	"github.com/hashicorp/go-safetemp"
+	safetemp "github.com/hashicorp/go-safetemp"
 )
 
 // HgGetter is a Getter implementation that will download a module from
 // a Mercurial repository.
-type HgGetter struct{}
+type HgGetter struct {
+	getter
+}
 
 func (g *HgGetter) ClientMode(_ *url.URL) (ClientMode, error) {
 	return ClientModeDir, nil
 }
 
 func (g *HgGetter) Get(dst string, u *url.URL) error {
+	ctx := g.Context()
 	if _, err := exec.LookPath("hg"); err != nil {
 		return fmt.Errorf("hg must be available and on the PATH")
 	}
@@ -58,7 +62,7 @@ func (g *HgGetter) Get(dst string, u *url.URL) error {
 		return err
 	}
 
-	return g.update(dst, newURL, rev)
+	return g.update(ctx, dst, newURL, rev)
 }
 
 // GetFile for Hg doesn't support updating at this time. It will download
@@ -93,7 +97,7 @@ func (g *HgGetter) GetFile(dst string, u *url.URL) error {
 		return err
 	}
 
-	fg := &FileGetter{Copy: true}
+	fg := &FileGetter{Copy: true, getter: g.getter}
 	return fg.GetFile(dst, u)
 }
 
@@ -108,13 +112,13 @@ func (g *HgGetter) pull(dst string, u *url.URL) error {
 	return getRunCommand(cmd)
 }
 
-func (g *HgGetter) update(dst string, u *url.URL, rev string) error {
+func (g *HgGetter) update(ctx context.Context, dst string, u *url.URL, rev string) error {
 	args := []string{"update"}
 	if rev != "" {
 		args = append(args, rev)
 	}
 
-	cmd := exec.Command("hg", args...)
+	cmd := exec.CommandContext(ctx, "hg", args...)
 	cmd.Dir = dst
 	return getRunCommand(cmd)
 }

vendor/github.com/hashicorp/go-getter/get_http.go

@@ -1,6 +1,7 @@
 package getter
 
 import (
+	"context"
 	"encoding/xml"
 	"fmt"
 	"io"
@@ -8,9 +9,10 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 
-	"github.com/hashicorp/go-safetemp"
+	safetemp "github.com/hashicorp/go-safetemp"
 )
 
 // HttpGetter is a Getter implementation that will download from an HTTP
@@ -18,7 +20,7 @@ import (
 //
 // For file downloads, HTTP is used directly.
 //
-// The protocol for downloading a directory from an HTTP endpoing is as follows:
+// The protocol for downloading a directory from an HTTP endpoint is as follows:
 //
 // An HTTP GET request is made to the URL with the additional GET parameter
 // "terraform-get=1". This lets you handle that scenario specially if you
@@ -34,6 +36,8 @@ import (
 // formed URL. The shorthand syntax of "github.com/foo/bar" or relative
 // paths are not allowed.
 type HttpGetter struct {
+	getter
+
 	// Netrc, if true, will lookup and use auth information found
 	// in the user's netrc file if available.
 	Netrc bool
@@ -41,6 +45,12 @@ type HttpGetter struct {
 	// Client is the http.Client to use for Get requests.
 	// This defaults to a cleanhttp.DefaultClient if left unset.
 	Client *http.Client
+
+	// Header contains optional request header fields that should be included
+	// with every HTTP request. Note that the zero value of this field is nil,
+	// and as such it needs to be initialized before use, via something like
+	// make(http.Header).
+	Header http.Header
 }
 
 func (g *HttpGetter) ClientMode(u *url.URL) (ClientMode, error) {
@@ -51,6 +61,7 @@ func (g *HttpGetter) ClientMode(u *url.URL) (ClientMode, error) {
 }
 
 func (g *HttpGetter) Get(dst string, u *url.URL) error {
+	ctx := g.Context()
 	// Copy the URL so we can modify it
 	var newU url.URL = *u
 	u = &newU
@@ -72,10 +83,17 @@ func (g *HttpGetter) Get(dst string, u *url.URL) error {
 	u.RawQuery = q.Encode()
 
 	// Get the URL
-	resp, err := g.Client.Get(u.String())
+	req, err := http.NewRequest("GET", u.String(), nil)
 	if err != nil {
 		return err
 	}
+
+	req.Header = g.Header
+	resp, err := g.Client.Do(req)
+	if err != nil {
+		return err
+	}
+
 	defer resp.Body.Close()
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
 		return fmt.Errorf("bad response code: %d", resp.StatusCode)
@@ -99,57 +117,107 @@ func (g *HttpGetter) Get(dst string, u *url.URL) error {
 	// into a temporary directory, then copy over the proper subdir.
 	source, subDir := SourceDirSubdir(source)
 	if subDir == "" {
-		return Get(dst, source)
+		var opts []ClientOption
+		if g.client != nil {
+			opts = g.client.Options
+		}
+		return Get(dst, source, opts...)
 	}
 
 	// We have a subdir, time to jump some hoops
-	return g.getSubdir(dst, source, subDir)
+	return g.getSubdir(ctx, dst, source, subDir)
 }
 
-func (g *HttpGetter) GetFile(dst string, u *url.URL) error {
+func (g *HttpGetter) GetFile(dst string, src *url.URL) error {
+	ctx := g.Context()
 	if g.Netrc {
 		// Add auth from netrc if we can
-		if err := addAuthFromNetrc(u); err != nil {
+		if err := addAuthFromNetrc(src); err != nil {
 			return err
 		}
 	}
 
+	// Create all the parent directories if needed
+	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
+		return err
+	}
+
+	f, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE, os.FileMode(0666))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
 	if g.Client == nil {
 		g.Client = httpClient
 	}
 
-	resp, err := g.Client.Get(u.String())
+	var currentFileSize int64
+
+	// We first make a HEAD request so we can check
+	// if the server supports range queries. If the server/URL doesn't
+	// support HEAD requests, we just fall back to GET.
+	req, err := http.NewRequest("HEAD", src.String(), nil)
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
-	if resp.StatusCode != 200 {
+	if g.Header != nil {
+		req.Header = g.Header
+	}
+	headResp, err := g.Client.Do(req)
+	if err == nil && headResp != nil {
+		headResp.Body.Close()
+		if headResp.StatusCode == 200 {
+			// If the HEAD request succeeded, then attempt to set the range
+			// query if we can.
+			if headResp.Header.Get("Accept-Ranges") == "bytes" {
+				if fi, err := f.Stat(); err == nil {
+					if _, err = f.Seek(0, os.SEEK_END); err == nil {
+						req.Header.Set("Range", fmt.Sprintf("bytes=%d-", fi.Size()))
+						currentFileSize = fi.Size()
+						totalFileSize, _ := strconv.ParseInt(headResp.Header.Get("Content-Length"), 10, 64)
+						if currentFileSize >= totalFileSize {
+							// file already present
+							return nil
+						}
+					}
+				}
+			}
+		}
+	}
+	req.Method = "GET"
+
+	resp, err := g.Client.Do(req)
+	if err != nil {
+		return err
+	}
+	switch resp.StatusCode {
+	case http.StatusOK, http.StatusPartialContent:
+		// all good
+	default:
+		resp.Body.Close()
 		return fmt.Errorf("bad response code: %d", resp.StatusCode)
 	}
 
-	// Create all the parent directories
-	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
-		return err
-	}
+	body := resp.Body
 
-	f, err := os.Create(dst)
-	if err != nil {
-		return err
+	if g.client != nil && g.client.ProgressListener != nil {
+		// track download
+		fn := filepath.Base(src.EscapedPath())
+		body = g.client.ProgressListener.TrackProgress(fn, currentFileSize, currentFileSize+resp.ContentLength, resp.Body)
 	}
+	defer body.Close()
 
-	n, err := io.Copy(f, resp.Body)
+	n, err := Copy(ctx, f, body)
 	if err == nil && n < resp.ContentLength {
 		err = io.ErrShortWrite
 	}
-	if err1 := f.Close(); err == nil {
-		err = err1
-	}
 	return err
 }
 
 // getSubdir downloads the source into the destination, but with
 // the proper subdir.
-func (g *HttpGetter) getSubdir(dst, source, subDir string) error {
+func (g *HttpGetter) getSubdir(ctx context.Context, dst, source, subDir string) error {
 	// Create a temporary directory to store the full source. This has to be
 	// a non-existent directory.
 	td, tdcloser, err := safetemp.Dir("", "getter")
@@ -158,8 +226,12 @@ func (g *HttpGetter) getSubdir(dst, source, subDir string) error {
 	}
 	defer tdcloser.Close()
 
+	var opts []ClientOption
+	if g.client != nil {
+		opts = g.client.Options
+	}
 	// Download that into the given directory
-	if err := Get(td, source); err != nil {
+	if err := Get(td, source, opts...); err != nil {
 		return err
 	}
 
@@ -185,7 +257,7 @@ func (g *HttpGetter) getSubdir(dst, source, subDir string) error {
 		return err
 	}
 
-	return copyDir(dst, sourcePath, false)
+	return copyDir(ctx, dst, sourcePath, false)
 }
 
 // parseMeta looks for the first meta tag in the given reader that

vendor/github.com/hashicorp/go-getter/get_mock.go

@@ -6,6 +6,8 @@ import (
 
 // MockGetter is an implementation of Getter that can be used for tests.
 type MockGetter struct {
+	getter
+
 	// Proxy, if set, will be called after recording the calls below.
 	// If it isn't set, then the *Err values will be returned.
 	Proxy Getter

vendor/github.com/hashicorp/go-getter/get_s3.go

@@ -1,8 +1,8 @@
 package getter
 
 import (
+	"context"
 	"fmt"
-	"io"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -18,7 +18,9 @@ import (
 
 // S3Getter is a Getter implementation that will download a module from
 // a S3 bucket.
-type S3Getter struct{}
+type S3Getter struct {
+	getter
+}
 
 func (g *S3Getter) ClientMode(u *url.URL) (ClientMode, error) {
 	// Parse URL
@@ -60,6 +62,8 @@ func (g *S3Getter) ClientMode(u *url.URL) (ClientMode, error) {
 }
 
 func (g *S3Getter) Get(dst string, u *url.URL) error {
+	ctx := g.Context()
+
 	// Parse URL
 	region, bucket, path, _, creds, err := g.parseUrl(u)
 	if err != nil {
@@ -124,7 +128,7 @@ func (g *S3Getter) Get(dst string, u *url.URL) error {
 			}
 			objDst = filepath.Join(dst, objDst)
 
-			if err := g.getObject(client, objDst, bucket, objPath, ""); err != nil {
+			if err := g.getObject(ctx, client, objDst, bucket, objPath, ""); err != nil {
 				return err
 			}
 		}
@@ -134,6 +138,7 @@ func (g *S3Getter) Get(dst string, u *url.URL) error {
 }
 
 func (g *S3Getter) GetFile(dst string, u *url.URL) error {
+	ctx := g.Context()
 	region, bucket, path, version, creds, err := g.parseUrl(u)
 	if err != nil {
 		return err
@@ -142,10 +147,10 @@ func (g *S3Getter) GetFile(dst string, u *url.URL) error {
 	config := g.getAWSConfig(region, u, creds)
 	sess := session.New(config)
 	client := s3.New(sess)
-	return g.getObject(client, dst, bucket, path, version)
+	return g.getObject(ctx, client, dst, bucket, path, version)
 }
 
-func (g *S3Getter) getObject(client *s3.S3, dst, bucket, key, version string) error {
+func (g *S3Getter) getObject(ctx context.Context, client *s3.S3, dst, bucket, key, version string) error {
 	req := &s3.GetObjectInput{
 		Bucket: aws.String(bucket),
 		Key:    aws.String(key),
@@ -170,7 +175,7 @@ func (g *S3Getter) getObject(client *s3.S3, dst, bucket, key, version string) er
 	}
 	defer f.Close()
 
-	_, err = io.Copy(f, resp.Body)
+	_, err = Copy(ctx, f, resp.Body)
 	return err
 }
 

vendor/github.com/hashicorp/go-getter/go.mod

@@ -0,0 +1,24 @@
+module github.com/hashicorp/go-getter
+
+require (
+	github.com/aws/aws-sdk-go v1.15.78
+	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d
+	github.com/cheggaaa/pb v1.0.27
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fatih/color v1.7.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.0
+	github.com/hashicorp/go-safetemp v1.0.0
+	github.com/hashicorp/go-version v1.1.0
+	github.com/mattn/go-colorable v0.0.9 // indirect
+	github.com/mattn/go-isatty v0.0.4 // indirect
+	github.com/mattn/go-runewidth v0.0.4 // indirect
+	github.com/mitchellh/go-homedir v1.0.0
+	github.com/mitchellh/go-testing-interface v1.0.0
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/stretchr/testify v1.2.2 // indirect
+	github.com/ulikunitz/xz v0.5.5
+	golang.org/x/net v0.0.0-20181114220301-adae6a3d119a // indirect
+	golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06 // indirect
+	golang.org/x/text v0.3.0 // indirect
+	gopkg.in/cheggaaa/pb.v1 v1.0.27 // indirect
+)

vendor/github.com/hashicorp/go-getter/go.sum

@@ -0,0 +1,44 @@
+github.com/aws/aws-sdk-go v1.15.78 h1:LaXy6lWR0YK7LKyuU0QWy2ws/LWTPfYV/UgfiBu4tvY=
+github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM=
+github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas=
+github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4=
+github.com/cheggaaa/pb v1.0.27 h1:wIkZHkNfC7R6GI5w7l/PdAdzXzlrbcI3p8OAlnkTsnc=
+github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig=
+github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo=
+github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I=
+github.com/hashicorp/go-version v1.0.0 h1:21MVWPKDphxa7ineQQTrCU5brh7OuVVAzGOCnnCPtE8=
+github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0=
+github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE=
+github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
+github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok=
+github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06 h1:0oC8rFnE+74kEmuHZ46F6KHsMr5Gx2gUQPuNz28iQZM=
+golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/cheggaaa/pb.v1 v1.0.27 h1:kJdccidYzt3CaHD1crCFTS1hxyhSi059NhOFUf03YFo=
+gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=

vendor/github.com/hashicorp/go-getter/source.go

@@ -6,18 +6,31 @@ import (
 	"strings"
 )
 
-// SourceDirSubdir takes a source and returns a tuple of the URL without
-// the subdir and the URL with the subdir.
+// SourceDirSubdir takes a source URL and returns a tuple of the URL without
+// the subdir and the subdir.
+//
+// ex:
+//   dom.com/path/?q=p               => dom.com/path/?q=p, ""
+//   proto://dom.com/path//*?q=p     => proto://dom.com/path?q=p, "*"
+//   proto://dom.com/path//path2?q=p => proto://dom.com/path?q=p, "path2"
+//
 func SourceDirSubdir(src string) (string, string) {
-	// Calcaulate an offset to avoid accidentally marking the scheme
+
+	// URL might contains another url in query parameters
+	stop := len(src)
+	if idx := strings.Index(src, "?"); idx > -1 {
+		stop = idx
+	}
+
+	// Calculate an offset to avoid accidentally marking the scheme
 	// as the dir.
 	var offset int
-	if idx := strings.Index(src, "://"); idx > -1 {
+	if idx := strings.Index(src[:stop], "://"); idx > -1 {
 		offset = idx + 3
 	}
 
 	// First see if we even have an explicit subdir
-	idx := strings.Index(src[offset:], "//")
+	idx := strings.Index(src[offset:stop], "//")
 	if idx == -1 {
 		return src, ""
 	}

vendor/vendor.json

@@ -190,7 +190,7 @@
 		{"path":"github.com/hashicorp/go-discover/provider/softlayer","checksumSHA1":"SIyZ44AHIUTBfI336ACpCeybsLg=","revision":"40ccfdee6c0d7136f98f2b54882b86aaf0250d2f","revisionTime":"2018-05-03T15:30:45Z","tree":true},
 		{"path":"github.com/hashicorp/go-discover/provider/triton","checksumSHA1":"n2iQu2IbTPw2XpWF2CqBrFSMjwI=","revision":"40ccfdee6c0d7136f98f2b54882b86aaf0250d2f","revisionTime":"2018-05-03T15:30:45Z","tree":true},
 		{"path":"github.com/hashicorp/go-envparse","checksumSHA1":"FKmqR4DC3nCXtnT9pe02z5CLNWo=","revision":"310ca1881b22af3522e3a8638c0b426629886196","revisionTime":"2018-01-19T21:58:41Z"},
-		{"path":"github.com/hashicorp/go-getter","checksumSHA1":"uGH6AI982csQJoRPsSooK7FoWqo=","revision":"3f60ec5cfbb2a39731571b9ddae54b303bb0a969","revisionTime":"2018-04-25T22:41:30Z"},
+		{"path":"github.com/hashicorp/go-getter","checksumSHA1":"DPz/YgCQgHt4RkkJhgOHtj1w2JM=","revision":"9363991638334fdee4f649819138250a3b6402c8","revisionTime":"2019-02-18T16:50:04Z","version":"v1.1.0","versionExact":"v1.1.0"},
 		{"path":"github.com/hashicorp/go-getter/helper/url","checksumSHA1":"9J+kDr29yDrwsdu2ULzewmqGjpA=","revision":"b345bfcec894fb7ff3fdf9b21baf2f56ea423d98","revisionTime":"2018-04-10T17:49:45Z"},
 		{"path":"github.com/hashicorp/go-hclog","checksumSHA1":"dOP7kCX3dACHc9mU79826N411QA=","revision":"ff2cf002a8dd750586d91dddd4470c341f981fe1","revisionTime":"2018-07-09T16:53:50Z"},
 		{"path":"github.com/hashicorp/go-immutable-radix","checksumSHA1":"Cas2nprG6pWzf05A2F/OlnjUu2Y=","revision":"8aac2701530899b64bdea735a1de8da899815220","revisionTime":"2017-07-25T22:12:15Z"},