From 5e8635ff15f9d831998b82fa330ccf279667e707 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Tue, 5 Sep 2017 13:55:24 -0700 Subject: [PATCH] Update go-getter Fixes https://github.com/hashicorp/nomad/issues/3154 --- CHANGELOG.md | 1 + .../github.com/hashicorp/go-getter/README.md | 31 +++++++++++++++++++ .../github.com/hashicorp/go-getter/client.go | 8 ++++- .../hashicorp/go-getter/decompress_tar.go | 29 +++++++++++++++++ .../github.com/hashicorp/go-getter/detect.go | 6 ++++ .../hashicorp/go-getter/detect_file.go | 2 +- .../hashicorp/go-getter/get_http.go | 12 +++++-- .../github.com/hashicorp/go-getter/source.go | 21 +++++++++++++ vendor/vendor.json | 22 ++++++------- 9 files changed, 117 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 355679dc9..a718c4421 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ BUG FIXES: * api: Don't merge empty update stanza from job into task groups [GH-3139] * cli: All status commands handle even UUID prefixes with hyphens [GH-3122] * cli: Fix autocompletion of paths that include directories on zsh [GH-3129] + * cli: Handle reading files that are in a symlinked directory [GH-3164] * cli: Status command honors exact job match even when it is the prefix of another job [GH-3120] * cli: Fix setting of TLSServerName for node API Client. This fixes an issue of diff --git a/vendor/github.com/hashicorp/go-getter/README.md b/vendor/github.com/hashicorp/go-getter/README.md index 89191d651..80623105f 100644 --- a/vendor/github.com/hashicorp/go-getter/README.md +++ b/vendor/github.com/hashicorp/go-getter/README.md @@ -119,6 +119,37 @@ The protocol-specific options are documented below the URL format section. But because they are part of the URL, we point it out here so you know they exist. +### Subdirectories + +If you want to download only a specific subdirectory from a downloaded +directory, you can specify a subdirectory after a double-slash `//`. +go-getter will first download the URL specified _before_ the double-slash +(as if you didn't specify a double-slash), but will then copy the +path after the double slash into the target directory. + +For example, if you're downloading this GitHub repository, but you only +want to download the `test-fixtures` directory, you can do the following: + +``` +https://github.com/hashicorp/go-getter.git//test-fixtures +``` + +If you downloaded this to the `/tmp` directory, then the file +`/tmp/archive.gz` would exist. Notice that this file is in the `test-fixtures` +directory in this repository, but because we specified a subdirectory, +go-getter automatically copied only that directory contents. + +Subdirectory paths may contain may also use filesystem glob patterns. +The path must match _exactly one_ entry or go-getter will return an error. +This is useful if you're not sure the exact directory name but it follows +a predictable naming structure. + +For example, the following URL would also work: + +``` +https://github.com/hashicorp/go-getter.git//test-* +``` + ### Checksumming For file downloads of any protocol, go-getter can automatically verify diff --git a/vendor/github.com/hashicorp/go-getter/client.go b/vendor/github.com/hashicorp/go-getter/client.go index 876812a0a..b67bb641c 100644 --- a/vendor/github.com/hashicorp/go-getter/client.go +++ b/vendor/github.com/hashicorp/go-getter/client.go @@ -305,7 +305,13 @@ func (c *Client) Get() error { return err } - return copyDir(realDst, filepath.Join(dst, subDir), false) + // Process any globs + subDir, err := SubdirGlob(dst, subDir) + if err != nil { + return err + } + + return copyDir(realDst, subDir, false) } return nil diff --git a/vendor/github.com/hashicorp/go-getter/decompress_tar.go b/vendor/github.com/hashicorp/go-getter/decompress_tar.go index 61f60431d..543c30d21 100644 --- a/vendor/github.com/hashicorp/go-getter/decompress_tar.go +++ b/vendor/github.com/hashicorp/go-getter/decompress_tar.go @@ -27,6 +27,11 @@ func untar(input io.Reader, dst, src string, dir bool) error { return err } + if hdr.Typeflag == tar.TypeXGlobalHeader || hdr.Typeflag == tar.TypeXHeader { + // don't unpack extended headers as files + continue + } + path := dst if dir { path = filepath.Join(path, hdr.Name) @@ -81,3 +86,27 @@ func untar(input io.Reader, dst, src string, dir bool) error { } } } + +// tarDecompressor is an implementation of Decompressor that can +// unpack tar files. +type tarDecompressor struct{} + +func (d *tarDecompressor) Decompress(dst, src string, dir bool) error { + // If we're going into a directory we should make that first + mkdir := dst + if !dir { + mkdir = filepath.Dir(dst) + } + if err := os.MkdirAll(mkdir, 0755); err != nil { + return err + } + + // File first + f, err := os.Open(src) + if err != nil { + return err + } + defer f.Close() + + return untar(f, dst, src, dir) +} diff --git a/vendor/github.com/hashicorp/go-getter/detect.go b/vendor/github.com/hashicorp/go-getter/detect.go index 481b737c6..c3695510b 100644 --- a/vendor/github.com/hashicorp/go-getter/detect.go +++ b/vendor/github.com/hashicorp/go-getter/detect.go @@ -72,12 +72,18 @@ func Detect(src string, pwd string, ds []Detector) (string, error) { subDir = detectSubdir } } + if subDir != "" { u, err := url.Parse(result) if err != nil { return "", fmt.Errorf("Error parsing URL: %s", err) } u.Path += "//" + subDir + + // a subdir may contain wildcards, but in order to support them we + // have to ensure the path isn't escaped. + u.RawPath = u.Path + result = u.String() } diff --git a/vendor/github.com/hashicorp/go-getter/detect_file.go b/vendor/github.com/hashicorp/go-getter/detect_file.go index 756ea43f8..4ef41ea73 100644 --- a/vendor/github.com/hashicorp/go-getter/detect_file.go +++ b/vendor/github.com/hashicorp/go-getter/detect_file.go @@ -32,7 +32,7 @@ func (d *FileDetector) Detect(src, pwd string) (string, bool, error) { return "", true, err } if fi.Mode()&os.ModeSymlink != 0 { - pwd, err = os.Readlink(pwd) + pwd, err = filepath.EvalSymlinks(pwd) if err != nil { return "", true, err } diff --git a/vendor/github.com/hashicorp/go-getter/get_http.go b/vendor/github.com/hashicorp/go-getter/get_http.go index 661d8989e..dbcb3ab1f 100644 --- a/vendor/github.com/hashicorp/go-getter/get_http.go +++ b/vendor/github.com/hashicorp/go-getter/get_http.go @@ -98,7 +98,6 @@ func (g *HttpGetter) Get(dst string, u *url.URL) error { } func (g *HttpGetter) GetFile(dst string, u *url.URL) error { - if g.Netrc { // Add auth from netrc if we can if err := addAuthFromNetrc(u); err != nil { @@ -140,13 +139,22 @@ func (g *HttpGetter) getSubdir(dst, source, subDir string) error { } defer os.RemoveAll(td) + // We have to create a subdirectory that doesn't exist for the file + // getter to work. + td = filepath.Join(td, "data") + // Download that into the given directory if err := Get(td, source); err != nil { return err } + // Process any globbing + sourcePath, err := SubdirGlob(td, subDir) + if err != nil { + return err + } + // Make sure the subdir path actually exists - sourcePath := filepath.Join(td, subDir) if _, err := os.Stat(sourcePath); err != nil { return fmt.Errorf( "Error downloading %s: %s", source, err) diff --git a/vendor/github.com/hashicorp/go-getter/source.go b/vendor/github.com/hashicorp/go-getter/source.go index 4d5ee3cce..47307ead5 100644 --- a/vendor/github.com/hashicorp/go-getter/source.go +++ b/vendor/github.com/hashicorp/go-getter/source.go @@ -1,6 +1,8 @@ package getter import ( + "fmt" + "path/filepath" "strings" ) @@ -34,3 +36,22 @@ func SourceDirSubdir(src string) (string, string) { return src, subdir } + +// SubdirGlob returns the actual subdir with globbing processed. +// +// dst should be a destination directory that is already populated (the +// download is complete) and subDir should be the set subDir. If subDir +// is an empty string, this returns an empty string. +// +// The returned path is the full absolute path. +func SubdirGlob(dst, subDir string) (string, error) { + matches, err := filepath.Glob(filepath.Join(dst, subDir)) + if err != nil { + return "", err + } + if len(matches) > 1 { + return "", fmt.Errorf("subdir %q matches multiple paths", subDir) + } + + return matches[0], nil +} diff --git a/vendor/vendor.json b/vendor/vendor.json index efec0e9d8..0b999780b 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -750,16 +750,16 @@ "revisionTime": "2017-06-02T22:43:19Z" }, { - "checksumSHA1": "Glhu9BTy826XCLfMw7rexQ5mKeY=", + "checksumSHA1": "aYYfKsxwxF3pz3YqKlmDVxGMrnM=", "path": "github.com/hashicorp/go-getter", - "revision": "6aae8e4e2dee8131187c6a54b52664796e5a02b0", - "revisionTime": "2017-07-13T01:23:01Z" + "revision": "80eb888f740e980f6585273d48582c74b3a34f88", + "revisionTime": "2017-09-05T20:52:06Z" }, { "checksumSHA1": "9J+kDr29yDrwsdu2ULzewmqGjpA=", "path": "github.com/hashicorp/go-getter/helper/url", - "revision": "2814e6fb2ca5b3bd950c97eff22553ecb3c7f77b", - "revisionTime": "2017-07-06T02:51:20Z" + "revision": "80eb888f740e980f6585273d48582c74b3a34f88", + "revisionTime": "2017-09-05T20:52:06Z" }, { "checksumSHA1": "miVF4/7JP0lRwZvFJGKwZWk7aAQ=", @@ -1293,17 +1293,17 @@ "revisionTime": "2017-06-05T21:53:11Z" }, { - "checksumSHA1": "nqWNlnMmVpt628zzvyo6Yv2CX5Q=", - "path": "golang.org/x/crypto/ssh/terminal", - "revision": "eb71ad9bd329b5ac0fd0148dd99bd62e8be8e035", - "revisionTime": "2017-08-07T10:11:13Z" - }, - { "checksumSHA1": "pE9lQ5mMiW10+m6CS9XQDhSACNU=", "path": "golang.org/x/crypto/blake2b", "revision": "eb71ad9bd329b5ac0fd0148dd99bd62e8be8e035", "revisionTime": "2017-08-07T10:11:13Z" }, + { + "checksumSHA1": "nqWNlnMmVpt628zzvyo6Yv2CX5Q=", + "path": "golang.org/x/crypto/ssh/terminal", + "revision": "eb71ad9bd329b5ac0fd0148dd99bd62e8be8e035", + "revisionTime": "2017-08-07T10:11:13Z" + }, { "checksumSHA1": "9jjO5GjLa0XF/nfWihF02RoH4qc=", "path": "golang.org/x/net/context",