Update go-getter

Fixes https://github.com/hashicorp/nomad/issues/3154
This commit is contained in:
Alex Dadgar 2017-09-05 13:55:24 -07:00
parent e3973a4603
commit 5e8635ff15
9 changed files with 117 additions and 15 deletions

View File

@ -12,6 +12,7 @@ BUG FIXES:
* api: Don't merge empty update stanza from job into task groups [GH-3139] * api: Don't merge empty update stanza from job into task groups [GH-3139]
* cli: All status commands handle even UUID prefixes with hyphens [GH-3122] * cli: All status commands handle even UUID prefixes with hyphens [GH-3122]
* cli: Fix autocompletion of paths that include directories on zsh [GH-3129] * cli: Fix autocompletion of paths that include directories on zsh [GH-3129]
* cli: Handle reading files that are in a symlinked directory [GH-3164]
* cli: Status command honors exact job match even when it is the prefix of * cli: Status command honors exact job match even when it is the prefix of
another job [GH-3120] another job [GH-3120]
* cli: Fix setting of TLSServerName for node API Client. This fixes an issue of * cli: Fix setting of TLSServerName for node API Client. This fixes an issue of

View File

@ -119,6 +119,37 @@ The protocol-specific options are documented below the URL format
section. But because they are part of the URL, we point it out here so section. But because they are part of the URL, we point it out here so
you know they exist. you know they exist.
### Subdirectories
If you want to download only a specific subdirectory from a downloaded
directory, you can specify a subdirectory after a double-slash `//`.
go-getter will first download the URL specified _before_ the double-slash
(as if you didn't specify a double-slash), but will then copy the
path after the double slash into the target directory.
For example, if you're downloading this GitHub repository, but you only
want to download the `test-fixtures` directory, you can do the following:
```
https://github.com/hashicorp/go-getter.git//test-fixtures
```
If you downloaded this to the `/tmp` directory, then the file
`/tmp/archive.gz` would exist. Notice that this file is in the `test-fixtures`
directory in this repository, but because we specified a subdirectory,
go-getter automatically copied only that directory contents.
Subdirectory paths may contain may also use filesystem glob patterns.
The path must match _exactly one_ entry or go-getter will return an error.
This is useful if you're not sure the exact directory name but it follows
a predictable naming structure.
For example, the following URL would also work:
```
https://github.com/hashicorp/go-getter.git//test-*
```
### Checksumming ### Checksumming
For file downloads of any protocol, go-getter can automatically verify For file downloads of any protocol, go-getter can automatically verify

View File

@ -305,7 +305,13 @@ func (c *Client) Get() error {
return err return err
} }
return copyDir(realDst, filepath.Join(dst, subDir), false) // Process any globs
subDir, err := SubdirGlob(dst, subDir)
if err != nil {
return err
}
return copyDir(realDst, subDir, false)
} }
return nil return nil

View File

@ -27,6 +27,11 @@ func untar(input io.Reader, dst, src string, dir bool) error {
return err return err
} }
if hdr.Typeflag == tar.TypeXGlobalHeader || hdr.Typeflag == tar.TypeXHeader {
// don't unpack extended headers as files
continue
}
path := dst path := dst
if dir { if dir {
path = filepath.Join(path, hdr.Name) path = filepath.Join(path, hdr.Name)
@ -81,3 +86,27 @@ func untar(input io.Reader, dst, src string, dir bool) error {
} }
} }
} }
// tarDecompressor is an implementation of Decompressor that can
// unpack tar files.
type tarDecompressor struct{}
func (d *tarDecompressor) Decompress(dst, src string, dir bool) error {
// If we're going into a directory we should make that first
mkdir := dst
if !dir {
mkdir = filepath.Dir(dst)
}
if err := os.MkdirAll(mkdir, 0755); err != nil {
return err
}
// File first
f, err := os.Open(src)
if err != nil {
return err
}
defer f.Close()
return untar(f, dst, src, dir)
}

View File

@ -72,12 +72,18 @@ func Detect(src string, pwd string, ds []Detector) (string, error) {
subDir = detectSubdir subDir = detectSubdir
} }
} }
if subDir != "" { if subDir != "" {
u, err := url.Parse(result) u, err := url.Parse(result)
if err != nil { if err != nil {
return "", fmt.Errorf("Error parsing URL: %s", err) return "", fmt.Errorf("Error parsing URL: %s", err)
} }
u.Path += "//" + subDir u.Path += "//" + subDir
// a subdir may contain wildcards, but in order to support them we
// have to ensure the path isn't escaped.
u.RawPath = u.Path
result = u.String() result = u.String()
} }

View File

@ -32,7 +32,7 @@ func (d *FileDetector) Detect(src, pwd string) (string, bool, error) {
return "", true, err return "", true, err
} }
if fi.Mode()&os.ModeSymlink != 0 { if fi.Mode()&os.ModeSymlink != 0 {
pwd, err = os.Readlink(pwd) pwd, err = filepath.EvalSymlinks(pwd)
if err != nil { if err != nil {
return "", true, err return "", true, err
} }

View File

@ -98,7 +98,6 @@ func (g *HttpGetter) Get(dst string, u *url.URL) error {
} }
func (g *HttpGetter) GetFile(dst string, u *url.URL) error { func (g *HttpGetter) GetFile(dst string, u *url.URL) error {
if g.Netrc { if g.Netrc {
// Add auth from netrc if we can // Add auth from netrc if we can
if err := addAuthFromNetrc(u); err != nil { if err := addAuthFromNetrc(u); err != nil {
@ -140,13 +139,22 @@ func (g *HttpGetter) getSubdir(dst, source, subDir string) error {
} }
defer os.RemoveAll(td) defer os.RemoveAll(td)
// We have to create a subdirectory that doesn't exist for the file
// getter to work.
td = filepath.Join(td, "data")
// Download that into the given directory // Download that into the given directory
if err := Get(td, source); err != nil { if err := Get(td, source); err != nil {
return err return err
} }
// Process any globbing
sourcePath, err := SubdirGlob(td, subDir)
if err != nil {
return err
}
// Make sure the subdir path actually exists // Make sure the subdir path actually exists
sourcePath := filepath.Join(td, subDir)
if _, err := os.Stat(sourcePath); err != nil { if _, err := os.Stat(sourcePath); err != nil {
return fmt.Errorf( return fmt.Errorf(
"Error downloading %s: %s", source, err) "Error downloading %s: %s", source, err)

View File

@ -1,6 +1,8 @@
package getter package getter
import ( import (
"fmt"
"path/filepath"
"strings" "strings"
) )
@ -34,3 +36,22 @@ func SourceDirSubdir(src string) (string, string) {
return src, subdir return src, subdir
} }
// SubdirGlob returns the actual subdir with globbing processed.
//
// dst should be a destination directory that is already populated (the
// download is complete) and subDir should be the set subDir. If subDir
// is an empty string, this returns an empty string.
//
// The returned path is the full absolute path.
func SubdirGlob(dst, subDir string) (string, error) {
matches, err := filepath.Glob(filepath.Join(dst, subDir))
if err != nil {
return "", err
}
if len(matches) > 1 {
return "", fmt.Errorf("subdir %q matches multiple paths", subDir)
}
return matches[0], nil
}

22
vendor/vendor.json vendored
View File

@ -750,16 +750,16 @@
"revisionTime": "2017-06-02T22:43:19Z" "revisionTime": "2017-06-02T22:43:19Z"
}, },
{ {
"checksumSHA1": "Glhu9BTy826XCLfMw7rexQ5mKeY=", "checksumSHA1": "aYYfKsxwxF3pz3YqKlmDVxGMrnM=",
"path": "github.com/hashicorp/go-getter", "path": "github.com/hashicorp/go-getter",
"revision": "6aae8e4e2dee8131187c6a54b52664796e5a02b0", "revision": "80eb888f740e980f6585273d48582c74b3a34f88",
"revisionTime": "2017-07-13T01:23:01Z" "revisionTime": "2017-09-05T20:52:06Z"
}, },
{ {
"checksumSHA1": "9J+kDr29yDrwsdu2ULzewmqGjpA=", "checksumSHA1": "9J+kDr29yDrwsdu2ULzewmqGjpA=",
"path": "github.com/hashicorp/go-getter/helper/url", "path": "github.com/hashicorp/go-getter/helper/url",
"revision": "2814e6fb2ca5b3bd950c97eff22553ecb3c7f77b", "revision": "80eb888f740e980f6585273d48582c74b3a34f88",
"revisionTime": "2017-07-06T02:51:20Z" "revisionTime": "2017-09-05T20:52:06Z"
}, },
{ {
"checksumSHA1": "miVF4/7JP0lRwZvFJGKwZWk7aAQ=", "checksumSHA1": "miVF4/7JP0lRwZvFJGKwZWk7aAQ=",
@ -1293,17 +1293,17 @@
"revisionTime": "2017-06-05T21:53:11Z" "revisionTime": "2017-06-05T21:53:11Z"
}, },
{ {
"checksumSHA1": "nqWNlnMmVpt628zzvyo6Yv2CX5Q=",
"path": "golang.org/x/crypto/ssh/terminal",
"revision": "eb71ad9bd329b5ac0fd0148dd99bd62e8be8e035",
"revisionTime": "2017-08-07T10:11:13Z"
},
{
"checksumSHA1": "pE9lQ5mMiW10+m6CS9XQDhSACNU=", "checksumSHA1": "pE9lQ5mMiW10+m6CS9XQDhSACNU=",
"path": "golang.org/x/crypto/blake2b", "path": "golang.org/x/crypto/blake2b",
"revision": "eb71ad9bd329b5ac0fd0148dd99bd62e8be8e035", "revision": "eb71ad9bd329b5ac0fd0148dd99bd62e8be8e035",
"revisionTime": "2017-08-07T10:11:13Z" "revisionTime": "2017-08-07T10:11:13Z"
}, },
{
"checksumSHA1": "nqWNlnMmVpt628zzvyo6Yv2CX5Q=",
"path": "golang.org/x/crypto/ssh/terminal",
"revision": "eb71ad9bd329b5ac0fd0148dd99bd62e8be8e035",
"revisionTime": "2017-08-07T10:11:13Z"
},
{ {
"checksumSHA1": "9jjO5GjLa0XF/nfWihF02RoH4qc=", "checksumSHA1": "9jjO5GjLa0XF/nfWihF02RoH4qc=",
"path": "golang.org/x/net/context", "path": "golang.org/x/net/context",