435c0d9fc8
This PR switches the Nomad repository from using govendor to Go modules for managing dependencies. Aspects of the Nomad workflow remain pretty much the same. The usual Makefile targets should continue to work as they always did. The API submodule simply defers to the parent Nomad version on the repository, keeping the semantics of API versioning that currently exists.
497 lines
16 KiB
Go
497 lines
16 KiB
Go
// Copyright 2017 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
// Package cache implements a build artifact cache.
|
|
//
|
|
// This package is a slightly modified fork of Go's
|
|
// cmd/go/internal/cache package.
|
|
package cache
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"honnef.co/go/tools/internal/renameio"
|
|
)
|
|
|
|
// An ActionID is a cache action key, the hash of a complete description of a
|
|
// repeatable computation (command line, environment variables,
|
|
// input file contents, executable contents).
|
|
type ActionID [HashSize]byte
|
|
|
|
// An OutputID is a cache output key, the hash of an output of a computation.
|
|
type OutputID [HashSize]byte
|
|
|
|
// A Cache is a package cache, backed by a file system directory tree.
|
|
type Cache struct {
|
|
dir string
|
|
now func() time.Time
|
|
}
|
|
|
|
// Open opens and returns the cache in the given directory.
|
|
//
|
|
// It is safe for multiple processes on a single machine to use the
|
|
// same cache directory in a local file system simultaneously.
|
|
// They will coordinate using operating system file locks and may
|
|
// duplicate effort but will not corrupt the cache.
|
|
//
|
|
// However, it is NOT safe for multiple processes on different machines
|
|
// to share a cache directory (for example, if the directory were stored
|
|
// in a network file system). File locking is notoriously unreliable in
|
|
// network file systems and may not suffice to protect the cache.
|
|
//
|
|
func Open(dir string) (*Cache, error) {
|
|
info, err := os.Stat(dir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !info.IsDir() {
|
|
return nil, &os.PathError{Op: "open", Path: dir, Err: fmt.Errorf("not a directory")}
|
|
}
|
|
for i := 0; i < 256; i++ {
|
|
name := filepath.Join(dir, fmt.Sprintf("%02x", i))
|
|
if err := os.MkdirAll(name, 0777); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
c := &Cache{
|
|
dir: dir,
|
|
now: time.Now,
|
|
}
|
|
return c, nil
|
|
}
|
|
|
|
// fileName returns the name of the file corresponding to the given id.
|
|
func (c *Cache) fileName(id [HashSize]byte, key string) string {
|
|
return filepath.Join(c.dir, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%x", id)+"-"+key)
|
|
}
|
|
|
|
var errMissing = errors.New("cache entry not found")
|
|
|
|
const (
|
|
// action entry file is "v1 <hex id> <hex out> <decimal size space-padded to 20 bytes> <unixnano space-padded to 20 bytes>\n"
|
|
hexSize = HashSize * 2
|
|
entrySize = 2 + 1 + hexSize + 1 + hexSize + 1 + 20 + 1 + 20 + 1
|
|
)
|
|
|
|
// verify controls whether to run the cache in verify mode.
|
|
// In verify mode, the cache always returns errMissing from Get
|
|
// but then double-checks in Put that the data being written
|
|
// exactly matches any existing entry. This provides an easy
|
|
// way to detect program behavior that would have been different
|
|
// had the cache entry been returned from Get.
|
|
//
|
|
// verify is enabled by setting the environment variable
|
|
// GODEBUG=gocacheverify=1.
|
|
var verify = false
|
|
|
|
// DebugTest is set when GODEBUG=gocachetest=1 is in the environment.
|
|
var DebugTest = false
|
|
|
|
func init() { initEnv() }
|
|
|
|
func initEnv() {
|
|
verify = false
|
|
debugHash = false
|
|
debug := strings.Split(os.Getenv("GODEBUG"), ",")
|
|
for _, f := range debug {
|
|
if f == "gocacheverify=1" {
|
|
verify = true
|
|
}
|
|
if f == "gocachehash=1" {
|
|
debugHash = true
|
|
}
|
|
if f == "gocachetest=1" {
|
|
DebugTest = true
|
|
}
|
|
}
|
|
}
|
|
|
|
// Get looks up the action ID in the cache,
|
|
// returning the corresponding output ID and file size, if any.
|
|
// Note that finding an output ID does not guarantee that the
|
|
// saved file for that output ID is still available.
|
|
func (c *Cache) Get(id ActionID) (Entry, error) {
|
|
if verify {
|
|
return Entry{}, errMissing
|
|
}
|
|
return c.get(id)
|
|
}
|
|
|
|
type Entry struct {
|
|
OutputID OutputID
|
|
Size int64
|
|
Time time.Time
|
|
}
|
|
|
|
// get is Get but does not respect verify mode, so that Put can use it.
|
|
func (c *Cache) get(id ActionID) (Entry, error) {
|
|
missing := func() (Entry, error) {
|
|
return Entry{}, errMissing
|
|
}
|
|
f, err := os.Open(c.fileName(id, "a"))
|
|
if err != nil {
|
|
return missing()
|
|
}
|
|
defer f.Close()
|
|
entry := make([]byte, entrySize+1) // +1 to detect whether f is too long
|
|
if n, err := io.ReadFull(f, entry); n != entrySize || err != io.ErrUnexpectedEOF {
|
|
return missing()
|
|
}
|
|
if entry[0] != 'v' || entry[1] != '1' || entry[2] != ' ' || entry[3+hexSize] != ' ' || entry[3+hexSize+1+hexSize] != ' ' || entry[3+hexSize+1+hexSize+1+20] != ' ' || entry[entrySize-1] != '\n' {
|
|
return missing()
|
|
}
|
|
eid, entry := entry[3:3+hexSize], entry[3+hexSize:]
|
|
eout, entry := entry[1:1+hexSize], entry[1+hexSize:]
|
|
esize, entry := entry[1:1+20], entry[1+20:]
|
|
//lint:ignore SA4006 See https://github.com/dominikh/go-tools/issues/465
|
|
etime, entry := entry[1:1+20], entry[1+20:]
|
|
var buf [HashSize]byte
|
|
if _, err := hex.Decode(buf[:], eid); err != nil || buf != id {
|
|
return missing()
|
|
}
|
|
if _, err := hex.Decode(buf[:], eout); err != nil {
|
|
return missing()
|
|
}
|
|
i := 0
|
|
for i < len(esize) && esize[i] == ' ' {
|
|
i++
|
|
}
|
|
size, err := strconv.ParseInt(string(esize[i:]), 10, 64)
|
|
if err != nil || size < 0 {
|
|
return missing()
|
|
}
|
|
i = 0
|
|
for i < len(etime) && etime[i] == ' ' {
|
|
i++
|
|
}
|
|
tm, err := strconv.ParseInt(string(etime[i:]), 10, 64)
|
|
if err != nil || tm < 0 {
|
|
return missing()
|
|
}
|
|
|
|
c.used(c.fileName(id, "a"))
|
|
|
|
return Entry{buf, size, time.Unix(0, tm)}, nil
|
|
}
|
|
|
|
// GetFile looks up the action ID in the cache and returns
|
|
// the name of the corresponding data file.
|
|
func (c *Cache) GetFile(id ActionID) (file string, entry Entry, err error) {
|
|
entry, err = c.Get(id)
|
|
if err != nil {
|
|
return "", Entry{}, err
|
|
}
|
|
file = c.OutputFile(entry.OutputID)
|
|
info, err := os.Stat(file)
|
|
if err != nil || info.Size() != entry.Size {
|
|
return "", Entry{}, errMissing
|
|
}
|
|
return file, entry, nil
|
|
}
|
|
|
|
// GetBytes looks up the action ID in the cache and returns
|
|
// the corresponding output bytes.
|
|
// GetBytes should only be used for data that can be expected to fit in memory.
|
|
func (c *Cache) GetBytes(id ActionID) ([]byte, Entry, error) {
|
|
entry, err := c.Get(id)
|
|
if err != nil {
|
|
return nil, entry, err
|
|
}
|
|
data, _ := ioutil.ReadFile(c.OutputFile(entry.OutputID))
|
|
if sha256.Sum256(data) != entry.OutputID {
|
|
return nil, entry, errMissing
|
|
}
|
|
return data, entry, nil
|
|
}
|
|
|
|
// OutputFile returns the name of the cache file storing output with the given OutputID.
|
|
func (c *Cache) OutputFile(out OutputID) string {
|
|
file := c.fileName(out, "d")
|
|
c.used(file)
|
|
return file
|
|
}
|
|
|
|
// Time constants for cache expiration.
|
|
//
|
|
// We set the mtime on a cache file on each use, but at most one per mtimeInterval (1 hour),
|
|
// to avoid causing many unnecessary inode updates. The mtimes therefore
|
|
// roughly reflect "time of last use" but may in fact be older by at most an hour.
|
|
//
|
|
// We scan the cache for entries to delete at most once per trimInterval (1 day).
|
|
//
|
|
// When we do scan the cache, we delete entries that have not been used for
|
|
// at least trimLimit (5 days). Statistics gathered from a month of usage by
|
|
// Go developers found that essentially all reuse of cached entries happened
|
|
// within 5 days of the previous reuse. See golang.org/issue/22990.
|
|
const (
|
|
mtimeInterval = 1 * time.Hour
|
|
trimInterval = 24 * time.Hour
|
|
trimLimit = 5 * 24 * time.Hour
|
|
)
|
|
|
|
// used makes a best-effort attempt to update mtime on file,
|
|
// so that mtime reflects cache access time.
|
|
//
|
|
// Because the reflection only needs to be approximate,
|
|
// and to reduce the amount of disk activity caused by using
|
|
// cache entries, used only updates the mtime if the current
|
|
// mtime is more than an hour old. This heuristic eliminates
|
|
// nearly all of the mtime updates that would otherwise happen,
|
|
// while still keeping the mtimes useful for cache trimming.
|
|
func (c *Cache) used(file string) {
|
|
info, err := os.Stat(file)
|
|
if err == nil && c.now().Sub(info.ModTime()) < mtimeInterval {
|
|
return
|
|
}
|
|
os.Chtimes(file, c.now(), c.now())
|
|
}
|
|
|
|
// Trim removes old cache entries that are likely not to be reused.
|
|
func (c *Cache) Trim() {
|
|
now := c.now()
|
|
|
|
// We maintain in dir/trim.txt the time of the last completed cache trim.
|
|
// If the cache has been trimmed recently enough, do nothing.
|
|
// This is the common case.
|
|
data, _ := renameio.ReadFile(filepath.Join(c.dir, "trim.txt"))
|
|
t, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64)
|
|
if err == nil && now.Sub(time.Unix(t, 0)) < trimInterval {
|
|
return
|
|
}
|
|
|
|
// Trim each of the 256 subdirectories.
|
|
// We subtract an additional mtimeInterval
|
|
// to account for the imprecision of our "last used" mtimes.
|
|
cutoff := now.Add(-trimLimit - mtimeInterval)
|
|
for i := 0; i < 256; i++ {
|
|
subdir := filepath.Join(c.dir, fmt.Sprintf("%02x", i))
|
|
c.trimSubdir(subdir, cutoff)
|
|
}
|
|
|
|
// Ignore errors from here: if we don't write the complete timestamp, the
|
|
// cache will appear older than it is, and we'll trim it again next time.
|
|
renameio.WriteFile(filepath.Join(c.dir, "trim.txt"), []byte(fmt.Sprintf("%d", now.Unix())), 0666)
|
|
}
|
|
|
|
// trimSubdir trims a single cache subdirectory.
|
|
func (c *Cache) trimSubdir(subdir string, cutoff time.Time) {
|
|
// Read all directory entries from subdir before removing
|
|
// any files, in case removing files invalidates the file offset
|
|
// in the directory scan. Also, ignore error from f.Readdirnames,
|
|
// because we don't care about reporting the error and we still
|
|
// want to process any entries found before the error.
|
|
f, err := os.Open(subdir)
|
|
if err != nil {
|
|
return
|
|
}
|
|
names, _ := f.Readdirnames(-1)
|
|
f.Close()
|
|
|
|
for _, name := range names {
|
|
// Remove only cache entries (xxxx-a and xxxx-d).
|
|
if !strings.HasSuffix(name, "-a") && !strings.HasSuffix(name, "-d") {
|
|
continue
|
|
}
|
|
entry := filepath.Join(subdir, name)
|
|
info, err := os.Stat(entry)
|
|
if err == nil && info.ModTime().Before(cutoff) {
|
|
os.Remove(entry)
|
|
}
|
|
}
|
|
}
|
|
|
|
// putIndexEntry adds an entry to the cache recording that executing the action
|
|
// with the given id produces an output with the given output id (hash) and size.
|
|
func (c *Cache) putIndexEntry(id ActionID, out OutputID, size int64, allowVerify bool) error {
|
|
// Note: We expect that for one reason or another it may happen
|
|
// that repeating an action produces a different output hash
|
|
// (for example, if the output contains a time stamp or temp dir name).
|
|
// While not ideal, this is also not a correctness problem, so we
|
|
// don't make a big deal about it. In particular, we leave the action
|
|
// cache entries writable specifically so that they can be overwritten.
|
|
//
|
|
// Setting GODEBUG=gocacheverify=1 does make a big deal:
|
|
// in verify mode we are double-checking that the cache entries
|
|
// are entirely reproducible. As just noted, this may be unrealistic
|
|
// in some cases but the check is also useful for shaking out real bugs.
|
|
entry := fmt.Sprintf("v1 %x %x %20d %20d\n", id, out, size, time.Now().UnixNano())
|
|
|
|
if verify && allowVerify {
|
|
old, err := c.get(id)
|
|
if err == nil && (old.OutputID != out || old.Size != size) {
|
|
// panic to show stack trace, so we can see what code is generating this cache entry.
|
|
msg := fmt.Sprintf("go: internal cache error: cache verify failed: id=%x changed:<<<\n%s\n>>>\nold: %x %d\nnew: %x %d", id, reverseHash(id), out, size, old.OutputID, old.Size)
|
|
panic(msg)
|
|
}
|
|
}
|
|
file := c.fileName(id, "a")
|
|
|
|
// Copy file to cache directory.
|
|
mode := os.O_WRONLY | os.O_CREATE
|
|
f, err := os.OpenFile(file, mode, 0666)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = f.WriteString(entry)
|
|
if err == nil {
|
|
// Truncate the file only *after* writing it.
|
|
// (This should be a no-op, but truncate just in case of previous corruption.)
|
|
//
|
|
// This differs from ioutil.WriteFile, which truncates to 0 *before* writing
|
|
// via os.O_TRUNC. Truncating only after writing ensures that a second write
|
|
// of the same content to the same file is idempotent, and does not — even
|
|
// temporarily! — undo the effect of the first write.
|
|
err = f.Truncate(int64(len(entry)))
|
|
}
|
|
if closeErr := f.Close(); err == nil {
|
|
err = closeErr
|
|
}
|
|
if err != nil {
|
|
// TODO(bcmills): This Remove potentially races with another go command writing to file.
|
|
// Can we eliminate it?
|
|
os.Remove(file)
|
|
return err
|
|
}
|
|
os.Chtimes(file, c.now(), c.now()) // mainly for tests
|
|
|
|
return nil
|
|
}
|
|
|
|
// Put stores the given output in the cache as the output for the action ID.
|
|
// It may read file twice. The content of file must not change between the two passes.
|
|
func (c *Cache) Put(id ActionID, file io.ReadSeeker) (OutputID, int64, error) {
|
|
return c.put(id, file, true)
|
|
}
|
|
|
|
// PutNoVerify is like Put but disables the verify check
|
|
// when GODEBUG=goverifycache=1 is set.
|
|
// It is meant for data that is OK to cache but that we expect to vary slightly from run to run,
|
|
// like test output containing times and the like.
|
|
func (c *Cache) PutNoVerify(id ActionID, file io.ReadSeeker) (OutputID, int64, error) {
|
|
return c.put(id, file, false)
|
|
}
|
|
|
|
func (c *Cache) put(id ActionID, file io.ReadSeeker, allowVerify bool) (OutputID, int64, error) {
|
|
// Compute output ID.
|
|
h := sha256.New()
|
|
if _, err := file.Seek(0, 0); err != nil {
|
|
return OutputID{}, 0, err
|
|
}
|
|
size, err := io.Copy(h, file)
|
|
if err != nil {
|
|
return OutputID{}, 0, err
|
|
}
|
|
var out OutputID
|
|
h.Sum(out[:0])
|
|
|
|
// Copy to cached output file (if not already present).
|
|
if err := c.copyFile(file, out, size); err != nil {
|
|
return out, size, err
|
|
}
|
|
|
|
// Add to cache index.
|
|
return out, size, c.putIndexEntry(id, out, size, allowVerify)
|
|
}
|
|
|
|
// PutBytes stores the given bytes in the cache as the output for the action ID.
|
|
func (c *Cache) PutBytes(id ActionID, data []byte) error {
|
|
_, _, err := c.Put(id, bytes.NewReader(data))
|
|
return err
|
|
}
|
|
|
|
// copyFile copies file into the cache, expecting it to have the given
|
|
// output ID and size, if that file is not present already.
|
|
func (c *Cache) copyFile(file io.ReadSeeker, out OutputID, size int64) error {
|
|
name := c.fileName(out, "d")
|
|
info, err := os.Stat(name)
|
|
if err == nil && info.Size() == size {
|
|
// Check hash.
|
|
if f, err := os.Open(name); err == nil {
|
|
h := sha256.New()
|
|
io.Copy(h, f)
|
|
f.Close()
|
|
var out2 OutputID
|
|
h.Sum(out2[:0])
|
|
if out == out2 {
|
|
return nil
|
|
}
|
|
}
|
|
// Hash did not match. Fall through and rewrite file.
|
|
}
|
|
|
|
// Copy file to cache directory.
|
|
mode := os.O_RDWR | os.O_CREATE
|
|
if err == nil && info.Size() > size { // shouldn't happen but fix in case
|
|
mode |= os.O_TRUNC
|
|
}
|
|
f, err := os.OpenFile(name, mode, 0666)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
if size == 0 {
|
|
// File now exists with correct size.
|
|
// Only one possible zero-length file, so contents are OK too.
|
|
// Early return here makes sure there's a "last byte" for code below.
|
|
return nil
|
|
}
|
|
|
|
// From here on, if any of the I/O writing the file fails,
|
|
// we make a best-effort attempt to truncate the file f
|
|
// before returning, to avoid leaving bad bytes in the file.
|
|
|
|
// Copy file to f, but also into h to double-check hash.
|
|
if _, err := file.Seek(0, 0); err != nil {
|
|
f.Truncate(0)
|
|
return err
|
|
}
|
|
h := sha256.New()
|
|
w := io.MultiWriter(f, h)
|
|
if _, err := io.CopyN(w, file, size-1); err != nil {
|
|
f.Truncate(0)
|
|
return err
|
|
}
|
|
// Check last byte before writing it; writing it will make the size match
|
|
// what other processes expect to find and might cause them to start
|
|
// using the file.
|
|
buf := make([]byte, 1)
|
|
if _, err := file.Read(buf); err != nil {
|
|
f.Truncate(0)
|
|
return err
|
|
}
|
|
h.Write(buf)
|
|
sum := h.Sum(nil)
|
|
if !bytes.Equal(sum, out[:]) {
|
|
f.Truncate(0)
|
|
return fmt.Errorf("file content changed underfoot")
|
|
}
|
|
|
|
// Commit cache file entry.
|
|
if _, err := f.Write(buf); err != nil {
|
|
f.Truncate(0)
|
|
return err
|
|
}
|
|
if err := f.Close(); err != nil {
|
|
// Data might not have been written,
|
|
// but file may look like it is the right size.
|
|
// To be extra careful, remove cached file.
|
|
os.Remove(name)
|
|
return err
|
|
}
|
|
os.Chtimes(name, c.now(), c.now()) // mainly for tests
|
|
|
|
return nil
|
|
}
|