open-vault/physical/file/file.go

package file

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	"github.com/hashicorp/errwrap"
	log "github.com/hashicorp/go-hclog"

	"github.com/hashicorp/vault/sdk/helper/consts"
	"github.com/hashicorp/vault/sdk/helper/jsonutil"
	"github.com/hashicorp/vault/sdk/physical"
)

// Verify FileBackend satisfies the correct interfaces
var _ physical.Backend = (*FileBackend)(nil)
var _ physical.Transactional = (*TransactionalFileBackend)(nil)
var _ physical.PseudoTransactional = (*FileBackend)(nil)

// FileBackend is a physical backend that stores data on disk
// at a given file path. It can be used for durable single server
// situations, or to develop locally where durability is not critical.
//
// WARNING: the file backend implementation is currently extremely unsafe
// and non-performant. It is meant mostly for local testing and development.
// It can be improved in the future.
type FileBackend struct {
	sync.RWMutex
	path       string
	logger     log.Logger
	permitPool *physical.PermitPool
}

type TransactionalFileBackend struct {
	FileBackend
}

type fileEntry struct {
	Value []byte
}

// NewFileBackend constructs a FileBackend using the given directory
func NewFileBackend(conf map[string]string, logger log.Logger) (physical.Backend, error) {
	path, ok := conf["path"]
	if !ok {
		return nil, fmt.Errorf("'path' must be set")
	}

	return &FileBackend{
		path:       path,
		logger:     logger,
		permitPool: physical.NewPermitPool(physical.DefaultParallelOperations),
	}, nil
}

func NewTransactionalFileBackend(conf map[string]string, logger log.Logger) (physical.Backend, error) {
	path, ok := conf["path"]
	if !ok {
		return nil, fmt.Errorf("'path' must be set")
	}

	// Create a pool of size 1 so only one operation runs at a time
	return &TransactionalFileBackend{
		FileBackend: FileBackend{
			path:       path,
			logger:     logger,
			permitPool: physical.NewPermitPool(1),
		},
	}, nil
}

func (b *FileBackend) Delete(ctx context.Context, path string) error {
	b.permitPool.Acquire()
	defer b.permitPool.Release()

	b.Lock()
	defer b.Unlock()

	return b.DeleteInternal(ctx, path)
}

func (b *FileBackend) DeleteInternal(ctx context.Context, path string) error {
	if path == "" {
		return nil
	}

	if err := b.validatePath(path); err != nil {
		return err
	}

	basePath, key := b.expandPath(path)
	fullPath := filepath.Join(basePath, key)

	select {
	case <-ctx.Done():
		return ctx.Err()
	default:
	}

	err := os.Remove(fullPath)
	if err != nil && !os.IsNotExist(err) {
		return errwrap.Wrapf(fmt.Sprintf("failed to remove %q: {{err}}", fullPath), err)
	}

	err = b.cleanupLogicalPath(path)

	return err
}

// cleanupLogicalPath is used to remove all empty nodes, beginning with deepest
// one, aborting on first non-empty one, up to top-level node.
func (b *FileBackend) cleanupLogicalPath(path string) error {
	nodes := strings.Split(path, fmt.Sprintf("%c", os.PathSeparator))
	for i := len(nodes) - 1; i > 0; i-- {
		fullPath := filepath.Join(b.path, filepath.Join(nodes[:i]...))

		dir, err := os.Open(fullPath)
		if err != nil {
			if dir != nil {
				dir.Close()
			}
			if os.IsNotExist(err) {
				return nil
			} else {
				return err
			}
		}

		list, err := dir.Readdir(1)
		dir.Close()
		if err != nil && err != io.EOF {
			return err
		}

		// If we have no entries, it's an empty directory; remove it
		if err == io.EOF || list == nil || len(list) == 0 {
			err = os.Remove(fullPath)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func (b *FileBackend) Get(ctx context.Context, k string) (*physical.Entry, error) {
	b.permitPool.Acquire()
	defer b.permitPool.Release()

	b.RLock()
	defer b.RUnlock()

	return b.GetInternal(ctx, k)
}

func (b *FileBackend) GetInternal(ctx context.Context, k string) (*physical.Entry, error) {
	if err := b.validatePath(k); err != nil {
		return nil, err
	}

	path, key := b.expandPath(k)
	path = filepath.Join(path, key)

	// If we stat it and it exists but is size zero, it may be left from some
	// previous FS error like out-of-space. No Vault entry will ever be zero
	// length, so simply remove it and return nil.
	fi, err := os.Stat(path)
	if err == nil {
		if fi.Size() == 0 {
			// Best effort, ignore errors
			os.Remove(path)
			return nil, nil
		}
	}

	f, err := os.Open(path)
	if f != nil {
		defer f.Close()
	}
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}

		return nil, err
	}

	var entry fileEntry
	if err := jsonutil.DecodeJSONFromReader(f, &entry); err != nil {
		return nil, err
	}

	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	default:
	}

	return &physical.Entry{
		Key:   k,
		Value: entry.Value,
	}, nil
}

func (b *FileBackend) Put(ctx context.Context, entry *physical.Entry) error {
	b.permitPool.Acquire()
	defer b.permitPool.Release()

	b.Lock()
	defer b.Unlock()

	return b.PutInternal(ctx, entry)
}

func (b *FileBackend) PutInternal(ctx context.Context, entry *physical.Entry) error {
	if err := b.validatePath(entry.Key); err != nil {
		return err
	}

	path, key := b.expandPath(entry.Key)

	// Make the parent tree
	if err := os.MkdirAll(path, 0700); err != nil {
		return err
	}

	// JSON encode the entry and write it
	fullPath := filepath.Join(path, key)
	f, err := os.OpenFile(
		fullPath,
		os.O_CREATE|os.O_TRUNC|os.O_WRONLY,
		0600)
	if err != nil {
		if f != nil {
			f.Close()
		}
		return err
	}
	if f == nil {
		return errors.New("could not successfully get a file handle")
	}

	select {
	case <-ctx.Done():
		return ctx.Err()
	default:
	}

	enc := json.NewEncoder(f)
	encErr := enc.Encode(&fileEntry{
		Value: entry.Value,
	})
	f.Close()
	if encErr == nil {
		return nil
	}

	// Everything below is best-effort and will result in encErr being returned

	// See if we ended up with a zero-byte file and if so delete it, might be a
	// case of disk being full but the file info is in metadata that is
	// reserved.
	fi, err := os.Stat(fullPath)
	if err != nil {
		return encErr
	}
	if fi == nil {
		return encErr
	}
	if fi.Size() == 0 {
		os.Remove(fullPath)
	}
	return encErr
}

func (b *FileBackend) List(ctx context.Context, prefix string) ([]string, error) {
	b.permitPool.Acquire()
	defer b.permitPool.Release()

	b.RLock()
	defer b.RUnlock()

	return b.ListInternal(ctx, prefix)
}

func (b *FileBackend) ListInternal(ctx context.Context, prefix string) ([]string, error) {
	if err := b.validatePath(prefix); err != nil {
		return nil, err
	}

	path := b.path
	if prefix != "" {
		path = filepath.Join(path, prefix)
	}

	// Read the directory contents
	f, err := os.Open(path)
	if f != nil {
		defer f.Close()
	}
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}

		return nil, err
	}

	names, err := f.Readdirnames(-1)
	if err != nil {
		return nil, err
	}

	for i, name := range names {
		fi, err := os.Stat(filepath.Join(path, name))
		if err != nil {
			return nil, err
		}
		if fi.IsDir() {
			names[i] = name + "/"
		} else {
			if name[0] == '_' {
				names[i] = name[1:]
			}
		}
	}

	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	default:
	}

	if len(names) > 0 {
		sort.Strings(names)
	}

	return names, nil
}

func (b *FileBackend) expandPath(k string) (string, string) {
	path := filepath.Join(b.path, k)
	key := filepath.Base(path)
	path = filepath.Dir(path)
	return path, "_" + key
}

func (b *FileBackend) validatePath(path string) error {
	switch {
	case strings.Contains(path, ".."):
		return consts.ErrPathContainsParentReferences
	}

	return nil
}

func (b *TransactionalFileBackend) Transaction(ctx context.Context, txns []*physical.TxnEntry) error {
	b.permitPool.Acquire()
	defer b.permitPool.Release()

	b.Lock()
	defer b.Unlock()

	return physical.GenericTransactionHandler(ctx, b, txns)
}