2017-08-03 17:24:27 +00:00
|
|
|
package file
|
2015-03-02 18:48:53 +00:00
|
|
|
|
2015-03-16 03:15:27 +00:00
|
|
|
import (
|
2018-01-19 06:44:44 +00:00
|
|
|
"context"
|
2015-03-16 03:15:27 +00:00
|
|
|
"encoding/json"
|
|
|
|
"fmt"
|
2016-08-31 18:12:28 +00:00
|
|
|
"io"
|
2015-03-16 03:15:27 +00:00
|
|
|
"os"
|
|
|
|
"path/filepath"
|
2016-10-04 13:33:54 +00:00
|
|
|
"strings"
|
2015-03-16 03:15:27 +00:00
|
|
|
"sync"
|
2016-07-06 16:25:40 +00:00
|
|
|
|
2018-04-05 15:49:21 +00:00
|
|
|
"github.com/hashicorp/errwrap"
|
2018-04-03 00:46:59 +00:00
|
|
|
log "github.com/hashicorp/go-hclog"
|
2016-08-19 20:45:17 +00:00
|
|
|
|
2017-05-12 17:52:33 +00:00
|
|
|
"github.com/hashicorp/vault/helper/consts"
|
2016-07-06 16:25:40 +00:00
|
|
|
"github.com/hashicorp/vault/helper/jsonutil"
|
2017-08-03 17:24:27 +00:00
|
|
|
"github.com/hashicorp/vault/physical"
|
2015-03-16 03:15:27 +00:00
|
|
|
)
|
|
|
|
|
2018-01-20 01:44:24 +00:00
|
|
|
// Verify FileBackend satisfies the correct interfaces
|
|
|
|
var _ physical.Backend = (*FileBackend)(nil)
|
|
|
|
var _ physical.Transactional = (*TransactionalFileBackend)(nil)
|
|
|
|
var _ physical.PseudoTransactional = (*FileBackend)(nil)
|
2018-01-19 06:44:44 +00:00
|
|
|
|
2015-03-02 18:48:53 +00:00
|
|
|
// FileBackend is a physical backend that stores data on disk
|
|
|
|
// at a given file path. It can be used for durable single server
|
|
|
|
// situations, or to develop locally where durability is not critical.
|
2015-03-16 03:15:27 +00:00
|
|
|
//
|
|
|
|
// WARNING: the file backend implementation is currently extremely unsafe
|
|
|
|
// and non-performant. It is meant mostly for local testing and development.
|
|
|
|
// It can be improved in the future.
|
2015-03-02 18:48:53 +00:00
|
|
|
type FileBackend struct {
|
2017-02-17 14:15:35 +00:00
|
|
|
sync.RWMutex
|
|
|
|
path string
|
|
|
|
logger log.Logger
|
2017-08-03 17:24:27 +00:00
|
|
|
permitPool *physical.PermitPool
|
2015-03-02 18:48:53 +00:00
|
|
|
}
|
|
|
|
|
2017-02-17 14:15:35 +00:00
|
|
|
type TransactionalFileBackend struct {
|
|
|
|
FileBackend
|
|
|
|
}
|
|
|
|
|
2017-11-30 14:43:07 +00:00
|
|
|
type fileEntry struct {
|
|
|
|
Value []byte
|
|
|
|
}
|
|
|
|
|
2017-08-03 17:24:27 +00:00
|
|
|
// NewFileBackend constructs a FileBackend using the given directory
|
|
|
|
func NewFileBackend(conf map[string]string, logger log.Logger) (physical.Backend, error) {
|
2015-03-16 03:15:27 +00:00
|
|
|
path, ok := conf["path"]
|
|
|
|
if !ok {
|
|
|
|
return nil, fmt.Errorf("'path' must be set")
|
|
|
|
}
|
|
|
|
|
2016-04-26 03:10:32 +00:00
|
|
|
return &FileBackend{
|
2017-02-17 14:15:35 +00:00
|
|
|
path: path,
|
|
|
|
logger: logger,
|
2017-08-03 17:24:27 +00:00
|
|
|
permitPool: physical.NewPermitPool(physical.DefaultParallelOperations),
|
2017-02-17 14:15:35 +00:00
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
2017-08-03 17:24:27 +00:00
|
|
|
func NewTransactionalFileBackend(conf map[string]string, logger log.Logger) (physical.Backend, error) {
|
2017-02-17 14:15:35 +00:00
|
|
|
path, ok := conf["path"]
|
|
|
|
if !ok {
|
|
|
|
return nil, fmt.Errorf("'path' must be set")
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create a pool of size 1 so only one operation runs at a time
|
|
|
|
return &TransactionalFileBackend{
|
|
|
|
FileBackend: FileBackend{
|
|
|
|
path: path,
|
|
|
|
logger: logger,
|
2017-08-03 17:24:27 +00:00
|
|
|
permitPool: physical.NewPermitPool(1),
|
2017-02-17 14:15:35 +00:00
|
|
|
},
|
2016-04-26 03:10:32 +00:00
|
|
|
}, nil
|
2015-03-16 03:15:27 +00:00
|
|
|
}
|
|
|
|
|
2018-01-19 06:44:44 +00:00
|
|
|
func (b *FileBackend) Delete(ctx context.Context, path string) error {
|
2017-02-17 14:15:35 +00:00
|
|
|
b.permitPool.Acquire()
|
|
|
|
defer b.permitPool.Release()
|
|
|
|
|
|
|
|
b.Lock()
|
|
|
|
defer b.Unlock()
|
|
|
|
|
2018-01-19 06:44:44 +00:00
|
|
|
return b.DeleteInternal(ctx, path)
|
2017-02-17 14:15:35 +00:00
|
|
|
}
|
|
|
|
|
2018-01-19 06:44:44 +00:00
|
|
|
func (b *FileBackend) DeleteInternal(ctx context.Context, path string) error {
|
2016-10-05 12:08:00 +00:00
|
|
|
if path == "" {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-05-12 17:52:33 +00:00
|
|
|
if err := b.validatePath(path); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2017-02-17 14:15:35 +00:00
|
|
|
basePath, key := b.expandPath(path)
|
2017-01-25 17:27:18 +00:00
|
|
|
fullPath := filepath.Join(basePath, key)
|
2016-12-22 07:46:23 +00:00
|
|
|
|
2017-01-25 17:27:18 +00:00
|
|
|
err := os.Remove(fullPath)
|
2016-10-04 13:33:54 +00:00
|
|
|
if err != nil && !os.IsNotExist(err) {
|
2018-04-05 15:49:21 +00:00
|
|
|
return errwrap.Wrapf(fmt.Sprintf("failed to remove %q: {{err}}", fullPath), err)
|
2016-08-31 18:12:28 +00:00
|
|
|
}
|
|
|
|
|
2016-10-04 13:33:54 +00:00
|
|
|
err = b.cleanupLogicalPath(path)
|
2015-03-16 03:15:27 +00:00
|
|
|
|
2016-10-04 13:33:54 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2018-03-20 18:54:10 +00:00
|
|
|
// cleanupLogicalPath is used to remove all empty nodes, beginning with deepest
|
2016-10-04 13:33:54 +00:00
|
|
|
// one, aborting on first non-empty one, up to top-level node.
|
|
|
|
func (b *FileBackend) cleanupLogicalPath(path string) error {
|
2016-10-05 12:08:00 +00:00
|
|
|
nodes := strings.Split(path, fmt.Sprintf("%c", os.PathSeparator))
|
2016-10-04 13:33:54 +00:00
|
|
|
for i := len(nodes) - 1; i > 0; i-- {
|
2017-02-17 14:15:35 +00:00
|
|
|
fullPath := filepath.Join(b.path, filepath.Join(nodes[:i]...))
|
2016-08-31 18:12:28 +00:00
|
|
|
|
2016-10-04 13:33:54 +00:00
|
|
|
dir, err := os.Open(fullPath)
|
2016-08-31 18:12:28 +00:00
|
|
|
if err != nil {
|
2017-05-09 13:24:43 +00:00
|
|
|
if dir != nil {
|
|
|
|
dir.Close()
|
|
|
|
}
|
2016-10-04 13:33:54 +00:00
|
|
|
if os.IsNotExist(err) {
|
|
|
|
return nil
|
|
|
|
} else {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
list, err := dir.Readdir(1)
|
|
|
|
dir.Close()
|
|
|
|
if err != nil && err != io.EOF {
|
2016-08-31 18:12:28 +00:00
|
|
|
return err
|
|
|
|
}
|
2016-10-04 13:33:54 +00:00
|
|
|
|
|
|
|
// If we have no entries, it's an empty directory; remove it
|
|
|
|
if err == io.EOF || list == nil || len(list) == 0 {
|
|
|
|
err = os.Remove(fullPath)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
2015-03-16 03:15:27 +00:00
|
|
|
}
|
|
|
|
|
2016-08-31 18:12:28 +00:00
|
|
|
return nil
|
2015-03-16 03:15:27 +00:00
|
|
|
}
|
|
|
|
|
2018-01-19 06:44:44 +00:00
|
|
|
func (b *FileBackend) Get(ctx context.Context, k string) (*physical.Entry, error) {
|
2017-02-17 14:15:35 +00:00
|
|
|
b.permitPool.Acquire()
|
|
|
|
defer b.permitPool.Release()
|
|
|
|
|
|
|
|
b.RLock()
|
|
|
|
defer b.RUnlock()
|
|
|
|
|
2018-01-19 06:44:44 +00:00
|
|
|
return b.GetInternal(ctx, k)
|
2017-02-17 14:15:35 +00:00
|
|
|
}
|
2015-03-16 03:15:27 +00:00
|
|
|
|
2018-01-19 06:44:44 +00:00
|
|
|
func (b *FileBackend) GetInternal(ctx context.Context, k string) (*physical.Entry, error) {
|
2017-05-12 17:52:33 +00:00
|
|
|
if err := b.validatePath(k); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2017-02-17 14:15:35 +00:00
|
|
|
path, key := b.expandPath(k)
|
2017-01-25 17:27:18 +00:00
|
|
|
path = filepath.Join(path, key)
|
2015-03-16 03:15:27 +00:00
|
|
|
|
2017-01-25 17:27:18 +00:00
|
|
|
f, err := os.Open(path)
|
2017-05-09 13:24:43 +00:00
|
|
|
if f != nil {
|
|
|
|
defer f.Close()
|
|
|
|
}
|
2015-03-16 03:15:27 +00:00
|
|
|
if err != nil {
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
return nil, nil
|
|
|
|
}
|
2017-01-25 17:27:18 +00:00
|
|
|
|
2015-03-16 03:15:27 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2017-11-30 14:43:07 +00:00
|
|
|
var entry fileEntry
|
2016-07-06 16:25:40 +00:00
|
|
|
if err := jsonutil.DecodeJSONFromReader(f, &entry); err != nil {
|
2015-03-16 03:15:27 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2017-11-30 14:43:07 +00:00
|
|
|
return &physical.Entry{
|
|
|
|
Key: k,
|
|
|
|
Value: entry.Value,
|
|
|
|
}, nil
|
2015-03-16 03:15:27 +00:00
|
|
|
}
|
|
|
|
|
2018-01-19 06:44:44 +00:00
|
|
|
func (b *FileBackend) Put(ctx context.Context, entry *physical.Entry) error {
|
2017-02-17 14:15:35 +00:00
|
|
|
b.permitPool.Acquire()
|
|
|
|
defer b.permitPool.Release()
|
|
|
|
|
|
|
|
b.Lock()
|
|
|
|
defer b.Unlock()
|
|
|
|
|
2018-01-19 06:44:44 +00:00
|
|
|
return b.PutInternal(ctx, entry)
|
2017-02-17 14:15:35 +00:00
|
|
|
}
|
2017-01-13 08:39:33 +00:00
|
|
|
|
2018-01-19 06:44:44 +00:00
|
|
|
func (b *FileBackend) PutInternal(ctx context.Context, entry *physical.Entry) error {
|
2017-05-12 17:52:33 +00:00
|
|
|
if err := b.validatePath(entry.Key); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2017-02-17 14:15:35 +00:00
|
|
|
path, key := b.expandPath(entry.Key)
|
2015-03-16 03:15:27 +00:00
|
|
|
|
|
|
|
// Make the parent tree
|
2017-10-12 18:24:30 +00:00
|
|
|
if err := os.MkdirAll(path, 0700); err != nil {
|
2017-01-25 17:27:18 +00:00
|
|
|
return err
|
2015-03-16 03:15:27 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// JSON encode the entry and write it
|
2015-04-29 18:31:59 +00:00
|
|
|
f, err := os.OpenFile(
|
2017-01-25 17:27:18 +00:00
|
|
|
filepath.Join(path, key),
|
2015-04-29 18:31:59 +00:00
|
|
|
os.O_CREATE|os.O_TRUNC|os.O_WRONLY,
|
|
|
|
0600)
|
2017-05-09 13:24:43 +00:00
|
|
|
if f != nil {
|
|
|
|
defer f.Close()
|
|
|
|
}
|
2015-03-16 03:15:27 +00:00
|
|
|
if err != nil {
|
2017-01-25 17:27:18 +00:00
|
|
|
return err
|
2015-03-16 03:15:27 +00:00
|
|
|
}
|
|
|
|
enc := json.NewEncoder(f)
|
2017-11-30 14:43:07 +00:00
|
|
|
return enc.Encode(&fileEntry{
|
|
|
|
Value: entry.Value,
|
|
|
|
})
|
2015-03-16 03:15:27 +00:00
|
|
|
}
|
|
|
|
|
2018-01-19 06:44:44 +00:00
|
|
|
func (b *FileBackend) List(ctx context.Context, prefix string) ([]string, error) {
|
2017-02-17 14:15:35 +00:00
|
|
|
b.permitPool.Acquire()
|
|
|
|
defer b.permitPool.Release()
|
2015-03-16 03:15:27 +00:00
|
|
|
|
2017-02-17 14:15:35 +00:00
|
|
|
b.RLock()
|
|
|
|
defer b.RUnlock()
|
|
|
|
|
|
|
|
return b.ListInternal(prefix)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (b *FileBackend) ListInternal(prefix string) ([]string, error) {
|
2017-05-12 17:52:33 +00:00
|
|
|
if err := b.validatePath(prefix); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2017-02-17 14:15:35 +00:00
|
|
|
path := b.path
|
2015-03-16 03:15:27 +00:00
|
|
|
if prefix != "" {
|
|
|
|
path = filepath.Join(path, prefix)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Read the directory contents
|
|
|
|
f, err := os.Open(path)
|
2017-05-09 13:24:43 +00:00
|
|
|
if f != nil {
|
|
|
|
defer f.Close()
|
|
|
|
}
|
2015-03-16 03:15:27 +00:00
|
|
|
if err != nil {
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
names, err := f.Readdirnames(-1)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
for i, name := range names {
|
2017-10-19 16:13:43 +00:00
|
|
|
fi, err := os.Stat(filepath.Join(path, name))
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if fi.IsDir() {
|
2015-03-16 03:15:27 +00:00
|
|
|
names[i] = name + "/"
|
2017-10-19 16:13:43 +00:00
|
|
|
} else {
|
|
|
|
if name[0] == '_' {
|
|
|
|
names[i] = name[1:]
|
|
|
|
}
|
2015-03-16 03:15:27 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return names, nil
|
|
|
|
}
|
|
|
|
|
2017-02-17 14:15:35 +00:00
|
|
|
func (b *FileBackend) expandPath(k string) (string, string) {
|
|
|
|
path := filepath.Join(b.path, k)
|
2017-01-25 17:27:18 +00:00
|
|
|
key := filepath.Base(path)
|
|
|
|
path = filepath.Dir(path)
|
|
|
|
return path, "_" + key
|
2015-03-02 18:48:53 +00:00
|
|
|
}
|
2017-02-17 14:15:35 +00:00
|
|
|
|
2017-05-12 17:52:33 +00:00
|
|
|
func (b *FileBackend) validatePath(path string) error {
|
|
|
|
switch {
|
|
|
|
case strings.Contains(path, ".."):
|
|
|
|
return consts.ErrPathContainsParentReferences
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-01-19 06:44:44 +00:00
|
|
|
func (b *TransactionalFileBackend) Transaction(ctx context.Context, txns []*physical.TxnEntry) error {
|
2017-02-17 14:15:35 +00:00
|
|
|
b.permitPool.Acquire()
|
|
|
|
defer b.permitPool.Release()
|
|
|
|
|
|
|
|
b.Lock()
|
|
|
|
defer b.Unlock()
|
|
|
|
|
2018-01-19 06:44:44 +00:00
|
|
|
return physical.GenericTransactionHandler(ctx, b, txns)
|
2017-02-17 14:15:35 +00:00
|
|
|
}
|