Do some best-effort cleanup in file backend (#4684)

* Do some best-effort cleanup in file backend

If put results in an encoding error and after the file is closed we
detect it's zero bytes, it could be caused by an out of space error on
the disk since file info is often stored in filesystem metadata with
reserved space. This tries to detect that scenario and perform
best-effort cleanup. We only do this on zero length files to ensure that
if an encode fails to write but the system hasn't already performed
truncation, we leave the existing data alone.

Vault should never write a zero-byte file (as opposed to a zero-byte
value in the encoded JSON) so if this case is hit it's always an error.

* Also run a check on Get

physical/file/file.go

@@ -3,6 +3,7 @@ package file
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -162,6 +163,18 @@ func (b *FileBackend) GetInternal(ctx context.Context, k string) (*physical.Entr
 	path, key := b.expandPath(k)
 	path = filepath.Join(path, key)
 
+	// If we stat it and it exists but is size zero, it may be left from some
+	// previous FS error like out-of-space. No Vault entry will ever be zero
+	// length, so simply remove it and return nil.
+	fi, err := os.Stat(path)
+	if err == nil {
+		if fi.Size() == 0 {
+			// Best effort, ignore errors
+			os.Remove(path)
+			return nil, nil
+		}
+	}
+
 	f, err := os.Open(path)
 	if f != nil {
 		defer f.Close()
@@ -208,20 +221,46 @@ func (b *FileBackend) PutInternal(ctx context.Context, entry *physical.Entry) er
 	}
 
 	// JSON encode the entry and write it
+	fullPath := filepath.Join(path, key)
 	f, err := os.OpenFile(
-		filepath.Join(path, key),
+		fullPath,
 		os.O_CREATE|os.O_TRUNC|os.O_WRONLY,
 		0600)
-	if f != nil {
-		defer f.Close()
-	}
 	if err != nil {
+		if f != nil {
+			f.Close()
+		}
 		return err
 	}
+	if f == nil {
+		return errors.New("could not successfully get a file handle")
+	}
+
 	enc := json.NewEncoder(f)
-	return enc.Encode(&fileEntry{
+	encErr := enc.Encode(&fileEntry{
 		Value: entry.Value,
 	})
+	f.Close()
+	if encErr == nil {
+		return nil
+	}
+
+	// Everything below is best-effort and will result in encErr being returned
+
+	// See if we ended up with a zero-byte file and if so delete it, might be a
+	// case of disk being full but the file info is in metadata that is
+	// reserved.
+	fi, err := os.Stat(fullPath)
+	if err != nil {
+		return encErr
+	}
+	if fi == nil {
+		return encErr
+	}
+	if fi.Size() == 0 {
+		os.Remove(fullPath)
+	}
+	return encErr
 }
 
 func (b *FileBackend) List(ctx context.Context, prefix string) ([]string, error) {