diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d4555d1f..adfbb3c1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ FEATURES: IMPROVEMENTS: BUG FIXES: +* agent: Clean up temporary files during disk write errors when persisting services and checks. [GH-3207] ## 0.9.0 (July 20, 2017) diff --git a/agent/agent.go b/agent/agent.go index 8fc32dce2..5fa43c5ab 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1428,7 +1428,7 @@ func (a *Agent) persistService(service *structs.NodeService) error { return err } - return writeFileAtomic(svcPath, encoded) + return writeFileAtomic(svcPath, encoded, a.logger) } // purgeService removes a persisted service definition file from the data dir @@ -1456,7 +1456,7 @@ func (a *Agent) persistCheck(check *structs.HealthCheck, chkType *structs.CheckT return err } - return writeFileAtomic(checkPath, encoded) + return writeFileAtomic(checkPath, encoded, a.logger) } // purgeCheck removes a persisted check definition file from the data dir @@ -1470,7 +1470,7 @@ func (a *Agent) purgeCheck(checkID types.CheckID) error { // writeFileAtomic writes the given contents to a temporary file in the same // directory, does an fsync and then renames the file to its real path -func writeFileAtomic(path string, contents []byte) error { +func writeFileAtomic(path string, contents []byte, logger *log.Logger) error { uuid, err := uuid.GenerateUUID() if err != nil { return err @@ -1485,15 +1485,30 @@ func writeFileAtomic(path string, contents []byte) error { return err } if _, err := fh.Write(contents); err != nil { + logger.Printf("[INFO] Writing to temp file at %v failed, deleting...\n", tempPath) + fh.Close() + os.Remove(tempPath) return err } if err := fh.Sync(); err != nil { + logger.Printf("[INFO] Syncing temp file at %v failed, deleting...\n", tempPath) + fh.Close() + os.Remove(tempPath) return err } if err := fh.Close(); err != nil { + logger.Printf("[INFO] Closing file handle to temp file at %v failed, deleting...\n", tempPath) + fh.Close() + os.Remove(tempPath) return err } - return os.Rename(tempPath, path) + if err := os.Rename(tempPath, path); err != nil { + logger.Printf("[INFO] Renaming temp file at %v failed, deleting...\n", tempPath) + fh.Close() + os.Remove(tempPath) + return err + } + return nil } // AddService is used to add a service entry. @@ -2072,6 +2087,11 @@ func (a *Agent) loadServices(conf *Config) error { continue } + // Skip all partially written temporary files + if strings.HasSuffix(fi.Name(), "tmp") { + a.logger.Printf("[WARN] Ignoring temporary service file %v", fi.Name()) + continue + } // Open the file for reading file := filepath.Join(svcDir, fi.Name()) fh, err := os.Open(file)