From af2fc0f1bc39b07d1d97a80682ac9a0a555fe6be Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Thu, 22 Jun 2017 17:32:20 -0700 Subject: [PATCH 1/3] Persist Alloc when EvalID changes --- client/alloc_runner.go | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/client/alloc_runner.go b/client/alloc_runner.go index 331ddd752..e4be01c8a 100644 --- a/client/alloc_runner.go +++ b/client/alloc_runner.go @@ -81,11 +81,16 @@ type AllocRunner struct { // stateDB is used to store the alloc runners state stateDB *bolt.DB - // immutablePersisted and allocDirPersisted are used to track whether the - // immutable data and the alloc dir have been persisted. Once persisted we - // can lower write volume by not re-writing these values - immutablePersisted bool - allocDirPersisted bool + // persistedEval is the last persisted evaluation ID. Since evaluation + // IDs change on every allocation update we only need to persist the + // allocation when its eval ID != the last persisted eval ID. + persistedEvalLock sync.Mutex + persistedEval string + + // allocDirPersisted is used to track whether the alloc dir has been + // persisted. Once persisted we can lower write volume by not + // re-writing it + allocDirPersisted bool } // COMPAT: Remove in 0.7.0 @@ -342,8 +347,11 @@ func (r *AllocRunner) saveAllocRunnerState() error { return fmt.Errorf("failed to retrieve allocation bucket: %v", err) } - // Write the immutable data - if !r.immutablePersisted { + // Write the allocation if the eval has changed + r.persistedEvalLock.Lock() + lastPersisted := r.persistedEval + r.persistedEvalLock.Unlock() + if alloc.EvalID != lastPersisted { immutable := &allocRunnerImmutableState{ Alloc: alloc, Version: r.config.Version, @@ -354,7 +362,9 @@ func (r *AllocRunner) saveAllocRunnerState() error { } tx.OnCommit(func() { - r.immutablePersisted = true + r.persistedEvalLock.Lock() + r.persistedEval = alloc.EvalID + r.persistedEvalLock.Unlock() }) } From d359d3b554c27b57f23b47af1169d2af6ca45d8e Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Fri, 23 Jun 2017 10:58:36 -0700 Subject: [PATCH 2/3] Rename immutable -> alloc meh; naming is hard --- client/alloc_runner.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/client/alloc_runner.go b/client/alloc_runner.go index e4be01c8a..f0276b393 100644 --- a/client/alloc_runner.go +++ b/client/alloc_runner.go @@ -30,9 +30,9 @@ const ( var ( // The following are the key paths written to the state database - allocRunnerStateImmutableKey = []byte("immutable") - allocRunnerStateMutableKey = []byte("mutable") - allocRunnerStateAllocDirKey = []byte("alloc-dir") + allocRunnreStateAllocKey = []byte("alloc") + allocRunnerStateMutableKey = []byte("mutable") + allocRunnerStateAllocDirKey = []byte("alloc-dir") ) // AllocStateUpdater is used to update the status of an allocation @@ -116,9 +116,9 @@ type allocRunnerState struct { } `json:"Context,omitempty"` } -// allocRunnerImmutableState is state that only has to be written once as it -// doesn't change over the life-cycle of the alloc_runner. -type allocRunnerImmutableState struct { +// allocRunnerAllocState is state that only has to be written when the alloc +// changes. +type allocRunnerAllocState struct { Version string Alloc *structs.Allocation } @@ -212,10 +212,10 @@ func (r *AllocRunner) RestoreState() error { // Get the state objects var mutable allocRunnerMutableState - var immutable allocRunnerImmutableState + var allocState allocRunnerAllocState var allocDir allocdir.AllocDir - if err := getObject(bkt, allocRunnerStateImmutableKey, &immutable); err != nil { + if err := getObject(bkt, allocRunnreStateAllocKey, &allocState); err != nil { return fmt.Errorf("failed to read alloc runner immutable state: %v", err) } if err := getObject(bkt, allocRunnerStateMutableKey, &mutable); err != nil { @@ -226,7 +226,7 @@ func (r *AllocRunner) RestoreState() error { } // Populate the fields - r.alloc = immutable.Alloc + r.alloc = allocState.Alloc r.allocDir = &allocDir r.allocClientStatus = mutable.AllocClientStatus r.allocClientDescription = mutable.AllocClientDescription @@ -352,12 +352,12 @@ func (r *AllocRunner) saveAllocRunnerState() error { lastPersisted := r.persistedEval r.persistedEvalLock.Unlock() if alloc.EvalID != lastPersisted { - immutable := &allocRunnerImmutableState{ + allocState := &allocRunnerAllocState{ Alloc: alloc, Version: r.config.Version, } - if err := putObject(allocBkt, allocRunnerStateImmutableKey, &immutable); err != nil { + if err := putObject(allocBkt, allocRunnreStateAllocKey, &allocState); err != nil { return fmt.Errorf("failed to write alloc_runner immutable state: %v", err) } From cff854603530ee65974a3b42f97c8367b7c1f3f2 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Fri, 23 Jun 2017 13:01:39 -0700 Subject: [PATCH 3/3] Fix spelling & re-add immutable state struct --- client/alloc_runner.go | 50 +++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/client/alloc_runner.go b/client/alloc_runner.go index f0276b393..ecbccbee6 100644 --- a/client/alloc_runner.go +++ b/client/alloc_runner.go @@ -30,9 +30,10 @@ const ( var ( // The following are the key paths written to the state database - allocRunnreStateAllocKey = []byte("alloc") - allocRunnerStateMutableKey = []byte("mutable") - allocRunnerStateAllocDirKey = []byte("alloc-dir") + allocRunnerStateAllocKey = []byte("alloc") + allocRunnerStateImmutableKey = []byte("immutable") + allocRunnerStateMutableKey = []byte("mutable") + allocRunnerStateAllocDirKey = []byte("alloc-dir") ) // AllocStateUpdater is used to update the status of an allocation @@ -87,10 +88,11 @@ type AllocRunner struct { persistedEvalLock sync.Mutex persistedEval string - // allocDirPersisted is used to track whether the alloc dir has been - // persisted. Once persisted we can lower write volume by not - // re-writing it - allocDirPersisted bool + // immutablePersisted and allocDirPersisted are used to track whether the + // immutable data and the alloc dir have been persisted. Once persisted we + // can lower write volume by not re-writing these values + immutablePersisted bool + allocDirPersisted bool } // COMPAT: Remove in 0.7.0 @@ -119,8 +121,12 @@ type allocRunnerState struct { // allocRunnerAllocState is state that only has to be written when the alloc // changes. type allocRunnerAllocState struct { + Alloc *structs.Allocation +} + +// allocRunnerImmutableState is state that only has to be written once. +type allocRunnerImmutableState struct { Version string - Alloc *structs.Allocation } // allocRunnerMutableState is state that has to be written on each save as it @@ -212,10 +218,14 @@ func (r *AllocRunner) RestoreState() error { // Get the state objects var mutable allocRunnerMutableState + var immutable allocRunnerImmutableState var allocState allocRunnerAllocState var allocDir allocdir.AllocDir - if err := getObject(bkt, allocRunnreStateAllocKey, &allocState); err != nil { + if err := getObject(bkt, allocRunnerStateAllocKey, &allocState); err != nil { + return fmt.Errorf("failed to read alloc runner alloc state: %v", err) + } + if err := getObject(bkt, allocRunnerStateImmutableKey, &immutable); err != nil { return fmt.Errorf("failed to read alloc runner immutable state: %v", err) } if err := getObject(bkt, allocRunnerStateMutableKey, &mutable); err != nil { @@ -353,12 +363,11 @@ func (r *AllocRunner) saveAllocRunnerState() error { r.persistedEvalLock.Unlock() if alloc.EvalID != lastPersisted { allocState := &allocRunnerAllocState{ - Alloc: alloc, - Version: r.config.Version, + Alloc: alloc, } - if err := putObject(allocBkt, allocRunnreStateAllocKey, &allocState); err != nil { - return fmt.Errorf("failed to write alloc_runner immutable state: %v", err) + if err := putObject(allocBkt, allocRunnerStateAllocKey, &allocState); err != nil { + return fmt.Errorf("failed to write alloc_runner alloc state: %v", err) } tx.OnCommit(func() { @@ -368,6 +377,21 @@ func (r *AllocRunner) saveAllocRunnerState() error { }) } + // Write immutable data iff it hasn't been written yet + if !r.immutablePersisted { + immutable := &allocRunnerImmutableState{ + Version: r.config.Version, + } + + if err := putObject(allocBkt, allocRunnerStateImmutableKey, &immutable); err != nil { + return fmt.Errorf("failed to write alloc_runner immutable state: %v", err) + } + + tx.OnCommit(func() { + r.immutablePersisted = true + }) + } + // Write the alloc dir data if it hasn't been written before and it exists. if !r.allocDirPersisted && r.allocDir != nil { if err := putObject(allocBkt, allocRunnerStateAllocDirKey, allocDir); err != nil {