Support Agent Caching for Service Discovery Results (#4541)

* Add cache types for catalog/services and health/services and basic test that caching works

* Support non-blocking cache types with Cache-Control semantics.

* Update API docs to include caching info for every endpoint.

* Comment updates per PR feedback.

* Add note on caching to the 10,000 foot view on the architecture page to make the new data path more clear.

* Document prepared query staleness quirk and force all background requests to AllowStale so we can spread service discovery load across servers.

agent/agent.go

@@ -3199,8 +3199,12 @@ func (a *Agent) ReloadConfig(newCfg *config.RuntimeConfig) error {
 // care should be taken to call this exactly once after the cache
 // field has been initialized.
 func (a *Agent) registerCache() {
+	// Note that you should register the _agent_ as the RPC implementation and not
+	// the a.delegate directly, otherwise tests that rely on overriding RPC
+	// routing via a.registerEndpoint will not work.
+
 	a.cache.RegisterType(cachetype.ConnectCARootName, &cachetype.ConnectCARoot{
-		RPC: a.delegate,
+		RPC: a,
 	}, &cache.RegisterOptions{
 		// Maintain a blocking query, retry dropped connections quickly
 		Refresh:        true,
@@ -3209,7 +3213,7 @@ func (a *Agent) registerCache() {
 	})
 
 	a.cache.RegisterType(cachetype.ConnectCALeafName, &cachetype.ConnectCALeaf{
-		RPC:   a.delegate,
+		RPC:   a,
 		Cache: a.cache,
 	}, &cache.RegisterOptions{
 		// Maintain a blocking query, retry dropped connections quickly
@@ -3219,13 +3223,38 @@ func (a *Agent) registerCache() {
 	})
 
 	a.cache.RegisterType(cachetype.IntentionMatchName, &cachetype.IntentionMatch{
-		RPC: a.delegate,
+		RPC: a,
 	}, &cache.RegisterOptions{
 		// Maintain a blocking query, retry dropped connections quickly
 		Refresh:        true,
 		RefreshTimer:   0 * time.Second,
 		RefreshTimeout: 10 * time.Minute,
 	})
+
+	a.cache.RegisterType(cachetype.CatalogServicesName, &cachetype.CatalogServices{
+		RPC: a,
+	}, &cache.RegisterOptions{
+		// Maintain a blocking query, retry dropped connections quickly
+		Refresh:        true,
+		RefreshTimer:   0 * time.Second,
+		RefreshTimeout: 10 * time.Minute,
+	})
+
+	a.cache.RegisterType(cachetype.HealthServicesName, &cachetype.HealthServices{
+		RPC: a,
+	}, &cache.RegisterOptions{
+		// Maintain a blocking query, retry dropped connections quickly
+		Refresh:        true,
+		RefreshTimer:   0 * time.Second,
+		RefreshTimeout: 10 * time.Minute,
+	})
+
+	a.cache.RegisterType(cachetype.PreparedQueryName, &cachetype.PreparedQuery{
+		RPC: a,
+	}, &cache.RegisterOptions{
+		// Prepared queries don't support blocking
+		Refresh: false,
+	})
 }
 
 // defaultProxyCommand returns the default Connect managed proxy command.

agent/cache-types/catalog_services.go

@@ -0,0 +1,52 @@
+package cachetype
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/consul/agent/cache"
+	"github.com/hashicorp/consul/agent/structs"
+)
+
+// Recommended name for registration.
+const CatalogServicesName = "catalog-services"
+
+// CatalogServices supports fetching discovering service instances via the
+// catalog.
+type CatalogServices struct {
+	RPC RPC
+}
+
+func (c *CatalogServices) Fetch(opts cache.FetchOptions, req cache.Request) (cache.FetchResult, error) {
+	var result cache.FetchResult
+
+	// The request should be a DCSpecificRequest.
+	reqReal, ok := req.(*structs.ServiceSpecificRequest)
+	if !ok {
+		return result, fmt.Errorf(
+			"Internal cache failure: request wrong type: %T", req)
+	}
+
+	// Set the minimum query index to our current index so we block
+	reqReal.QueryOptions.MinQueryIndex = opts.MinIndex
+	reqReal.QueryOptions.MaxQueryTime = opts.Timeout
+
+	// Allways allow stale - there's no point in hitting leader if the request is
+	// going to be served from cache and endup arbitrarily stale anyway. This
+	// allows cached service-discover to automatically read scale across all
+	// servers too.
+	reqReal.AllowStale = true
+
+	// Fetch
+	var reply structs.IndexedServiceNodes
+	if err := c.RPC.RPC("Catalog.ServiceNodes", reqReal, &reply); err != nil {
+		return result, err
+	}
+
+	result.Value = &reply
+	result.Index = reply.QueryMeta.Index
+	return result, nil
+}
+
+func (c *CatalogServices) SupportsBlocking() bool {
+	return true
+}

agent/cache-types/catalog_services_test.go

@@ -0,0 +1,64 @@
+package cachetype
+
+import (
+	"testing"
+	"time"
+
+	"github.com/hashicorp/consul/agent/cache"
+	"github.com/hashicorp/consul/agent/structs"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCatalogServices(t *testing.T) {
+	require := require.New(t)
+	rpc := TestRPC(t)
+	defer rpc.AssertExpectations(t)
+	typ := &CatalogServices{RPC: rpc}
+
+	// Expect the proper RPC call. This also sets the expected value
+	// since that is return-by-pointer in the arguments.
+	var resp *structs.IndexedServiceNodes
+	rpc.On("RPC", "Catalog.ServiceNodes", mock.Anything, mock.Anything).Return(nil).
+		Run(func(args mock.Arguments) {
+			req := args.Get(1).(*structs.ServiceSpecificRequest)
+			require.Equal(uint64(24), req.QueryOptions.MinQueryIndex)
+			require.Equal(1*time.Second, req.QueryOptions.MaxQueryTime)
+			require.Equal("web", req.ServiceName)
+			require.Equal("canary", req.ServiceTag)
+			require.True(req.AllowStale)
+
+			reply := args.Get(2).(*structs.IndexedServiceNodes)
+			reply.QueryMeta.Index = 48
+			resp = reply
+		})
+
+	// Fetch
+	result, err := typ.Fetch(cache.FetchOptions{
+		MinIndex: 24,
+		Timeout:  1 * time.Second,
+	}, &structs.ServiceSpecificRequest{
+		Datacenter:  "dc1",
+		ServiceName: "web",
+		ServiceTag:  "canary",
+	})
+	require.NoError(err)
+	require.Equal(cache.FetchResult{
+		Value: resp,
+		Index: 48,
+	}, result)
+}
+
+func TestCatalogServices_badReqType(t *testing.T) {
+	require := require.New(t)
+	rpc := TestRPC(t)
+	defer rpc.AssertExpectations(t)
+	typ := &CatalogServices{RPC: rpc}
+
+	// Fetch
+	_, err := typ.Fetch(cache.FetchOptions{}, cache.TestRequest(
+		t, cache.RequestInfo{Key: "foo", MinIndex: 64}))
+	require.Error(err)
+	require.Contains(err.Error(), "wrong type")
+
+}

agent/cache-types/connect_ca_leaf.go

@@ -219,6 +219,10 @@ func (c *ConnectCALeaf) waitNewRootCA(datacenter string, ch chan<- error,
 	ch <- nil
 }
 
+func (c *ConnectCALeaf) SupportsBlocking() bool {
+	return true
+}
+
 // ConnectCALeafRequest is the cache.Request implementation for the
 // ConnectCALeaf cache type. This is implemented here and not in structs
 // since this is only used for cache-related requests and not forwarded

agent/cache-types/connect_ca_root.go

@@ -41,3 +41,7 @@ func (c *ConnectCARoot) Fetch(opts cache.FetchOptions, req cache.Request) (cache
 	result.Index = reply.QueryMeta.Index
 	return result, nil
 }
+
+func (c *ConnectCARoot) SupportsBlocking() bool {
+	return true
+}

agent/cache-types/health_services.go

@@ -0,0 +1,52 @@
+package cachetype
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/consul/agent/cache"
+	"github.com/hashicorp/consul/agent/structs"
+)
+
+// Recommended name for registration.
+const HealthServicesName = "health-services"
+
+// HealthServices supports fetching discovering service instances via the
+// catalog.
+type HealthServices struct {
+	RPC RPC
+}
+
+func (c *HealthServices) Fetch(opts cache.FetchOptions, req cache.Request) (cache.FetchResult, error) {
+	var result cache.FetchResult
+
+	// The request should be a DCSpecificRequest.
+	reqReal, ok := req.(*structs.ServiceSpecificRequest)
+	if !ok {
+		return result, fmt.Errorf(
+			"Internal cache failure: request wrong type: %T", req)
+	}
+
+	// Set the minimum query index to our current index so we block
+	reqReal.QueryOptions.MinQueryIndex = opts.MinIndex
+	reqReal.QueryOptions.MaxQueryTime = opts.Timeout
+
+	// Allways allow stale - there's no point in hitting leader if the request is
+	// going to be served from cache and endup arbitrarily stale anyway. This
+	// allows cached service-discover to automatically read scale across all
+	// servers too.
+	reqReal.AllowStale = true
+
+	// Fetch
+	var reply structs.IndexedCheckServiceNodes
+	if err := c.RPC.RPC("Health.ServiceNodes", reqReal, &reply); err != nil {
+		return result, err
+	}
+
+	result.Value = &reply
+	result.Index = reply.QueryMeta.Index
+	return result, nil
+}
+
+func (c *HealthServices) SupportsBlocking() bool {
+	return true
+}

agent/cache-types/health_services_test.go

@@ -0,0 +1,64 @@
+package cachetype
+
+import (
+	"testing"
+	"time"
+
+	"github.com/hashicorp/consul/agent/cache"
+	"github.com/hashicorp/consul/agent/structs"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHealthServices(t *testing.T) {
+	require := require.New(t)
+	rpc := TestRPC(t)
+	defer rpc.AssertExpectations(t)
+	typ := &HealthServices{RPC: rpc}
+
+	// Expect the proper RPC call. This also sets the expected value
+	// since that is return-by-pointer in the arguments.
+	var resp *structs.IndexedCheckServiceNodes
+	rpc.On("RPC", "Health.ServiceNodes", mock.Anything, mock.Anything).Return(nil).
+		Run(func(args mock.Arguments) {
+			req := args.Get(1).(*structs.ServiceSpecificRequest)
+			require.Equal(uint64(24), req.QueryOptions.MinQueryIndex)
+			require.Equal(1*time.Second, req.QueryOptions.MaxQueryTime)
+			require.Equal("web", req.ServiceName)
+			require.Equal("canary", req.ServiceTag)
+			require.True(req.AllowStale)
+
+			reply := args.Get(2).(*structs.IndexedCheckServiceNodes)
+			reply.QueryMeta.Index = 48
+			resp = reply
+		})
+
+	// Fetch
+	result, err := typ.Fetch(cache.FetchOptions{
+		MinIndex: 24,
+		Timeout:  1 * time.Second,
+	}, &structs.ServiceSpecificRequest{
+		Datacenter:  "dc1",
+		ServiceName: "web",
+		ServiceTag:  "canary",
+	})
+	require.NoError(err)
+	require.Equal(cache.FetchResult{
+		Value: resp,
+		Index: 48,
+	}, result)
+}
+
+func TestHealthServices_badReqType(t *testing.T) {
+	require := require.New(t)
+	rpc := TestRPC(t)
+	defer rpc.AssertExpectations(t)
+	typ := &HealthServices{RPC: rpc}
+
+	// Fetch
+	_, err := typ.Fetch(cache.FetchOptions{}, cache.TestRequest(
+		t, cache.RequestInfo{Key: "foo", MinIndex: 64}))
+	require.Error(err)
+	require.Contains(err.Error(), "wrong type")
+
+}

agent/cache-types/intention_match.go

@@ -39,3 +39,7 @@ func (c *IntentionMatch) Fetch(opts cache.FetchOptions, req cache.Request) (cach
 	result.Index = reply.Index
 	return result, nil
 }
+
+func (c *IntentionMatch) SupportsBlocking() bool {
+	return true
+}

agent/cache-types/prepared_query.go

@@ -0,0 +1,50 @@
+package cachetype
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/consul/agent/cache"
+	"github.com/hashicorp/consul/agent/structs"
+)
+
+// Recommended name for registration.
+const PreparedQueryName = "prepared-query"
+
+// PreparedQuery supports fetching discovering service instances via prepared
+// queries.
+type PreparedQuery struct {
+	RPC RPC
+}
+
+func (c *PreparedQuery) Fetch(opts cache.FetchOptions, req cache.Request) (cache.FetchResult, error) {
+	var result cache.FetchResult
+
+	// The request should be a PreparedQueryExecuteRequest.
+	reqReal, ok := req.(*structs.PreparedQueryExecuteRequest)
+	if !ok {
+		return result, fmt.Errorf(
+			"Internal cache failure: request wrong type: %T", req)
+	}
+
+	// Allways allow stale - there's no point in hitting leader if the request is
+	// going to be served from cache and endup arbitrarily stale anyway. This
+	// allows cached service-discover to automatically read scale across all
+	// servers too.
+	reqReal.AllowStale = true
+
+	// Fetch
+	var reply structs.PreparedQueryExecuteResponse
+	if err := c.RPC.RPC("PreparedQuery.Execute", reqReal, &reply); err != nil {
+		return result, err
+	}
+
+	result.Value = &reply
+	result.Index = reply.QueryMeta.Index
+
+	return result, nil
+}
+
+func (c *PreparedQuery) SupportsBlocking() bool {
+	// Prepared queries don't support blocking.
+	return false
+}

agent/cache-types/prepared_query_test.go

@@ -0,0 +1,57 @@
+package cachetype
+
+import (
+	"testing"
+
+	"github.com/hashicorp/consul/agent/cache"
+	"github.com/hashicorp/consul/agent/structs"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPreparedQuery(t *testing.T) {
+	require := require.New(t)
+	rpc := TestRPC(t)
+	defer rpc.AssertExpectations(t)
+	typ := &PreparedQuery{RPC: rpc}
+
+	// Expect the proper RPC call. This also sets the expected value
+	// since that is return-by-pointer in the arguments.
+	var resp *structs.PreparedQueryExecuteResponse
+	rpc.On("RPC", "PreparedQuery.Execute", mock.Anything, mock.Anything).Return(nil).
+		Run(func(args mock.Arguments) {
+			req := args.Get(1).(*structs.PreparedQueryExecuteRequest)
+			require.Equal("geo-db", req.QueryIDOrName)
+			require.Equal(10, req.Limit)
+			require.True(req.AllowStale)
+
+			reply := args.Get(2).(*structs.PreparedQueryExecuteResponse)
+			reply.QueryMeta.Index = 48
+			resp = reply
+		})
+
+	// Fetch
+	result, err := typ.Fetch(cache.FetchOptions{}, &structs.PreparedQueryExecuteRequest{
+		Datacenter:    "dc1",
+		QueryIDOrName: "geo-db",
+		Limit:         10,
+	})
+	require.NoError(err)
+	require.Equal(cache.FetchResult{
+		Value: resp,
+		Index: 48,
+	}, result)
+}
+
+func TestPreparedQuery_badReqType(t *testing.T) {
+	require := require.New(t)
+	rpc := TestRPC(t)
+	defer rpc.AssertExpectations(t)
+	typ := &PreparedQuery{RPC: rpc}
+
+	// Fetch
+	_, err := typ.Fetch(cache.FetchOptions{}, cache.TestRequest(
+		t, cache.RequestInfo{Key: "foo", MinIndex: 64}))
+	require.Error(err)
+	require.Contains(err.Error(), "wrong type")
+}

agent/cache/cache.go

@@ -89,8 +89,24 @@ type typeEntry struct {
 // ResultMeta is returned from Get calls along with the value and can be used
 // to expose information about the cache status for debugging or testing.
 type ResultMeta struct {
-	// Return whether or not the request was a cache hit
+	// Hit indicates whether or not the request was a cache hit
 	Hit bool
+
+	// Age identifies how "stale" the result is. It's semantics differ based on
+	// whether or not the cache type performs background refresh or not as defined
+	// in https://www.consul.io/api/index.html#agent-caching.
+	//
+	// For background refresh types, Age is 0 unless the background blocking query
+	// is currently in a failed state and so not keeping up with the server's
+	// values. If it is non-zero it represents the time since the first failure to
+	// connect during background refresh, and is reset after a background request
+	// does manage to reconnect and either return successfully, or block for at
+	// least the yamux keepalive timeout of 30 seconds (which indicates the
+	// connection is OK but blocked as expected).
+	//
+	// For simple cache types, Age is the time since the result being returned was
+	// fetched from the servers.
+	Age time.Duration
 }
 
 // Options are options for the Cache.
@@ -212,30 +228,76 @@ RETRY_GET:
 	entry, ok := c.entries[key]
 	c.entriesLock.RUnlock()
 
-	// If we have a current value and the index is greater than the
-	// currently stored index then we return that right away. If the
-	// index is zero and we have something in the cache we accept whatever
-	// we have.
-	if ok && entry.Valid {
-		if info.MinIndex == 0 || info.MinIndex < entry.Index {
-			meta := ResultMeta{}
-			if first {
-				metrics.IncrCounter([]string{"consul", "cache", t, "hit"}, 1)
-				meta.Hit = true
-			}
+	// Get the type that we're fetching
+	c.typesLock.RLock()
+	tEntry, ok := c.types[t]
+	c.typesLock.RUnlock()
+	if !ok {
+		// Shouldn't happen given that we successfully fetched this at least
+		// once. But be robust against panics.
+		return nil, ResultMeta{}, fmt.Errorf("unknown type in cache: %s", t)
+	}
 
-			// Touch the expiration and fix the heap.
-			c.entriesLock.Lock()
-			entry.Expiry.Reset()
-			c.entriesExpiryHeap.Fix(entry.Expiry)
-			c.entriesLock.Unlock()
+	// Check if we have a hit
+	cacheHit := ok && entry.Valid
 
-			// We purposely do not return an error here since the cache
-			// only works with fetching values that either have a value
-			// or have an error, but not both. The Error may be non-nil
-			// in the entry because of this to note future fetch errors.
-			return entry.Value, meta, nil
+	supportsBlocking := tEntry.Type.SupportsBlocking()
+
+	// Check index is not specified or lower than value, or the type doesn't
+	// support blocking.
+	if cacheHit && supportsBlocking &&
+		info.MinIndex > 0 && info.MinIndex >= entry.Index {
+		// MinIndex was given and matches or is higher than current value so we
+		// ignore the cache and fallthrough to blocking on a new value below.
+		cacheHit = false
+	}
+
+	// Check MaxAge is not exceeded if this is not a background refreshing type
+	// and MaxAge was specified.
+	if cacheHit && !tEntry.Opts.Refresh && info.MaxAge > 0 &&
+		!entry.FetchedAt.IsZero() && info.MaxAge < time.Since(entry.FetchedAt) {
+		cacheHit = false
+	}
+
+	// Check if we are requested to revalidate. If so the first time round the
+	// loop is not a hit but subsequent ones should be treated normally.
+	if cacheHit && !tEntry.Opts.Refresh && info.MustRevalidate && first {
+		cacheHit = false
+	}
+
+	if cacheHit {
+		meta := ResultMeta{}
+		if first {
+			metrics.IncrCounter([]string{"consul", "cache", t, "hit"}, 1)
+			meta.Hit = true
 		}
+
+		// If refresh is enabled, calculate age based on whether the background
+		// routine is still connected.
+		if tEntry.Opts.Refresh {
+			meta.Age = time.Duration(0)
+			if !entry.RefreshLostContact.IsZero() {
+				meta.Age = time.Since(entry.RefreshLostContact)
+			}
+		} else {
+			// For non-background refresh types, the age is just how long since we
+			// fetched it last.
+			if !entry.FetchedAt.IsZero() {
+				meta.Age = time.Since(entry.FetchedAt)
+			}
+		}
+
+		// Touch the expiration and fix the heap.
+		c.entriesLock.Lock()
+		entry.Expiry.Reset()
+		c.entriesExpiryHeap.Fix(entry.Expiry)
+		c.entriesLock.Unlock()
+
+		// We purposely do not return an error here since the cache
+		// only works with fetching values that either have a value
+		// or have an error, but not both. The Error may be non-nil
+		// in the entry because of this to note future fetch errors.
+		return entry.Value, meta, nil
 	}
 
 	// If this isn't our first time through and our last value has an error,
@@ -342,11 +404,39 @@ func (c *Cache) fetch(t, key string, r Request, allowNew bool, attempt uint) (<-
 
 	// The actual Fetch must be performed in a goroutine.
 	go func() {
+		// If we have background refresh and currently are in "disconnected" state,
+		// waiting for a response might mean we mark our results as stale for up to
+		// 10 minutes (max blocking timeout) after connection is restored. To reduce
+		// that window, we assume that if the fetch takes more than 31 seconds then
+		// they are correctly blocking. We choose 31 seconds because yamux
+		// keepalives are every 30 seconds so the RPC should fail if the packets are
+		// being blackholed for more than 30 seconds.
+		var connectedTimer *time.Timer
+		if tEntry.Opts.Refresh && entry.Index > 0 &&
+			tEntry.Opts.RefreshTimeout > (31*time.Second) {
+			connectedTimer = time.AfterFunc(31*time.Second, func() {
+				c.entriesLock.Lock()
+				defer c.entriesLock.Unlock()
+				entry, ok := c.entries[key]
+				if !ok || entry.RefreshLostContact.IsZero() {
+					return
+				}
+				entry.RefreshLostContact = time.Time{}
+				c.entries[key] = entry
+			})
+		}
+
+		fOpts := FetchOptions{}
+		if tEntry.Type.SupportsBlocking() {
+			fOpts.MinIndex = entry.Index
+			fOpts.Timeout = tEntry.Opts.RefreshTimeout
+		}
+
 		// Start building the new entry by blocking on the fetch.
-		result, err := tEntry.Type.Fetch(FetchOptions{
-			MinIndex: entry.Index,
-			Timeout:  tEntry.Opts.RefreshTimeout,
-		}, r)
+		result, err := tEntry.Type.Fetch(fOpts, r)
+		if connectedTimer != nil {
+			connectedTimer.Stop()
+		}
 
 		// Copy the existing entry to start.
 		newEntry := entry
@@ -355,6 +445,7 @@ func (c *Cache) fetch(t, key string, r Request, allowNew bool, attempt uint) (<-
 			// A new value was given, so we create a brand new entry.
 			newEntry.Value = result.Value
 			newEntry.Index = result.Index
+			newEntry.FetchedAt = time.Now()
 			if newEntry.Index < 1 {
 				// Less than one is invalid unless there was an error and in this case
 				// there wasn't since a value was returned. If a badly behaved RPC
@@ -395,6 +486,12 @@ func (c *Cache) fetch(t, key string, r Request, allowNew bool, attempt uint) (<-
 				// for the future.
 				attempt++
 			}
+
+			// If we have refresh active, this successful response means cache is now
+			// "connected" and should not be stale. Reset the lost contact timer.
+			if tEntry.Opts.Refresh {
+				newEntry.RefreshLostContact = time.Time{}
+			}
 		} else {
 			metrics.IncrCounter([]string{"consul", "cache", "fetch_error"}, 1)
 			metrics.IncrCounter([]string{"consul", "cache", t, "fetch_error"}, 1)
@@ -408,6 +505,14 @@ func (c *Cache) fetch(t, key string, r Request, allowNew bool, attempt uint) (<-
 			// we want Error to be set so that we can return early with the
 			// error.
 			newEntry.Error = err
+
+			// If we are refreshing and just failed, updated the lost contact time as
+			// our cache will be stale until we get successfully reconnected. We only
+			// set this on the first failure (if it's zero) so we can track how long
+			// it's been since we had a valid connection/up-to-date view of the state.
+			if tEntry.Opts.Refresh && newEntry.RefreshLostContact.IsZero() {
+				newEntry.RefreshLostContact = time.Now()
+			}
 		}
 
 		// Create a new waiter that will be used for the next fetch.
@@ -482,7 +587,11 @@ func (c *Cache) refresh(opts *RegisterOptions, attempt uint, t string, key strin
 
 	// If we're over the attempt minimum, start an exponential backoff.
 	if attempt > CacheRefreshBackoffMin {
-		waitTime := (1 << (attempt - CacheRefreshBackoffMin)) * time.Second
+		shift := attempt - CacheRefreshBackoffMin
+		waitTime := CacheRefreshMaxWait
+		if shift < 31 {
+			waitTime = (1 << shift) * time.Second
+		}
 		if waitTime > CacheRefreshMaxWait {
 			waitTime = CacheRefreshMaxWait
 		}

agent/cache/cache_test.go

@@ -1,6 +1,7 @@
 package cache
 
 import (
+	"errors"
 	"fmt"
 	"sort"
 	"sync"
@@ -631,12 +632,19 @@ func TestCacheGet_expire(t *testing.T) {
 	require.Equal(42, result)
 	require.False(meta.Hit)
 
+	// Wait for a non-trivial amount of time to sanity check the age increases at
+	// least this amount. Note that this is not a fudge for some timing-dependent
+	// background work it's just ensuring a non-trivial time elapses between the
+	// request above and below serilaly in this thread so short time is OK.
+	time.Sleep(5 * time.Millisecond)
+
 	// Get, should not fetch, verified via the mock assertions above
 	req = TestRequest(t, RequestInfo{Key: "hello"})
 	result, meta, err = c.Get("t", req)
 	require.NoError(err)
 	require.Equal(42, result)
 	require.True(meta.Hit)
+	require.True(meta.Age > 5*time.Millisecond)
 
 	// Sleep for the expiry
 	time.Sleep(500 * time.Millisecond)
@@ -806,3 +814,280 @@ func (t *testPartitionType) Fetch(opts FetchOptions, r Request) (FetchResult, er
 		Value: fmt.Sprintf("%s%s", info.Datacenter, info.Token),
 	}, nil
 }
+
+func (t *testPartitionType) SupportsBlocking() bool {
+	return true
+}
+
+// Test that background refreshing reports correct Age in failure and happy
+// states.
+func TestCacheGet_refreshAge(t *testing.T) {
+	t.Parallel()
+
+	require := require.New(t)
+
+	typ := TestType(t)
+	defer typ.AssertExpectations(t)
+	c := TestCache(t)
+	c.RegisterType("t", typ, &RegisterOptions{
+		Refresh:        true,
+		RefreshTimer:   0,
+		RefreshTimeout: 5 * time.Minute,
+	})
+
+	// Configure the type
+	var index, shouldFail uint64
+
+	typ.On("Fetch", mock.Anything, mock.Anything).
+		Return(func(o FetchOptions, r Request) FetchResult {
+			idx := atomic.LoadUint64(&index)
+			if atomic.LoadUint64(&shouldFail) == 1 {
+				t.Logf("Failing Fetch at index %d", idx)
+				return FetchResult{Value: nil, Index: idx}
+			}
+			if o.MinIndex == idx {
+				t.Logf("Sleeping Fetch at index %d", idx)
+				// Simulate waiting for a new value
+				time.Sleep(5 * time.Millisecond)
+			}
+			t.Logf("Returning Fetch at index %d", idx)
+			return FetchResult{Value: int(idx * 2), Index: idx}
+		}, func(o FetchOptions, r Request) error {
+			if atomic.LoadUint64(&shouldFail) == 1 {
+				return errors.New("test error")
+			}
+			return nil
+		})
+
+	// Set initial index/value
+	atomic.StoreUint64(&index, 4)
+
+	// Fetch
+	resultCh := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
+	TestCacheGetChResult(t, resultCh, 8)
+
+	{
+		// Wait a few milliseconds after initial fetch to check age is not reporting
+		// actual age.
+		time.Sleep(2 * time.Millisecond)
+
+		// Fetch again, non-blocking
+		result, meta, err := c.Get("t", TestRequest(t, RequestInfo{Key: "hello"}))
+		require.NoError(err)
+		require.Equal(8, result)
+		require.True(meta.Hit)
+		// Age should be zero since background refresh was "active"
+		require.Equal(time.Duration(0), meta.Age)
+	}
+
+	// Now fail the next background sync
+	atomic.StoreUint64(&shouldFail, 1)
+
+	// Wait until the current request times out and starts failing
+	time.Sleep(6 * time.Millisecond)
+
+	var lastAge time.Duration
+	{
+		result, meta, err := c.Get("t", TestRequest(t, RequestInfo{Key: "hello"}))
+		require.NoError(err)
+		require.Equal(8, result)
+		require.True(meta.Hit)
+		// Age should be non-zero since background refresh was "active"
+		require.True(meta.Age > 0)
+		lastAge = meta.Age
+	}
+	// Wait a bit longer - age should increase by at least this much
+	time.Sleep(1 * time.Millisecond)
+	{
+		result, meta, err := c.Get("t", TestRequest(t, RequestInfo{Key: "hello"}))
+		require.NoError(err)
+		require.Equal(8, result)
+		require.True(meta.Hit)
+		require.True(meta.Age > (lastAge + (1 * time.Millisecond)))
+	}
+
+	// Now unfail the background refresh
+	atomic.StoreUint64(&shouldFail, 0)
+
+	// And update the data so we can see when the background task is working again
+	// (won't be immediate due to backoff on the errors).
+	atomic.AddUint64(&index, 1)
+
+	t0 := time.Now()
+
+	timeout := true
+	// Allow up to 5 seconds since the error backoff is likely to have kicked in
+	// and causes this to take different amounts of time depending on how quickly
+	// the test thread got down here relative to the failures.
+	for attempts := 0; attempts < 50; attempts++ {
+		time.Sleep(100 * time.Millisecond)
+		result, meta, err := c.Get("t", TestRequest(t, RequestInfo{Key: "hello"}))
+		// Should never error even if background is failing as we have cached value
+		require.NoError(err)
+		require.True(meta.Hit)
+		// Got the new value!
+		if result == 10 {
+			// Age should be zero since background refresh is "active" again
+			t.Logf("Succeeded after %d attempts", attempts)
+			require.Equal(time.Duration(0), meta.Age)
+			timeout = false
+			break
+		}
+	}
+	require.False(timeout, "failed to observe update after %s", time.Since(t0))
+}
+
+func TestCacheGet_nonRefreshAge(t *testing.T) {
+	t.Parallel()
+
+	require := require.New(t)
+
+	typ := TestType(t)
+	defer typ.AssertExpectations(t)
+	c := TestCache(t)
+	c.RegisterType("t", typ, &RegisterOptions{
+		Refresh:    false,
+		LastGetTTL: 100 * time.Millisecond,
+	})
+
+	// Configure the type
+	var index uint64
+
+	typ.On("Fetch", mock.Anything, mock.Anything).
+		Return(func(o FetchOptions, r Request) FetchResult {
+			idx := atomic.LoadUint64(&index)
+			return FetchResult{Value: int(idx * 2), Index: idx}
+		}, nil)
+
+	// Set initial index/value
+	atomic.StoreUint64(&index, 4)
+
+	// Fetch
+	resultCh := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
+	TestCacheGetChResult(t, resultCh, 8)
+
+	var lastAge time.Duration
+	{
+		// Wait a few milliseconds after initial fetch to check age IS reporting
+		// actual age.
+		time.Sleep(5 * time.Millisecond)
+
+		// Fetch again, non-blocking
+		result, meta, err := c.Get("t", TestRequest(t, RequestInfo{Key: "hello"}))
+		require.NoError(err)
+		require.Equal(8, result)
+		require.True(meta.Hit)
+		require.True(meta.Age > (5 * time.Millisecond))
+		lastAge = meta.Age
+	}
+
+	// Wait for expiry
+	time.Sleep(200 * time.Millisecond)
+
+	{
+		result, meta, err := c.Get("t", TestRequest(t, RequestInfo{Key: "hello"}))
+		require.NoError(err)
+		require.Equal(8, result)
+		require.False(meta.Hit)
+		// Age should smaller again
+		require.True(meta.Age < lastAge)
+	}
+
+	{
+		// Wait for a non-trivial amount of time to sanity check the age increases at
+		// least this amount. Note that this is not a fudge for some timing-dependent
+		// background work it's just ensuring a non-trivial time elapses between the
+		// request above and below serilaly in this thread so short time is OK.
+		time.Sleep(5 * time.Millisecond)
+
+		// Fetch again, non-blocking
+		result, meta, err := c.Get("t", TestRequest(t, RequestInfo{Key: "hello"}))
+		require.NoError(err)
+		require.Equal(8, result)
+		require.True(meta.Hit)
+		require.True(meta.Age > (5 * time.Millisecond))
+		lastAge = meta.Age
+	}
+
+	// Now verify that setting MaxAge results in cache invalidation
+	{
+		result, meta, err := c.Get("t", TestRequest(t, RequestInfo{
+			Key:    "hello",
+			MaxAge: 1 * time.Millisecond,
+		}))
+		require.NoError(err)
+		require.Equal(8, result)
+		require.False(meta.Hit)
+		// Age should smaller again
+		require.True(meta.Age < lastAge)
+	}
+}
+
+func TestCacheGet_nonBlockingType(t *testing.T) {
+	t.Parallel()
+
+	typ := TestTypeNonBlocking(t)
+	defer typ.AssertExpectations(t)
+	c := TestCache(t)
+	c.RegisterType("t", typ, nil)
+
+	// Configure the type
+	typ.Static(FetchResult{Value: 42, Index: 1}, nil).Once()
+	typ.Static(FetchResult{Value: 43, Index: 2}, nil).Twice().
+		Run(func(args mock.Arguments) {
+			opts := args.Get(0).(FetchOptions)
+			// MinIndex should never be set for a non-blocking type.
+			require.Equal(t, uint64(0), opts.MinIndex)
+		})
+
+	require := require.New(t)
+
+	// Get, should fetch
+	req := TestRequest(t, RequestInfo{Key: "hello"})
+	result, meta, err := c.Get("t", req)
+	require.NoError(err)
+	require.Equal(42, result)
+	require.False(meta.Hit)
+
+	// Get, should not fetch since we have a cached value
+	req = TestRequest(t, RequestInfo{Key: "hello"})
+	result, meta, err = c.Get("t", req)
+	require.NoError(err)
+	require.Equal(42, result)
+	require.True(meta.Hit)
+
+	// Get, should not attempt to fetch with blocking even if requested. The
+	// assertions below about the value being the same combined with the fact the
+	// mock will only return that value on first call suffice to show that
+	// blocking request is not being attempted.
+	req = TestRequest(t, RequestInfo{
+		Key:      "hello",
+		MinIndex: 1,
+		Timeout:  10 * time.Minute,
+	})
+	result, meta, err = c.Get("t", req)
+	require.NoError(err)
+	require.Equal(42, result)
+	require.True(meta.Hit)
+
+	time.Sleep(10 * time.Millisecond)
+
+	// Get with a max age should fetch again
+	req = TestRequest(t, RequestInfo{Key: "hello", MaxAge: 5 * time.Millisecond})
+	result, meta, err = c.Get("t", req)
+	require.NoError(err)
+	require.Equal(43, result)
+	require.False(meta.Hit)
+
+	// Get with a must revalidate should fetch again even without a delay.
+	req = TestRequest(t, RequestInfo{Key: "hello", MustRevalidate: true})
+	result, meta, err = c.Get("t", req)
+	require.NoError(err)
+	require.Equal(43, result)
+	require.False(meta.Hit)
+
+	// Sleep a tiny bit just to let maybe some background calls happen
+	// then verify that we still only got the one call
+	time.Sleep(20 * time.Millisecond)
+	typ.AssertExpectations(t)
+}

agent/cache/entry.go

@@ -24,6 +24,16 @@ type cacheEntry struct {
 	// entry. This is a pointer as its shared as a value in the
 	// expiryHeap as well.
 	Expiry *cacheEntryExpiry
+
+	// FetchedAt stores the time the cache entry was retrieved for determining
+	// it's age later.
+	FetchedAt time.Time
+
+	// RefreshLostContact stores the time background refresh failed. It gets reset
+	// to zero after a background fetch has returned successfully, or after a
+	// background request has be blocking for at least 5 seconds, which ever
+	// happens first.
+	RefreshLostContact time.Time
 }
 
 // cacheEntryExpiry contains the expiration information for a cache

agent/cache/mock_Type.go

@@ -1,4 +1,4 @@
-// Code generated by mockery v1.0.0
+// Code generated by mockery v1.0.0. DO NOT EDIT.
 package cache
 
 import mock "github.com/stretchr/testify/mock"
@@ -28,3 +28,17 @@ func (_m *MockType) Fetch(_a0 FetchOptions, _a1 Request) (FetchResult, error) {
 
 	return r0, r1
 }
+
+// SupportsBlocking provides a mock function with given fields:
+func (_m *MockType) SupportsBlocking() bool {
+	ret := _m.Called()
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func() bool); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	return r0
+}

agent/cache/request.go

@@ -48,4 +48,18 @@ type RequestInfo struct {
 	// if there was no prior value). This "last known value" behavior matches
 	// normal Consul blocking queries.
 	Timeout time.Duration
+
+	// MaxAge if set limits how stale a cache entry can be. If it is non-zero and
+	// there is an entry in cache that is older than specified, it is treated as a
+	// cache miss and re-fetched. It is ignored for cachetypes with Refresh =
+	// true.
+	MaxAge time.Duration
+
+	// MustRevalidate forces a new lookup of the cache even if there is an
+	// existing one that has not expired. It is implied by HTTP requests with
+	// `Cache-Control: max-age=0` but we can't distinguish that case from the
+	// unset case for MaxAge. Later we may support revalidating the index without
+	// a full re-fetch but for now the only option is to refetch. It is ignored
+	// for cachetypes with Refresh = true.
+	MustRevalidate bool
 }

agent/cache/testing.go

@@ -61,7 +61,17 @@ func TestRequest(t testing.T, info RequestInfo) *MockRequest {
 // TestType returns a MockType that can be used to setup expectations
 // on data fetching.
 func TestType(t testing.T) *MockType {
+	return testTypeInternal(t, true)
+}
+
+// TestTypeNonBlocking returns a MockType that returns false to SupportsBlocking.
+func TestTypeNonBlocking(t testing.T) *MockType {
+	return testTypeInternal(t, false)
+}
+
+func testTypeInternal(t testing.T, enableBlocking bool) *MockType {
 	typ := &MockType{}
+	typ.On("SupportsBlocking").Return(enableBlocking).Maybe()
 	return typ
 }
 

agent/cache/type.go

@@ -23,6 +23,12 @@ type Type interface {
 	// no extra logic. Second, FetchResult can return an unset value and index.
 	// In this case, the cache will reuse the last value automatically.
 	Fetch(FetchOptions, Request) (FetchResult, error)
+
+	// SupportsBlocking should return true if the type supports blocking queries.
+	// Types that do not support blocking queries will not be able to use
+	// background refresh nor will the cache attempt blocking fetches if the
+	// client requests them with MinIndex.
+	SupportsBlocking() bool
 }
 
 // FetchOptions are various settable options when a Fetch is called.

agent/catalog_endpoint.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	metrics "github.com/armon/go-metrics"
+	cachetype "github.com/hashicorp/consul/agent/cache-types"
 	"github.com/hashicorp/consul/agent/structs"
 )
 
@@ -202,17 +203,35 @@ func (s *HTTPServer) catalogServiceNodes(resp http.ResponseWriter, req *http.Req
 	// Make the RPC request
 	var out structs.IndexedServiceNodes
 	defer setMeta(resp, &out.QueryMeta)
-RETRY_ONCE:
-	if err := s.agent.RPC("Catalog.ServiceNodes", &args, &out); err != nil {
-		metrics.IncrCounterWithLabels([]string{"client", "rpc", "error", "catalog_service_nodes"}, 1,
-			[]metrics.Label{{Name: "node", Value: s.nodeName()}})
-		return nil, err
-	}
-	if args.QueryOptions.AllowStale && args.MaxStaleDuration > 0 && args.MaxStaleDuration < out.LastContact {
-		args.AllowStale = false
-		args.MaxStaleDuration = 0
-		goto RETRY_ONCE
+
+	if args.QueryOptions.UseCache {
+		raw, m, err := s.agent.cache.Get(cachetype.CatalogServicesName, &args)
+		if err != nil {
+			metrics.IncrCounterWithLabels([]string{"client", "rpc", "error", "catalog_service_nodes"}, 1,
+				[]metrics.Label{{Name: "node", Value: s.nodeName()}})
+			return nil, err
+		}
+		defer setCacheMeta(resp, &m)
+		reply, ok := raw.(*structs.IndexedServiceNodes)
+		if !ok {
+			// This should never happen, but we want to protect against panics
+			return nil, fmt.Errorf("internal error: response type not correct")
+		}
+		out = *reply
+	} else {
+	RETRY_ONCE:
+		if err := s.agent.RPC("Catalog.ServiceNodes", &args, &out); err != nil {
+			metrics.IncrCounterWithLabels([]string{"client", "rpc", "error", "catalog_service_nodes"}, 1,
+				[]metrics.Label{{Name: "node", Value: s.nodeName()}})
+			return nil, err
+		}
+		if args.QueryOptions.AllowStale && args.MaxStaleDuration > 0 && args.MaxStaleDuration < out.LastContact {
+			args.AllowStale = false
+			args.MaxStaleDuration = 0
+			goto RETRY_ONCE
+		}
 	}
+
 	out.ConsistencyLevel = args.QueryOptions.ConsistencyLevel()
 	s.agent.TranslateAddresses(args.Datacenter, out.ServiceNodes)
 

agent/catalog_endpoint_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/hashicorp/consul/testutil/retry"
 	"github.com/hashicorp/serf/coordinate"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestCatalogRegister_Service_InvalidAddress(t *testing.T) {
@@ -486,6 +487,9 @@ func TestCatalogServiceNodes(t *testing.T) {
 	a := NewTestAgent(t.Name(), "")
 	defer a.Shutdown()
 
+	assert := assert.New(t)
+	require := require.New(t)
+
 	// Make sure an empty list is returned, not a nil
 	{
 		req, _ := http.NewRequest("GET", "/v1/catalog/service/api?tag=a", nil)
@@ -532,6 +536,63 @@ func TestCatalogServiceNodes(t *testing.T) {
 	if len(nodes) != 1 {
 		t.Fatalf("bad: %v", obj)
 	}
+
+	// Test caching
+	{
+		// List instances with cache enabled
+		req, _ := http.NewRequest("GET", "/v1/catalog/service/api?cached", nil)
+		resp := httptest.NewRecorder()
+		obj, err := a.srv.CatalogServiceNodes(resp, req)
+		require.NoError(err)
+		nodes := obj.(structs.ServiceNodes)
+		assert.Len(nodes, 1)
+
+		// Should be a cache miss
+		assert.Equal("MISS", resp.Header().Get("X-Cache"))
+	}
+
+	{
+		// List instances with cache enabled
+		req, _ := http.NewRequest("GET", "/v1/catalog/service/api?cached", nil)
+		resp := httptest.NewRecorder()
+		obj, err := a.srv.CatalogServiceNodes(resp, req)
+		require.NoError(err)
+		nodes := obj.(structs.ServiceNodes)
+		assert.Len(nodes, 1)
+
+		// Should be a cache HIT now!
+		assert.Equal("HIT", resp.Header().Get("X-Cache"))
+		assert.Equal("0", resp.Header().Get("Age"))
+	}
+
+	// Ensure background refresh works
+	{
+		// Register a new instance of the service
+		args2 := args
+		args2.Node = "bar"
+		args2.Address = "127.0.0.2"
+		require.NoError(a.RPC("Catalog.Register", args, &out))
+
+		retry.Run(t, func(r *retry.R) {
+			// List it again
+			req, _ := http.NewRequest("GET", "/v1/catalog/service/api?cached", nil)
+			resp := httptest.NewRecorder()
+			obj, err := a.srv.CatalogServiceNodes(resp, req)
+			r.Check(err)
+
+			nodes := obj.(structs.ServiceNodes)
+			if len(nodes) != 2 {
+				r.Fatalf("Want 2 nodes")
+			}
+
+			// Should be a cache hit! The data should've updated in the cache
+			// in the background so this should've been fetched directly from
+			// the cache.
+			if resp.Header().Get("X-Cache") != "HIT" {
+				r.Fatalf("should be a cache hit")
+			}
+		})
+	}
 }
 
 func TestCatalogServiceNodes_NodeMetaFilter(t *testing.T) {

agent/health_endpoint.go

@@ -6,6 +6,7 @@ import (
 	"strconv"
 	"strings"
 
+	cachetype "github.com/hashicorp/consul/agent/cache-types"
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/api"
 )
@@ -184,14 +185,29 @@ func (s *HTTPServer) healthServiceNodes(resp http.ResponseWriter, req *http.Requ
 	// Make the RPC request
 	var out structs.IndexedCheckServiceNodes
 	defer setMeta(resp, &out.QueryMeta)
-RETRY_ONCE:
-	if err := s.agent.RPC("Health.ServiceNodes", &args, &out); err != nil {
-		return nil, err
-	}
-	if args.QueryOptions.AllowStale && args.MaxStaleDuration > 0 && args.MaxStaleDuration < out.LastContact {
-		args.AllowStale = false
-		args.MaxStaleDuration = 0
-		goto RETRY_ONCE
+
+	if args.QueryOptions.UseCache {
+		raw, m, err := s.agent.cache.Get(cachetype.HealthServicesName, &args)
+		if err != nil {
+			return nil, err
+		}
+		defer setCacheMeta(resp, &m)
+		reply, ok := raw.(*structs.IndexedCheckServiceNodes)
+		if !ok {
+			// This should never happen, but we want to protect against panics
+			return nil, fmt.Errorf("internal error: response type not correct")
+		}
+		out = *reply
+	} else {
+	RETRY_ONCE:
+		if err := s.agent.RPC("Health.ServiceNodes", &args, &out); err != nil {
+			return nil, err
+		}
+		if args.QueryOptions.AllowStale && args.MaxStaleDuration > 0 && args.MaxStaleDuration < out.LastContact {
+			args.AllowStale = false
+			args.MaxStaleDuration = 0
+			goto RETRY_ONCE
+		}
 	}
 	out.ConsistencyLevel = args.QueryOptions.ConsistencyLevel()
 

agent/health_endpoint_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/hashicorp/consul/testutil/retry"
 	"github.com/hashicorp/serf/coordinate"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestHealthChecksInState(t *testing.T) {
@@ -406,6 +407,9 @@ func TestHealthServiceNodes(t *testing.T) {
 	defer a.Shutdown()
 	testrpc.WaitForTestAgent(t, a.RPC, "dc1")
 
+	assert := assert.New(t)
+	require := require.New(t)
+
 	req, _ := http.NewRequest("GET", "/v1/health/service/consul?dc=dc1", nil)
 	resp := httptest.NewRecorder()
 	obj, err := a.srv.HealthServiceNodes(resp, req)
@@ -465,6 +469,62 @@ func TestHealthServiceNodes(t *testing.T) {
 	if len(nodes) != 1 || nodes[0].Checks == nil || len(nodes[0].Checks) != 0 {
 		t.Fatalf("bad: %v", obj)
 	}
+
+	// Test caching
+	{
+		// List instances with cache enabled
+		req, _ := http.NewRequest("GET", "/v1/health/service/test?cached", nil)
+		resp := httptest.NewRecorder()
+		obj, err := a.srv.HealthServiceNodes(resp, req)
+		require.NoError(err)
+		nodes := obj.(structs.CheckServiceNodes)
+		assert.Len(nodes, 1)
+
+		// Should be a cache miss
+		assert.Equal("MISS", resp.Header().Get("X-Cache"))
+	}
+
+	{
+		// List instances with cache enabled
+		req, _ := http.NewRequest("GET", "/v1/health/service/test?cached", nil)
+		resp := httptest.NewRecorder()
+		obj, err := a.srv.HealthServiceNodes(resp, req)
+		require.NoError(err)
+		nodes := obj.(structs.CheckServiceNodes)
+		assert.Len(nodes, 1)
+
+		// Should be a cache HIT now!
+		assert.Equal("HIT", resp.Header().Get("X-Cache"))
+	}
+
+	// Ensure background refresh works
+	{
+		// Register a new instance of the service
+		args2 := args
+		args2.Node = "baz"
+		args2.Address = "127.0.0.2"
+		require.NoError(a.RPC("Catalog.Register", args, &out))
+
+		retry.Run(t, func(r *retry.R) {
+			// List it again
+			req, _ := http.NewRequest("GET", "/v1/health/service/test?cached", nil)
+			resp := httptest.NewRecorder()
+			obj, err := a.srv.HealthServiceNodes(resp, req)
+			r.Check(err)
+
+			nodes := obj.(structs.CheckServiceNodes)
+			if len(nodes) != 2 {
+				r.Fatalf("Want 2 nodes")
+			}
+
+			// Should be a cache hit! The data should've updated in the cache
+			// in the background so this should've been fetched directly from
+			// the cache.
+			if resp.Header().Get("X-Cache") != "HIT" {
+				r.Fatalf("should be a cache hit")
+			}
+		})
+	}
 }
 
 func TestHealthServiceNodes_NodeMetaFilter(t *testing.T) {

agent/http.go

@@ -473,6 +473,9 @@ func setCacheMeta(resp http.ResponseWriter, m *cache.ResultMeta) {
 		str = "HIT"
 	}
 	resp.Header().Set("X-Cache", str)
+	if m.Hit {
+		resp.Header().Set("Age", fmt.Sprintf("%.0f", m.Age.Seconds()))
+	}
 }
 
 // setHeaders is used to set canonical response header fields
@@ -507,6 +510,64 @@ func parseWait(resp http.ResponseWriter, req *http.Request, b *structs.QueryOpti
 	return false
 }
 
+// parseCacheControl parses the CacheControl HTTP header value. So far we only
+// support maxage directive.
+func parseCacheControl(resp http.ResponseWriter, req *http.Request, b *structs.QueryOptions) bool {
+	raw := strings.ToLower(req.Header.Get("Cache-Control"))
+
+	if raw == "" {
+		return false
+	}
+
+	// Didn't want to import a full parser for this. While quoted strings are
+	// allowed in some directives, max-age does not allow them per
+	// https://tools.ietf.org/html/rfc7234#section-5.2.2.8 so we assume all
+	// well-behaved clients use the exact token form of max-age=<delta-seconds>
+	// where delta-seconds is a non-negative decimal integer.
+	directives := strings.Split(raw, ",")
+
+	parseDurationOrFail := func(raw string) (time.Duration, bool) {
+		i, err := strconv.Atoi(raw)
+		if err != nil {
+			resp.WriteHeader(http.StatusBadRequest)
+			fmt.Fprint(resp, "Invalid Cache-Control header.")
+			return 0, true
+		}
+		return time.Duration(i) * time.Second, false
+	}
+
+	for _, d := range directives {
+		d = strings.ToLower(strings.TrimSpace(d))
+
+		if d == "must-revalidate" {
+			b.MustRevalidate = true
+		}
+
+		if strings.HasPrefix(d, "max-age=") {
+			d, failed := parseDurationOrFail(d[8:])
+			if failed {
+				return true
+			}
+			b.MaxAge = d
+			if d == 0 {
+				// max-age=0 specifically means that we need to consider the cache stale
+				// immediately however MaxAge = 0 is indistinguishable from the default
+				// where MaxAge is unset.
+				b.MustRevalidate = true
+			}
+		}
+		if strings.HasPrefix(d, "stale-if-error=") {
+			d, failed := parseDurationOrFail(d[15:])
+			if failed {
+				return true
+			}
+			b.StaleIfError = d
+		}
+	}
+
+	return false
+}
+
 // parseConsistency is used to parse the ?stale and ?consistent query params.
 // Returns true on error
 func (s *HTTPServer) parseConsistency(resp http.ResponseWriter, req *http.Request, b *structs.QueryOptions) bool {
@@ -523,6 +584,10 @@ func (s *HTTPServer) parseConsistency(resp http.ResponseWriter, req *http.Reques
 	if _, ok := query["leader"]; ok {
 		defaults = false
 	}
+	if _, ok := query["cached"]; ok {
+		b.UseCache = true
+		defaults = false
+	}
 	if maxStale := query.Get("max_stale"); maxStale != "" {
 		dur, err := time.ParseDuration(maxStale)
 		if err != nil {
@@ -551,6 +616,11 @@ func (s *HTTPServer) parseConsistency(resp http.ResponseWriter, req *http.Reques
 		fmt.Fprint(resp, "Cannot specify ?stale with ?consistent, conflicting semantics.")
 		return true
 	}
+	if b.UseCache && b.RequireConsistent {
+		resp.WriteHeader(http.StatusBadRequest)
+		fmt.Fprint(resp, "Cannot specify ?cached with ?consistent, conflicting semantics.")
+		return true
+	}
 	return false
 }
 
@@ -682,6 +752,9 @@ func (s *HTTPServer) parseInternal(resp http.ResponseWriter, req *http.Request,
 	if s.parseConsistency(resp, req, b) {
 		return true
 	}
+	if parseCacheControl(resp, req, b) {
+		return true
+	}
 	return parseWait(resp, req, b)
 }
 

agent/http_test.go

@@ -567,6 +567,146 @@ func TestParseSource(t *testing.T) {
 	}
 }
 
+func TestParseCacheControl(t *testing.T) {
+
+	tests := []struct {
+		name      string
+		headerVal string
+		want      structs.QueryOptions
+		wantErr   bool
+	}{
+		{
+			name:      "empty header",
+			headerVal: "",
+			want:      structs.QueryOptions{},
+			wantErr:   false,
+		},
+		{
+			name:      "simple max-age",
+			headerVal: "max-age=30",
+			want: structs.QueryOptions{
+				MaxAge: 30 * time.Second,
+			},
+			wantErr: false,
+		},
+		{
+			name:      "zero max-age",
+			headerVal: "max-age=0",
+			want: structs.QueryOptions{
+				MustRevalidate: true,
+			},
+			wantErr: false,
+		},
+		{
+			name:      "must-revalidate",
+			headerVal: "must-revalidate",
+			want: structs.QueryOptions{
+				MustRevalidate: true,
+			},
+			wantErr: false,
+		},
+		{
+			name:      "mixes age, must-revalidate",
+			headerVal: "max-age=123, must-revalidate",
+			want: structs.QueryOptions{
+				MaxAge:         123 * time.Second,
+				MustRevalidate: true,
+			},
+			wantErr: false,
+		},
+		{
+			name:      "quoted max-age",
+			headerVal: "max-age=\"30\"",
+			want:      structs.QueryOptions{},
+			wantErr:   true,
+		},
+		{
+			name:      "mixed case max-age",
+			headerVal: "Max-Age=30",
+			want: structs.QueryOptions{
+				MaxAge: 30 * time.Second,
+			},
+			wantErr: false,
+		},
+		{
+			name:      "simple stale-if-error",
+			headerVal: "stale-if-error=300",
+			want: structs.QueryOptions{
+				StaleIfError: 300 * time.Second,
+			},
+			wantErr: false,
+		},
+		{
+			name:      "combined with space",
+			headerVal: "max-age=30, stale-if-error=300",
+			want: structs.QueryOptions{
+				MaxAge:       30 * time.Second,
+				StaleIfError: 300 * time.Second,
+			},
+			wantErr: false,
+		},
+		{
+			name:      "combined no space",
+			headerVal: "stale-IF-error=300,max-age=30",
+			want: structs.QueryOptions{
+				MaxAge:       30 * time.Second,
+				StaleIfError: 300 * time.Second,
+			},
+			wantErr: false,
+		},
+		{
+			name:      "unsupported directive",
+			headerVal: "no-cache",
+			want:      structs.QueryOptions{},
+			wantErr:   false,
+		},
+		{
+			name:      "mixed unsupported directive",
+			headerVal: "no-cache, max-age=120",
+			want: structs.QueryOptions{
+				MaxAge: 120 * time.Second,
+			},
+			wantErr: false,
+		},
+		{
+			name:      "garbage value",
+			headerVal: "max-age=\"I'm not, an int\"",
+			want:      structs.QueryOptions{},
+			wantErr:   true,
+		},
+		{
+			name:      "garbage value with quotes",
+			headerVal: "max-age=\"I'm \\\"not an int\"",
+			want:      structs.QueryOptions{},
+			wantErr:   true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			require := require.New(t)
+
+			r, _ := http.NewRequest("GET", "/foo/bar", nil)
+			if tt.headerVal != "" {
+				r.Header.Set("Cache-Control", tt.headerVal)
+			}
+
+			rr := httptest.NewRecorder()
+			var got structs.QueryOptions
+
+			failed := parseCacheControl(rr, r, &got)
+			if tt.wantErr {
+				require.True(failed)
+				require.Equal(http.StatusBadRequest, rr.Code)
+			} else {
+				require.False(failed)
+			}
+
+			require.Equal(tt.want, got)
+		})
+	}
+}
+
 func TestParseWait(t *testing.T) {
 	t.Parallel()
 	resp := httptest.NewRecorder()

agent/prepared_query_endpoint.go

@@ -6,6 +6,7 @@ import (
 	"strconv"
 	"strings"
 
+	cachetype "github.com/hashicorp/consul/agent/cache-types"
 	"github.com/hashicorp/consul/agent/consul"
 	"github.com/hashicorp/consul/agent/structs"
 )
@@ -119,21 +120,42 @@ func (s *HTTPServer) preparedQueryExecute(id string, resp http.ResponseWriter, r
 
 	var reply structs.PreparedQueryExecuteResponse
 	defer setMeta(resp, &reply.QueryMeta)
-RETRY_ONCE:
-	if err := s.agent.RPC("PreparedQuery.Execute", &args, &reply); err != nil {
-		// We have to check the string since the RPC sheds
-		// the specific error type.
-		if err.Error() == consul.ErrQueryNotFound.Error() {
-			resp.WriteHeader(http.StatusNotFound)
-			fmt.Fprint(resp, err.Error())
-			return nil, nil
+
+	if args.QueryOptions.UseCache {
+		raw, m, err := s.agent.cache.Get(cachetype.PreparedQueryName, &args)
+		if err != nil {
+			// Don't return error if StaleIfError is set and we are within it and had
+			// a cached value.
+			if raw != nil && m.Hit && args.QueryOptions.StaleIfError > m.Age {
+				// Fall through to the happy path below
+			} else {
+				return nil, err
+			}
+		}
+		defer setCacheMeta(resp, &m)
+		r, ok := raw.(*structs.PreparedQueryExecuteResponse)
+		if !ok {
+			// This should never happen, but we want to protect against panics
+			return nil, fmt.Errorf("internal error: response type not correct")
+		}
+		reply = *r
+	} else {
+	RETRY_ONCE:
+		if err := s.agent.RPC("PreparedQuery.Execute", &args, &reply); err != nil {
+			// We have to check the string since the RPC sheds
+			// the specific error type.
+			if err.Error() == consul.ErrQueryNotFound.Error() {
+				resp.WriteHeader(http.StatusNotFound)
+				fmt.Fprint(resp, err.Error())
+				return nil, nil
+			}
+			return nil, err
+		}
+		if args.QueryOptions.AllowStale && args.MaxStaleDuration > 0 && args.MaxStaleDuration < reply.LastContact {
+			args.AllowStale = false
+			args.MaxStaleDuration = 0
+			goto RETRY_ONCE
 		}
-		return nil, err
-	}
-	if args.QueryOptions.AllowStale && args.MaxStaleDuration > 0 && args.MaxStaleDuration < reply.LastContact {
-		args.AllowStale = false
-		args.MaxStaleDuration = 0
-		goto RETRY_ONCE
 	}
 	reply.ConsistencyLevel = args.QueryOptions.ConsistencyLevel()
 

agent/prepared_query_endpoint_test.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"reflect"
+	"sync/atomic"
 	"testing"
 
 	"github.com/hashicorp/consul/testrpc"
@@ -610,6 +611,63 @@ func TestPreparedQuery_Execute(t *testing.T) {
 	})
 }
 
+func TestPreparedQuery_ExecuteCached(t *testing.T) {
+	t.Parallel()
+
+	a := NewTestAgent(t.Name(), "")
+	defer a.Shutdown()
+
+	failovers := int32(99)
+
+	m := MockPreparedQuery{
+		executeFn: func(args *structs.PreparedQueryExecuteRequest, reply *structs.PreparedQueryExecuteResponse) error {
+			// Just set something so we can tell this is returned.
+			reply.Failovers = int(atomic.LoadInt32(&failovers))
+			return nil
+		},
+	}
+	if err := a.registerEndpoint("PreparedQuery", &m); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	doRequest := func(expectFailovers int, expectCache string, revalidate bool) {
+		body := bytes.NewBuffer(nil)
+		req, _ := http.NewRequest("GET", "/v1/query/my-id/execute?cached", body)
+
+		if revalidate {
+			req.Header.Set("Cache-Control", "must-revalidate")
+		}
+
+		resp := httptest.NewRecorder()
+		obj, err := a.srv.PreparedQuerySpecific(resp, req)
+
+		require := require.New(t)
+		require.NoError(err)
+		require.Equal(200, resp.Code)
+
+		r, ok := obj.(structs.PreparedQueryExecuteResponse)
+		require.True(ok)
+		require.Equal(expectFailovers, r.Failovers)
+
+		require.Equal(expectCache, resp.Header().Get("X-Cache"))
+	}
+
+	// Should be a miss at first
+	doRequest(99, "MISS", false)
+
+	// Change the actual response
+	atomic.StoreInt32(&failovers, 66)
+
+	// Request again, should be a cache hit and have the cached (not current)
+	// value.
+	doRequest(99, "HIT", false)
+
+	// Request with max age that should invalidate cache. note that this will be
+	// sent as max-age=0 as that uses seconds but that should cause immediate
+	// invalidation rather than being ignored as an unset value.
+	doRequest(66, "MISS", true)
+}
+
 func TestPreparedQuery_Explain(t *testing.T) {
 	t.Parallel()
 	t.Run("", func(t *testing.T) {

agent/structs/prepared_query.go

@@ -1,6 +1,12 @@
 package structs
 
-import "github.com/hashicorp/consul/types"
+import (
+	"strconv"
+
+	"github.com/hashicorp/consul/agent/cache"
+	"github.com/hashicorp/consul/types"
+	"github.com/mitchellh/hashstructure"
+)
 
 // QueryDatacenterOptions sets options about how we fail over if there are no
 // healthy nodes in the local datacenter.
@@ -227,6 +233,39 @@ func (q *PreparedQueryExecuteRequest) RequestDatacenter() string {
 	return q.Datacenter
 }
 
+// CacheInfo implements cache.Request allowing requests to be cached on agent.
+func (q *PreparedQueryExecuteRequest) CacheInfo() cache.RequestInfo {
+	info := cache.RequestInfo{
+		Token:          q.Token,
+		Datacenter:     q.Datacenter,
+		MinIndex:       q.MinQueryIndex,
+		Timeout:        q.MaxQueryTime,
+		MaxAge:         q.MaxAge,
+		MustRevalidate: q.MustRevalidate,
+	}
+
+	// To calculate the cache key we hash over all the fields that affect the
+	// output other than Datacenter and Token which are dealt with in the cache
+	// framework already. Note the order here is important for the outcome - if we
+	// ever care about cache-invalidation on updates e.g. because we persist
+	// cached results, we need to be careful we maintain the same order of fields
+	// here. We could alternatively use `hash:set` struct tag on an anonymous
+	// struct to make it more robust if it becomes significant.
+	v, err := hashstructure.Hash([]interface{}{
+		q.QueryIDOrName,
+		q.Limit,
+		q.Connect,
+	}, nil)
+	if err == nil {
+		// If there is an error, we don't set the key. A blank key forces
+		// no cache for this request so the request is forwarded directly
+		// to the server.
+		info.Key = strconv.FormatUint(v, 10)
+	}
+
+	return info
+}
+
 // PreparedQueryExecuteRemoteRequest is used when running a local query in a
 // remote datacenter.
 type PreparedQueryExecuteRemoteRequest struct {

agent/structs/structs.go

@@ -119,10 +119,42 @@ type QueryOptions struct {
 	// servicing the request. Prevents a stale read.
 	RequireConsistent bool
 
+	// If set, the local agent may respond with an arbitrarily stale locally
+	// cached response. The semantics differ from AllowStale since the agent may
+	// be entirely partitioned from the servers and still considered "healthy" by
+	// operators. Stale responses from Servers are also arbitrarily stale, but can
+	// provide additional bounds on the last contact time from the leader. It's
+	// expected that servers that are partitioned are noticed and replaced in a
+	// timely way by operators while the same may not be true for client agents.
+	UseCache bool
+
 	// If set and AllowStale is true, will try first a stale
 	// read, and then will perform a consistent read if stale
-	// read is older than value
+	// read is older than value.
 	MaxStaleDuration time.Duration
+
+	// MaxAge limits how old a cached value will be returned if UseCache is true.
+	// If there is a cached response that is older than the MaxAge, it is treated
+	// as a cache miss and a new fetch invoked. If the fetch fails, the error is
+	// returned. Clients that wish to allow for stale results on error can set
+	// StaleIfError to a longer duration to change this behaviour. It is ignored
+	// if the endpoint supports background refresh caching. See
+	// https://www.consul.io/api/index.html#agent-caching for more details.
+	MaxAge time.Duration
+
+	// MustRevalidate forces the agent to fetch a fresh version of a cached
+	// resource or at least validate that the cached version is still fresh. It is
+	// implied by either max-age=0 or must-revalidate Cache-Control headers. It
+	// only makes sense when UseCache is true. We store it since MaxAge = 0 is the
+	// default unset value.
+	MustRevalidate bool
+
+	// StaleIfError specifies how stale the client will accept a cached response
+	// if the servers are unavailable to fetch a fresh one. Only makes sense when
+	// UseCache is true and MaxAge is set to a lower, non-zero value. It is
+	// ignored if the endpoint supports background refresh caching. See
+	// https://www.consul.io/api/index.html#agent-caching for more details.
+	StaleIfError time.Duration
 }
 
 // IsRead is always true for QueryOption.
@@ -283,10 +315,12 @@ func (r *DCSpecificRequest) RequestDatacenter() string {
 
 func (r *DCSpecificRequest) CacheInfo() cache.RequestInfo {
 	info := cache.RequestInfo{
-		Token:      r.Token,
-		Datacenter: r.Datacenter,
-		MinIndex:   r.MinQueryIndex,
-		Timeout:    r.MaxQueryTime,
+		Token:          r.Token,
+		Datacenter:     r.Datacenter,
+		MinIndex:       r.MinQueryIndex,
+		Timeout:        r.MaxQueryTime,
+		MaxAge:         r.MaxAge,
+		MustRevalidate: r.MustRevalidate,
 	}
 
 	// To calculate the cache key we only hash the node filters. The
@@ -327,6 +361,45 @@ func (r *ServiceSpecificRequest) RequestDatacenter() string {
 	return r.Datacenter
 }
 
+func (r *ServiceSpecificRequest) CacheInfo() cache.RequestInfo {
+	info := cache.RequestInfo{
+		Token:          r.Token,
+		Datacenter:     r.Datacenter,
+		MinIndex:       r.MinQueryIndex,
+		Timeout:        r.MaxQueryTime,
+		MaxAge:         r.MaxAge,
+		MustRevalidate: r.MustRevalidate,
+	}
+
+	// To calculate the cache key we hash over all the fields that affect the
+	// output other than Datacenter and Token which are dealt with in the cache
+	// framework already. Note the order here is important for the outcome - if we
+	// ever care about cache-invalidation on updates e.g. because we persist
+	// cached results, we need to be careful we maintain the same order of fields
+	// here. We could alternatively use `hash:set` struct tag on an anonymous
+	// struct to make it more robust if it becomes significant.
+	v, err := hashstructure.Hash([]interface{}{
+		r.NodeMetaFilters,
+		r.ServiceName,
+		r.ServiceTag,
+		r.ServiceAddress,
+		r.TagFilter,
+		r.Connect,
+	}, nil)
+	if err == nil {
+		// If there is an error, we don't set the key. A blank key forces
+		// no cache for this request so the request is forwarded directly
+		// to the server.
+		info.Key = strconv.FormatUint(v, 10)
+	}
+
+	return info
+}
+
+func (r *ServiceSpecificRequest) CacheMinIndex() uint64 {
+	return r.QueryOptions.MinQueryIndex
+}
+
 // NodeSpecificRequest is used to request the information about a single node
 type NodeSpecificRequest struct {
 	Datacenter string

api/api.go

@@ -78,6 +78,27 @@ type QueryOptions struct {
 	// read.
 	RequireConsistent bool
 
+	// UseCache requests that the agent cache results locally. See
+	// https://www.consul.io/api/index.html#agent-caching for more details on the
+	// semantics.
+	UseCache bool
+
+	// MaxAge limits how old a cached value will be returned if UseCache is true.
+	// If there is a cached response that is older than the MaxAge, it is treated
+	// as a cache miss and a new fetch invoked. If the fetch fails, the error is
+	// returned. Clients that wish to allow for stale results on error can set
+	// StaleIfError to a longer duration to change this behaviour. It is ignored
+	// if the endpoint supports background refresh caching. See
+	// https://www.consul.io/api/index.html#agent-caching for more details.
+	MaxAge time.Duration
+
+	// StaleIfError specifies how stale the client will accept a cached response
+	// if the servers are unavailable to fetch a fresh one. Only makes sense when
+	// UseCache is true and MaxAge is set to a lower, non-zero value. It is
+	// ignored if the endpoint supports background refresh caching. See
+	// https://www.consul.io/api/index.html#agent-caching for more details.
+	StaleIfError time.Duration
+
 	// WaitIndex is used to enable a blocking query. Waits
 	// until the timeout or the next index is reached
 	WaitIndex uint64
@@ -196,6 +217,13 @@ type QueryMeta struct {
 
 	// Is address translation enabled for HTTP responses on this agent
 	AddressTranslationEnabled bool
+
+	// CacheHit is true if the result was served from agent-local cache.
+	CacheHit bool
+
+	// CacheAge is set if request was ?cached and indicates how stale the cached
+	// response is.
+	CacheAge time.Duration
 }
 
 // WriteMeta is used to return meta data about a write
@@ -591,6 +619,20 @@ func (r *request) setQueryOptions(q *QueryOptions) {
 	if q.Connect {
 		r.params.Set("connect", "true")
 	}
+	if q.UseCache && !q.RequireConsistent {
+		r.params.Set("cached", "")
+
+		cc := []string{}
+		if q.MaxAge > 0 {
+			cc = append(cc, fmt.Sprintf("max-age=%.0f", q.MaxAge.Seconds()))
+		}
+		if q.StaleIfError > 0 {
+			cc = append(cc, fmt.Sprintf("stale-if-error=%.0f", q.StaleIfError.Seconds()))
+		}
+		if len(cc) > 0 {
+			r.header.Set("Cache-Control", strings.Join(cc, ", "))
+		}
+	}
 	r.ctx = q.ctx
 }
 
@@ -802,6 +844,18 @@ func parseQueryMeta(resp *http.Response, q *QueryMeta) error {
 		q.AddressTranslationEnabled = false
 	}
 
+	// Parse Cache info
+	if cacheStr := header.Get("X-Cache"); cacheStr != "" {
+		q.CacheHit = strings.EqualFold(cacheStr, "HIT")
+	}
+	if ageStr := header.Get("Age"); ageStr != "" {
+		age, err := strconv.ParseUint(ageStr, 10, 64)
+		if err != nil {
+			return fmt.Errorf("Failed to parse Age Header: %v", err)
+		}
+		q.CacheAge = time.Duration(age) * time.Second
+	}
+
 	return nil
 }
 

api/api_test.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/consul/testutil"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -381,6 +382,8 @@ func TestAPI_SetQueryOptions(t *testing.T) {
 	c, s := makeClient(t)
 	defer s.Stop()
 
+	assert := assert.New(t)
+
 	r := c.newRequest("GET", "/v1/kv/foo")
 	q := &QueryOptions{
 		Datacenter:        "foo",
@@ -414,6 +417,19 @@ func TestAPI_SetQueryOptions(t *testing.T) {
 	if r.params.Get("near") != "nodex" {
 		t.Fatalf("bad: %v", r.params)
 	}
+	assert.Equal("", r.header.Get("Cache-Control"))
+
+	r = c.newRequest("GET", "/v1/kv/foo")
+	q = &QueryOptions{
+		UseCache:     true,
+		MaxAge:       30 * time.Second,
+		StaleIfError: 345678 * time.Millisecond, // Fractional seconds should be rounded
+	}
+	r.setQueryOptions(q)
+
+	_, ok := r.params["cached"]
+	assert.True(ok)
+	assert.Equal("max-age=30, stale-if-error=346", r.header.Get("Cache-Control"))
 }
 
 func TestAPI_SetWriteOptions(t *testing.T) {

api/catalog_test.go

@@ -2,6 +2,9 @@ package api
 
 import (
 	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
 
 	"github.com/hashicorp/consul/testutil"
 	"github.com/hashicorp/consul/testutil/retry"
@@ -215,6 +218,45 @@ func TestAPI_CatalogService(t *testing.T) {
 	})
 }
 
+func TestAPI_CatalogServiceCached(t *testing.T) {
+	t.Parallel()
+	c, s := makeClient(t)
+	defer s.Stop()
+
+	catalog := c.Catalog()
+
+	q := &QueryOptions{
+		UseCache: true,
+	}
+
+	retry.Run(t, func(r *retry.R) {
+		services, meta, err := catalog.Service("consul", "", q)
+		if err != nil {
+			r.Fatal(err)
+		}
+
+		if meta.LastIndex == 0 {
+			r.Fatalf("Bad: %v", meta)
+		}
+
+		if len(services) == 0 {
+			r.Fatalf("Bad: %v", services)
+		}
+
+		if services[0].Datacenter != "dc1" {
+			r.Fatalf("Bad datacenter: %v", services[0])
+		}
+	})
+
+	require := require.New(t)
+
+	// Got success, next hit must be cache hit
+	_, meta, err := catalog.Service("consul", "", q)
+	require.NoError(err)
+	require.True(meta.CacheHit)
+	require.Equal(time.Duration(0), meta.CacheAge)
+}
+
 func TestAPI_CatalogService_NodeMetaFilter(t *testing.T) {
 	t.Parallel()
 	meta := map[string]string{"somekey": "somevalue"}
@@ -316,6 +358,7 @@ func TestAPI_CatalogConnect(t *testing.T) {
 			r.Fatalf("Returned port should be for proxy: %v", services[0])
 		}
 	})
+
 }
 
 func TestAPI_CatalogConnectNative(t *testing.T) {

website/source/api/acl.html.md

@@ -27,12 +27,13 @@ configuration files.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `none`       |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `none`       |
 
 ### Sample Request
 
@@ -68,12 +69,13 @@ This endpoint makes a new ACL token.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `management` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `management` |
 
 ### Parameters
 
@@ -126,12 +128,13 @@ generating a new token ID, the `ID` field must be provided.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `management` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `management` |
 
 ### Parameters
 
@@ -168,12 +171,13 @@ This endpoint deletes an ACL token with the given ID.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `management` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `management` |
 
 ### Parameters
 
@@ -198,12 +202,13 @@ This endpoint reads an ACL token with the given ID.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `YES`            | `all`             | `none`       |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `YES`            | `all`             | `none`        | `none`       |
 
 Note: No ACL is required because the ACL is specified in the URL path.
 
@@ -246,12 +251,13 @@ complex rule management.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `management` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `management` |
 
 ### Parameters
 
@@ -284,12 +290,13 @@ This endpoint lists all the active ACL tokens.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `YES`            | `all`             | `management` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `YES`            | `all`             | `none`        | `management` |
 
 ### Sample Request
 
@@ -328,12 +335,13 @@ section for more details.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `consistent`      | `none`       |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `consistent`      | `none`        | `none`       |
 
 ### Parameters
 

website/source/api/agent.html.md

@@ -31,12 +31,13 @@ by agent. The strongly consistent view of nodes is instead provided by
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `node:read`  |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `node:read`  |
 
 ### Parameters
 
@@ -96,12 +97,13 @@ to change without notice or deprecation.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `agent:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `agent:read` |
 
 ### Sample Request
 
@@ -175,12 +177,13 @@ section on the agent options page for details on which options are supported.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required  |
-| ---------------- | ----------------- | ------------- |
-| `NO`             | `none`            | `agent:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required  |
+| ---------------- | ----------------- | ------------- | ------------- |
+| `NO`             | `none`            | `none`        | `agent:write` |
 
 ### Sample Request
 
@@ -205,12 +208,13 @@ restart.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `node:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `node:write` |
 
 ### Parameters
 
@@ -234,14 +238,6 @@ $ curl \
 
 ## View Metrics
 
-This endpoint returns the configuration and member information of the local
-agent.
-
-| Method | Path                               | Produces                                   |
-| ------ | ---------------------------------- | ------------------------------------------ |
-| `GET`  | `/agent/metrics`                   | `application/json`                         |
-| `GET`  | `/agent/metrics?format=prometheus` | `text/plain; version=0.0.4; charset=utf-8` |
-
 This endpoint will dump the metrics for the most recent finished interval.
 For more information about metrics, see the [telemetry](/docs/agent/telemetry.html)
 page.
@@ -250,9 +246,20 @@ In order to enable [Prometheus](https://prometheus.io/) support, you need to use
 configuration directive
 [`prometheus_retention_time`](/docs/agent/options.html#telemetry-prometheus_retention_time).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `agent:read` |
+| Method | Path                               | Produces                                   |
+| ------ | ---------------------------------- | ------------------------------------------ |
+| `GET`  | `/agent/metrics`                   | `application/json`                         |
+| `GET`  | `/agent/metrics?format=prometheus` | `text/plain; version=0.0.4; charset=utf-8` |
+
+The table below shows this endpoint's support for
+[blocking queries](/api/index.html#blocking-queries),
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
+[required ACLs](/api/index.html#acls).
+
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `agent:read` |
 
 ### Sample Request
 
@@ -368,12 +375,13 @@ This endpoint streams logs from the local agent until the connection is closed.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `agent:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `agent:read` |
 
 ### Parameters
 
@@ -409,12 +417,13 @@ This endpoint instructs the agent to attempt to connect to a given address.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required  |
-| ---------------- | ----------------- | ------------- |
-| `NO`             | `none`            | `agent:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required  |
+| ---------------- | ----------------- | ------------- | ------------- |
+| `NO`             | `none`            | `none`        | `agent:write` |
 
 ### Parameters
 
@@ -448,12 +457,13 @@ can affect cluster availability.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required  |
-| ---------------- | ----------------- | ------------- |
-| `NO`             | `none`            | `agent:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required  |
+| ---------------- | ----------------- | ------------- | ------------- |
+| `NO`             | `none`            | `none`        | `agent:write` |
 
 ### Sample Request
 
@@ -477,12 +487,13 @@ state allows its old entries to be removed.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required  |
-| ---------------- | ----------------- | ------------- |
-| `NO`             | `none`            | `agent:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required  |
+| ---------------- | ----------------- | ------------- | ------------- |
+| `NO`             | `none`            | `none`        | `agent:write` |
 
 ### Sample Request
 
@@ -513,12 +524,13 @@ The paths above correspond to the token names as found in the agent configuratio
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required  |
-| ---------------- | ----------------- | ------------- |
-| `NO`             | `none`            | `agent:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required  |
+| ---------------- | ----------------- | ------------- | ------------- |
+| `NO`             | `none`            | `none`        | `agent:write` |
 
 ### Parameters
 

website/source/api/agent/check.html.md

@@ -29,12 +29,13 @@ everything will be in sync within a few seconds.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required             |
-| ---------------- | ----------------- | ------------------------ |
-| `NO`             | `none`            | `node:read,service:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required             |
+| ---------------- | ----------------- | ------------- | ------------------------ |
+| `NO`             | `none`            | `none`        | `node:read,service:read` |
 
 ### Sample Request
 
@@ -73,12 +74,13 @@ check and keeping the Catalog in sync.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required               |
-| ---------------- | ----------------- | -------------------------- |
-| `NO`             | `none`            | `node:write,service:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required               |
+| ---------------- | ----------------- | ------------- | -------------------------- |
+| `NO`             | `none`            | `none`        | `node:write,service:write` |
 
 ### Parameters
 
@@ -219,12 +221,13 @@ not exist, no action is taken.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required               |
-| ---------------- | ----------------- | -------------------------- |
-| `NO`             | `none`            | `node:write,service:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required               |
+| ---------------- | ----------------- | ------------- | -------------------------- |
+| `NO`             | `none`            | `none`        | `node:write,service:write` |
 
 ### Parameters
 
@@ -250,12 +253,13 @@ This endpoint is used with a TTL type check to set the status of the check to
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required               |
-| ---------------- | ----------------- | -------------------------- |
-| `NO`             | `none`            | `node:write,service:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required               |
+| ---------------- | ----------------- | ------------- | -------------------------- |
+| `NO`             | `none`            | `none`        | `node:write,service:write` |
 
 ### Parameters
 
@@ -283,12 +287,13 @@ This endpoint is used with a TTL type check to set the status of the check to
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required               |
-| ---------------- | ----------------- | -------------------------- |
-| `NO`             | `none`            | `node:write,service:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required               |
+| ---------------- | ----------------- | ------------- | -------------------------- |
+| `NO`             | `none`            | `none`        | `node:write,service:write` |
 
 ### Parameters
 
@@ -316,12 +321,13 @@ This endpoint is used with a TTL type check to set the status of the check to
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required               |
-| ---------------- | ----------------- | -------------------------- |
-| `NO`             | `none`            | `node:write,service:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required               |
+| ---------------- | ----------------- | ------------- | -------------------------- |
+| `NO`             | `none`            | `none`        | `node:write,service:write` |
 
 ### Parameters
 
@@ -349,12 +355,13 @@ to reset the TTL clock.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required               |
-| ---------------- | ----------------- | -------------------------- |
-| `NO`             | `none`            | `node:write,service:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required               |
+| ---------------- | ----------------- | ------------- | -------------------------- |
+| `NO`             | `none`            | `none`        | `node:write,service:write` |
 
 ### Parameters
 

website/source/api/agent/connect.html.md

@@ -36,12 +36,13 @@ connection attempt.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required             |
-| ---------------- | ----------------- | ------------------------ |
-| `NO`             | `none`            | `service:write` |
+| Blocking Queries | Consistency Modes | Agent Caching        | ACL Required    |
+| ---------------- | ----------------- | -------------------- | --------------- |
+| `NO`             | `none`            | `background refresh` | `service:write` |
 
 ### Parameters
 
@@ -102,12 +103,13 @@ unavailable. This endpoint should be used by proxies and native integrations.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required               |
-| ---------------- | ----------------- | -------------------------- |
-| `YES`             | `all`            | `none` |
+| Blocking Queries | Consistency Modes | Agent Caching        | ACL Required |
+| ---------------- | ----------------- | -------------------- | ------------ |
+| `YES`            | `all`             | `background refresh` | `none`       |
 
 ### Sample Request
 
@@ -162,12 +164,13 @@ clients to efficiently wait for certificate rotations.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required               |
-| ---------------- | ----------------- | -------------------------- |
-| `YES`             | `all`            | `service:write` |
+| Blocking Queries | Consistency Modes | Agent Caching        | ACL Required    |
+| ---------------- | ----------------- | -------------------- | --------------- |
+| `YES`            | `all`             | `background refresh` | `service:write` |
 
 ### Parameters
 
@@ -229,16 +232,17 @@ a blocking query to detect any configuration changes.
 
 | Method | Path                         | Produces                   |
 | ------ | ---------------------------- | -------------------------- |
-| `GET`  | `/agent/connect/proxy/:id`    | `application/json`         |
+| `GET`  | `/agent/connect/proxy/:id`    | `application/json`        |
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required               |
-| ---------------- | ----------------- | -------------------------- |
-| `YES`             | `all`            | `service:write, proxy token` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required                 |
+| ---------------- | ----------------- | ------------- | ---------------------------- |
+| `YES`            | `all`             | `none`        | `service:write, proxy token` |
 
 ### Parameters
 

website/source/api/agent/service.html.md

@@ -30,12 +30,13 @@ everything will be in sync within a few seconds.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required   |
-| ---------------- | ----------------- | -------------- |
-| `NO`             | `none`            | `service:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required   |
+| ---------------- | ----------------- | ------------- | -------------- |
+| `NO`             | `none`            | `none`        | `service:read` |
 
 ### Sample Request
 
@@ -79,12 +80,13 @@ For "connect-proxy" kind services, the `service:write` ACL for the
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `NO`             | `none`            | `service:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `NO`             | `none`            | `none`        | `service:write` |
 
 ### Parameters
 
@@ -216,12 +218,13 @@ is an associated check, that is also deregistered.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `NO`             | `none`            | `service:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `NO`             | `none`            | `none`        | `service:write` |
 
 ### Parameters
 
@@ -249,12 +252,13 @@ will be automatically restored on agent restart.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `NO`             | `none`            | `service:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `NO`             | `none`            | `none`        | `service:write` |
 
 ### Parameters
 

website/source/api/catalog.html.md

@@ -26,12 +26,13 @@ perform [anti-entropy](/docs/internals/anti-entropy.html).
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required               |
-| ---------------- | ----------------- | -------------------------- |
-| `NO`             | `none`            | `node:write,service:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required              |
+| ---------------- | ----------------- | ------------- | ------------------------- |
+| `NO`             | `none`            | `none`        |`node:write,service:write` |
 
 ### Parameters
 
@@ -150,12 +151,13 @@ perform [anti-entropy](/docs/internals/anti-entropy.html).
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required               |
-| ---------------- | ----------------- | -------------------------- |
-| `NO`             | `none`            | `node:write,service:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required               |
+| ---------------- | ----------------- | ------------- | -------------------------- |
+| `NO`             | `none`            | `none`        | `node:write,service:write` |
 
 ### Parameters
 
@@ -223,12 +225,13 @@ Consul servers are routable.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `none`       |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `none`       |
 
 ### Sample Request
 
@@ -253,12 +256,13 @@ This endpoint and returns the nodes registered in a given datacenter.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `YES`            | `all`             | `node:read`  |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `YES`            | `all`             | `none`        | `node:read`  |
 
 ### Parameters
 
@@ -326,12 +330,13 @@ This endpoint returns the services registered in a given datacenter.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required   |
-| ---------------- | ----------------- | -------------- |
-| `YES`            | `all`             | `service:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required   |
+| ---------------- | ----------------- | ------------- | -------------- |
+| `YES`            | `all`             | `none`        | `service:read` |
 
 ### Parameters
 
@@ -377,12 +382,13 @@ This endpoint returns the nodes providing a service in a given datacenter.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required             |
-| ---------------- | ----------------- | ------------------------ |
-| `YES`            | `all`             | `node:read,service:read` |
+| Blocking Queries | Consistency Modes | Agent Caching        | ACL Required             |
+| ---------------- | ----------------- | -------------------- | ------------------------ |
+| `YES`            | `all`             | `background refresh` | `node:read,service:read` |
 
 ### Parameters
 
@@ -511,17 +517,15 @@ This endpoint returns the node's registered services.
 | ------ | ---------------------------- | -------------------------- |
 | `GET`  | `/catalog/node/:node`        | `application/json`         |
 
-The table below shows this endpoint's support for blocking queries and
-consistency modes.
-
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required             |
-| ---------------- | ----------------- | ------------------------ |
-| `YES`            | `all`             | `node:read,service:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required             |
+| ---------------- | ----------------- | ------------- | ------------------------ |
+| `YES`            | `all`             | `none`        | `node:read,service:read` |
 
 ### Parameters
 

website/source/api/connect/ca.html.md

@@ -23,12 +23,13 @@ the cluster.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required     |
-| ---------------- | ----------------- | ---------------- |
-| `YES`            | `all`             | `operator:read`  |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required     |
+| ---------------- | ----------------- | ------------- | ---------------- |
+| `YES`            | `all`             | `none`        | `operator:read`  |
 
 ### Sample Request
 
@@ -71,12 +72,13 @@ This endpoint returns the current CA configuration.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `YES`            | `all`             | `operator:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `YES`            | `all`             | `none`        | `operator:read` |
 
 ### Sample Request
 
@@ -111,12 +113,13 @@ new root certificate being used, the [Root Rotation]
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `NO`             | `none`            | `operator:write`|
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `NO`             | `none`            | `none`        | `operator:write`|
 
 ### Parameters
 

website/source/api/connect/intentions.html.md

@@ -27,12 +27,13 @@ existing intention or delete it prior to creating a new one.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required                   |
-| ---------------- | ----------------- | ------------------------------ |
-| `NO`             | `none`            | `intentions:write`<sup>1</sup> |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required                    |
+| ---------------- | ----------------- | ------------- | ------------------------------- |
+| `NO`             | `none`            | `none`        | `intentions:write`<sup>1</sup>  |
 
 <sup>1</sup> Intention ACL rules are specified as part of a `service` rule.
 See [Intention Management Permissions](/docs/connect/intentions.html#intention-management-permissions) for more details.
@@ -97,12 +98,13 @@ This endpoint reads a specific intention.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `YES`            | `all`             | `intentions:read`<sup>1</sup> |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `YES`            | `all`             | `none`        | `intentions:read`<sup>1</sup> |
 
 <sup>1</sup> Intention ACL rules are specified as part of a `service` rule.
 See [Intention Management Permissions](/docs/connect/intentions.html#intention-management-permissions) for more details.
@@ -152,12 +154,13 @@ This endpoint lists all intentions.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `YES`            | `all`             | `intentions:read`<sup>1</sup> |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `YES`            | `all`             | `none`        | `intentions:read`<sup>1</sup> |
 
 <sup>1</sup> Intention ACL rules are specified as part of a `service` rule.
 See [Intention Management Permissions](/docs/connect/intentions.html#intention-management-permissions) for more details.
@@ -204,12 +207,13 @@ This endpoint updates an intention with the given values.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required     |
-| ---------------- | ----------------- | ---------------- |
-| `NO`             | `none`            | `intentions:write`<sup>1</sup> |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required     |
+| ---------------- | ----------------- | ------------- | ---------------- |
+| `NO`             | `none`            | `none`        | `intentions:write`<sup>1</sup> |
 
 
 <sup>1</sup> Intention ACL rules are specified as part of a `service` rule.
@@ -252,12 +256,13 @@ This endpoint deletes a specific intention.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required     |
-| ---------------- | ----------------- | ---------------- |
-| `NO`             | `none`            | `intentions:write`<sup>1</sup> |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `NO`             | `none`            | `none`        | `intentions:write`<sup>1</sup> |
 
 
 <sup>1</sup> Intention ACL rules are specified as part of a `service` rule.
@@ -293,12 +298,13 @@ does not contain any information about the intention itself.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required     |
-| ---------------- | ----------------- | ---------------- |
-| `NO`             | `none`            | `intentions:read`<sup>1</sup> |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required   |
+| ---------------- | ----------------- | ------------- | -------------- |
+| `NO`             | `none`            | `none`        | `intentions:read`<sup>1</sup> |
 
 <sup>1</sup> Intention ACL rules are specified as part of a `service` rule.
 See [Intention Management Permissions](/docs/connect/intentions.html#intention-management-permissions) for more details.
@@ -339,12 +345,13 @@ The intentions in the response are in evaluation order.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `NO`             | `none`            | `intentions:read`<sup>1</sup> |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required   |
+| ---------------- | ----------------- | ------------- | -------------- |
+| `NO`             | `none`            | `none`        | `intentions:read`<sup>1</sup> |
 
 <sup>1</sup> Intention ACL rules are specified as part of a `service` rule.
 See [Intention Management Permissions](/docs/connect/intentions.html#intention-management-permissions) for more details.

website/source/api/coordinate.html.md

@@ -31,12 +31,13 @@ cluster.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `none`       |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `none`       |
 
 ### Sample Request
 
@@ -82,12 +83,13 @@ datacenter.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `YES`            | `all`             | `node:read`  |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `YES`            | `all`             | `none`        | `node:read`  |
 
 ### Parameters
 
@@ -138,12 +140,13 @@ This endpoint returns the LAN network coordinates for the given node.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `YES`            | `all`             | `node:read`  |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `YES`            | `all`             | `none`        | `node:read`  |
 
 ### Parameters
 
@@ -195,12 +198,13 @@ datacenter.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `node:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `node:write` |
 
 ### Parameters
 

website/source/api/event.html.md

@@ -22,12 +22,13 @@ This endpoint triggers a new user event.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required  |
-| ---------------- | ----------------- | ------------- |
-| `NO`             | `none`            | `event:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required  |
+| ---------------- | ----------------- | ------------- | ------------- |
+| `NO`             | `none`            | `none`        | `event:write` |
 
 ### Parameters
 
@@ -97,12 +98,13 @@ nor do they make a promise of delivery.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `YES`            | `none`            | `event:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `YES`            | `none`            | `none`        | `event:read` |
 
 ### Parameters
 

website/source/api/health.html.md

@@ -25,12 +25,13 @@ This endpoint returns the checks specific to the node provided on the path.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required             |
-| ---------------- | ----------------- | ------------------------ |
-| `YES`            | `all`             | `node:read,service:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required             |
+| ---------------- | ----------------- | ------------- | ------------------------ |
+| `YES`            | `all`             | `none`        | `node:read,service:read` |
 
 ### Parameters
 
@@ -90,12 +91,13 @@ path.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required             |
-| ---------------- | ----------------- | ------------------------ |
-| `YES`            | `all`             | `node:read,service:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required             |
+| ---------------- | ----------------- | ------------- | ------------------------ |
+| `YES`            | `all`             | `none`        | `node:read,service:read` |
 
 ### Parameters
 
@@ -153,12 +155,13 @@ incorporating the use of health checks.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required             |
-| ---------------- | ----------------- | ------------------------ |
-| `YES`            | `all`             | `node:read,service:read` |
+| Blocking Queries | Consistency Modes | Agent Caching        | ACL Required             |
+| ---------------- | ----------------- | -------------------- | ------------------------ |
+| `YES`            | `all`             | `background refresh` | `node:read,service:read` |
 
 ### Parameters
 
@@ -274,12 +277,13 @@ This endpoint returns the checks in the state provided on the path.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required             |
-| ---------------- | ----------------- | ------------------------ |
-| `YES`            | `all`             | `node:read,service:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required             |
+| ---------------- | ----------------- | ------------- | ------------------------ |
+| `YES`            | `all`             | `none`        | `node:read,service:read` |
 
 ### Parameters
 

website/source/api/index.html.md

@@ -109,12 +109,105 @@ The three read modes are:
 To switch these modes, either the `stale` or `consistent` query parameters
 should be provided on requests. It is an error to provide both.
 
+Note that some endpoints support a `cached` parameter which has some of the same
+semantics as `stale` but different trade offs. This behaviour is described in
+[Agent Caching](#agent-caching).
+
 To support bounding the acceptable staleness of data, responses provide the
 `X-Consul-LastContact` header containing the time in milliseconds that a server
 was last contacted by the leader node. The `X-Consul-KnownLeader` header also
 indicates if there is a known leader. These can be used by clients to gauge the
 staleness of a result and take appropriate action.
 
+## Agent Caching
+
+Some read endpoints support agent caching. They are clearly marked in the
+documentation. Agent caching can take two forms, [`simple`](#simple-caching) or 
+[`background refresh`](#blocking-refresh-caching) depending on the endpoint's 
+semantics. The documentation for each endpoint clearly identify which if any 
+form of caching is supported. The details for each are described below.
+
+Where supported, caching can be enabled though the `?cached` parameter.
+Combining `?cached` with `?consistent` is an error.
+
+### Simple Caching
+
+Endpoints supporting simple caching may return a result directly from the local
+agent's cache without a round trip to the servers. By default the agent caches
+results for a relatively long time (3 days) such that it can still return a
+result even if the servers are unavailable for an extended period to enable
+"fail static" semantics.
+
+That means that with no other arguments, `?cached` queries might receive a
+response which is days old. To request better freshness, the HTTP
+`Cache-Control` header may be set with a directive like `max-age=<seconds>`. In
+this case the agent will attempt to re-fetch the result from the servers if the
+cached value is older than the given `max-age`. If the servers can't be reached
+a 500 is returned as normal.
+
+To allow clients to maintain fresh results in normal operation but allow stale
+ones if the servers are unavailable, the `stale-if-error=<seconds>` directive
+may be additionally provided in the `Cache-Control` header. This will return the
+cached value anyway even it it's older than `max-age` (provided it's not older
+than `stale-if-error`) rather than a 500. It must be provided along with a
+`max-age` or `must-revalidate`. The `Age` response header, if larger than
+`max-age` can be used to determine if the server was unreachable and a cached
+version returned instead.
+
+For example, assuming there is a cached response that is 65 seconds old, and
+that the servers are currently unavailable, `Cache-Control: max-age=30` will
+result in a 500 error, while `Cache-Control: max-age=30 stale-if-error=259200`
+will result in the cached response being returned.
+
+A request setting either `max-age=0` or `must-revalidate` directives will cause
+the agent to always re-fetch the response from servers. Either can be combined
+with `stale-if-error=<seconds>` to ensure fresh results when the servers are
+available, but falling back to cached results if the request to the servers
+fails.
+
+Requests that do not use `?cached` currently bypass the cache entirely so the
+cached response returned might be more stale than the last uncached response
+returned on the same agent. If this causes problems, it is possible to make
+requests using `?cached` and setting `Cache-Control: must-revalidate` to have
+always-fresh results yet keeping the cache populated with the most recent
+result.
+
+In all cases the HTTP `X-Cache` header is always set in the response to either
+`HIT` or `MISS` indicating whether the response was served from cache or not.
+
+For cache hits, the HTTP `Age` header is always set in the response to indicate
+how many seconds since that response was fetched from the servers.
+
+### Background Refresh Caching
+
+Endpoints supporting background refresh caching may return a result directly
+from the local agent's cache without a round trip to the severs. The first fetch
+that is a miss will cause an initial fetch from the servers, but will also
+trigger the agent to begin a background blocking query that watches for any
+changes to that result and updates the cached value if changes occur.
+
+Following requests will _always_ be a cache hit until there has been no request
+for the resource for the TTL (which is typically 3 days).
+
+Clients can perform blocking queries against the local agent which will be
+served from the cache. This allows multiple clients to watch the same resource
+locally while only a single blocking watch for that resource will be made to the
+servers from a given client agent.
+
+HTTP `Cache-Control` headers are ignored in this mode since the cache is being
+actively updated and has different semantics to a typical passive cache.
+
+In all cases the HTTP `X-Cache` header is always set in the response to either
+`HIT` or `MISS` indicating whether the response was served from cache or not.
+
+For cache hits, the HTTP `Age` header is always set in the response to indicate
+how many seconds since that response was fetched from the servers. As long as
+the local agent has an active connection to the servers, the age will always be
+`0` since the value is up-to-date. If the agent get's disconnected, the cached
+result is still returned but with an `Age` that indicates how many seconds have
+elapsed since the local agent got disconnected from the servers, during which
+time updates to the result might have been missed.
+
 ## Formatted JSON Output
 
 By default, the output of all HTTP API requests is minimized JSON. If the client

website/source/api/kv.html.md

@@ -34,12 +34,13 @@ For multi-key reads, please consider using [transaction](/api/txn.html).
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `YES`            | `all`             | `key:read`   |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `YES`            | `all`             | `none`        | `key:read`   |
 
 ### Parameters
 
@@ -154,12 +155,13 @@ Even though the return type is `application/json`, the value is either `true` or
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `key:write`  |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `key:write`  |
 
 ### Parameters
 
@@ -234,12 +236,13 @@ This endpoint deletes a single key or all keys sharing a prefix.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `key:write`  |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `key:write`  |
 
 ### Parameters
 

website/source/api/operator/area.html.md

@@ -39,12 +39,13 @@ successfully.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required     |
-| ---------------- | ----------------- | ---------------- |
-| `NO`             | `none`            | `operator:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required     |
+| ---------------- | ----------------- | ------------- | ---------------- |
+| `NO`             | `none`            | `none`        | `operator:write` |
 
 ### Parameters
 
@@ -104,12 +105,13 @@ This endpoint lists all network areas.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `YES`            | `all`             | `operator:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `YES`            | `all`             | `none`        | `operator:read` |
 
 ### Parameters
 
@@ -146,12 +148,13 @@ This endpoint updates a network area to the given configuration.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required     |
-| ---------------- | ----------------- | ---------------- |
-| `NO`             | `none`            | `operator:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required     |
+| ---------------- | ----------------- | ------------- | ---------------- |
+| `NO`             | `none`            | `none`        | `operator:write` |
 
 ### Parameters
 
@@ -189,12 +192,13 @@ This endpoint lists a specific network area.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `YES`            | `all`             | `operator:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `YES`            | `all`             | `none`        | `operator:read` |
 
 ### Parameters
 
@@ -234,12 +238,13 @@ This endpoint deletes a specific network area.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required     |
-| ---------------- | ----------------- | ---------------- |
-| `NO`             | `none`            | `operator:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required     |
+| ---------------- | ----------------- | ------------- | ---------------- |
+| `NO`             | `none`            | `none`        | `operator:write` |
 
 ### Parameters
 
@@ -269,12 +274,13 @@ area.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required     |
-| ---------------- | ----------------- | ---------------- |
-| `NO`             | `none`            | `operator:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required     |
+| ---------------- | ----------------- | ------------- | ---------------- |
+| `NO`             | `none`            | `none`        | `operator:write` |
 
 ### Parameters
 
@@ -341,12 +347,13 @@ network area.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `NO`             | `none`            | `operator:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `NO`             | `none`            | `none`        | `operator:read` |
 
 ### Parameters
 

website/source/api/operator/autopilot.html.md

@@ -26,12 +26,13 @@ This endpoint retrieves its latest Autopilot configuration.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `NO`             | `none`            | `operator:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `NO`             | `none`            | `none`        | `operator:read` |
 
 ### Parameters
 
@@ -79,12 +80,13 @@ This endpoint updates the Autopilot configuration of the cluster.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required     |
-| ---------------- | ----------------- | ---------------- |
-| `NO`             | `none`            | `operator:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required     |
+| ---------------- | ----------------- | ------------- | ---------------- |
+| `NO`             | `none`            | `none`        | `operator:write` |
 
 ### Parameters
 
@@ -151,12 +153,13 @@ This endpoint queries the health of the autopilot status.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `NO`             | `none`            | `operator:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `NO`             | `none`            | `none`        | `operator:read` |
 
 ### Parameters
 

website/source/api/operator/keyring.html.md

@@ -27,12 +27,13 @@ read privileges.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required   |
-| ---------------- | ----------------- | -------------- |
-| `NO`             | `none`            | `keyring:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required   |
+| ---------------- | ----------------- | ------------- | -------------- |
+| `NO`             | `none`            | `none`        | `keyring:read` |
 
 ### Parameters
 
@@ -99,12 +100,13 @@ This endpoint installs a new gossip encryption key into the cluster.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `NO`             | `none`            | `keyring:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `NO`             | `none`            | `none`        | `keyring:write` |
 
 ### Parameters
 
@@ -144,12 +146,13 @@ installed before this operation can succeed.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `NO`             | `none`            | `keyring:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `NO`             | `none`            | `none`        | `keyring:write` |
 
 ### Parameters
 
@@ -189,12 +192,13 @@ may only be performed on keys which are not currently the primary key.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `NO`             | `none`            | `keyring:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `NO`             | `none`            | `none`        | `keyring:write` |
 
 ### Parameters
 

website/source/api/operator/license.html.md

@@ -25,12 +25,13 @@ This endpoint gets information about the current license.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required     |
-| ---------------- | ----------------- | ---------------- |
-| `NO`             | `all`             | `none`            |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required     |
+| ---------------- | ----------------- | ------------- | ---------------- |
+| `NO`             | `all`             | `none`        | `none`           |
 
 ### Parameters
 
@@ -86,12 +87,13 @@ license contents as well as any warning messages regarding its validity.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required     |
-| ---------------- | ----------------- | ---------------- |
-| `NO`             | `none`            | `operator:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required     |
+| ---------------- | ----------------- | ------------- | ---------------- |
+| `NO`             | `none`            | `none`        | `operator:write` |
 
 ### Parameters
 

website/source/api/operator/raft.html.md

@@ -25,12 +25,13 @@ This endpoint reads the current raft configuration.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes     | ACL Required    |
-| ---------------- | --------------------- | --------------- |
-| `NO`             | `default` and `stale` | `operator:read` |
+| Blocking Queries | Consistency Modes     | Agent Caching | ACL Required    |
+| ---------------- | --------------------- | ------------- | --------------- |
+| `NO`             | `default` and `stale` | `none`        | `operator:read` |
 
 ### Parameters
 
@@ -122,12 +123,13 @@ write privileges.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required     |
-| ---------------- | ----------------- | ---------------- |
-| `NO`             | `none`            | `operator:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required     |
+| ---------------- | ----------------- | ------------- | ---------------- |
+| `NO`             | `none`            | `none`        | `operator:write` |
 
 ### Parameters
 

website/source/api/operator/segment.html.md

@@ -32,12 +32,13 @@ This endpoint lists all network areas.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `NO`             | `none`            | `operator:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `NO`             | `none`            | `none`        | `operator:read` |
 
 ### Parameters
 

website/source/api/query.html.md

@@ -142,12 +142,13 @@ successfully.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required  |
-| ---------------- | ----------------- | ------------- |
-| `NO`             | `none`            | `query:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required  |
+| ---------------- | ----------------- | ------------- | ------------- |
+| `NO`             | `none`            | `none`        | `query:write` |
 
 ### Parameters
 
@@ -297,12 +298,13 @@ This endpoint returns a list of all prepared queries.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `query:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `query:read` |
 
 ### Parameters
 
@@ -358,12 +360,13 @@ given ID, an error is returned.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required  |
-| ---------------- | ----------------- | ------------- |
-| `NO`             | `none`            | `query:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required  |
+| ---------------- | ----------------- | ------------- | ------------- |
+| `NO`             | `none`            | `none`        | `query:write` |
 
 ### Parameters
 
@@ -397,12 +400,13 @@ given ID, an error is returned.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `query:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `query:read` |
 
 ### Parameters
 
@@ -436,12 +440,13 @@ given ID, an error is returned.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required  |
-| ---------------- | ----------------- | ------------- |
-| `NO`             | `none`            | `query:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required  |
+| ---------------- | ----------------- | ------------- | ------------- |
+| `NO`             | `none`            | `none`        | `query:write` |
 
 ### Parameters
 
@@ -471,12 +476,13 @@ given ID, an error is returned.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `depends`<sup>1</sup>    |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `simple`      | `depends`<sup>1</sup>    |
 
 <sup>1</sup> If an ACL Token was bound to the query when it was defined then it
 will be used when executing the request. Otherwise, the client's supplied ACL
@@ -598,12 +604,13 @@ interpolation.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `query:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `query:read` |
 
 ### Parameters
 

website/source/api/session.html.md

@@ -21,12 +21,13 @@ node and may be associated with any number of checks.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `NO`             | `none`            | `session:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `NO`             | `none`            | `none`        | `session:write` |
 
 ### Parameters
 
@@ -106,12 +107,13 @@ successful.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `NO`             | `none`            | `session:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `NO`             | `none`            | `none`        | `session:write` |
 
 ### Parameters
 
@@ -146,12 +148,13 @@ This endpoint returns the requested session information.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required   |
-| ---------------- | ----------------- | -------------- |
-| `YES`            | `all`             | `session:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required   |
+| ---------------- | ----------------- | ------------- | -------------- |
+| `YES`            | `all`             | `none`        | `session:read` |
 
 ### Parameters
 
@@ -197,12 +200,13 @@ This endpoint returns the active sessions for a given node.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required   |
-| ---------------- | ----------------- | -------------- |
-| `YES`            | `all`             | `session:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required   |
+| ---------------- | ----------------- | ------------- | -------------- |
+| `YES`            | `all`             | `none`        | `session:read` |
 
 ### Parameters
 
@@ -246,12 +250,13 @@ This endpoint returns the list of active sessions.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required   |
-| ---------------- | ----------------- | -------------- |
-| `YES`            | `all`             | `session:read` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required   |
+| ---------------- | ----------------- | ------------- | -------------- |
+| `YES`            | `all`             | `none`        | `session:read` |
 
 ### Parameters
 
@@ -293,12 +298,13 @@ TTL, and it extends the expiration by the TTL.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required    |
-| ---------------- | ----------------- | --------------- |
-| `NO`             | `none`            | `session:write` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required    |
+| ---------------- | ----------------- | ------------- | --------------- |
+| `NO`             | `none`            | `none`        | `session:write` |
 
 ### Parameters
 

website/source/api/snapshot.html.md

@@ -32,12 +32,13 @@ restore.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `default,stale`   | `management` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `default,stale`   | `none`        | `management` |
 
 ### Parameters
 
@@ -86,12 +87,13 @@ call to the `GET` method.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `management` |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `management` |
 ### Parameters
 
 - `dc` `(string: "")` - Specifies the datacenter to query. This will default

website/source/api/status.html.md

@@ -25,12 +25,13 @@ running.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `none`       |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `none`       |
 
 ### Sample Request
 
@@ -56,12 +57,13 @@ determining when a given server has successfully joined the cluster.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `none`            | `none`       |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `none`            | `none`        | `none`       |
 
 ### Sample Request
 

website/source/api/txn.html.md

@@ -37,12 +37,13 @@ the leader via the Raft consensus protocol.
 
 The table below shows this endpoint's support for
 [blocking queries](/api/index.html#blocking-queries),
-[consistency modes](/api/index.html#consistency-modes), and
+[consistency modes](/api/index.html#consistency-modes),
+[agent caching](/api/index.html#agent-caching), and
 [required ACLs](/api/index.html#acls).
 
-| Blocking Queries | Consistency Modes | ACL Required |
-| ---------------- | ----------------- | ------------ |
-| `NO`             | `all`<sup>1</sup> | `key:read,key:write`<sup>2</sup> |
+| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
+| ---------------- | ----------------- | ------------- | ------------ |
+| `NO`             | `all`<sup>1</sup> | `none`        | `key:read,key:write`<sup>2</sup> |
 
 <sup>1</sup> For read-only transactions
 <br>

website/source/docs/internals/architecture.html.md

@@ -123,6 +123,15 @@ situations where a limited subset of data can be replicated, such as with Consul
 [ACL replication](/docs/guides/acl.html#outages-and-acl-replication) capability, or
 external tools like [consul-replicate](https://github.com/hashicorp/consul-replicate).
 
+In some places, client agents may cache data from the servers to make it
+available locally for performance and reliability. Examples include Connect
+certificates and intentions which allow the client agent to make local decisions
+about inbound connection requests without a round trip to the servers. Some API
+endpoints also support optional result caching. This helps reliability because
+the local agent can continue to respond to some queries like service-discovery
+or Connect authorization from cache even if the connection to the servers is
+disrupted or the servers are temporarily unavailable.
+
 ## Getting in depth
 
 At this point we've covered the high level architecture of Consul, but there are many