From 0c39b613c85951cfc0655ce31e44eeca22cab52a Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Thu, 16 Feb 2017 15:15:02 -0500 Subject: [PATCH] Port some replication bits to OSS (#2386) --- .travis.yml | 2 +- Makefile | 5 + README.md | 6 +- .../cassandra/test-fixtures/cassandra.yaml | 8 +- helper/consts/error.go | 13 ++ http/handler.go | 57 +------ http/handler_test.go | 9 +- http/help.go | 2 +- http/logical.go | 33 +--- http/logical_test.go | 2 +- http/sys_audit_test.go | 2 + http/sys_auth_test.go | 8 + http/sys_mount_test.go | 44 ++++++ http/sys_rekey.go | 10 +- http/sys_seal.go | 3 +- logical/connection.go | 2 +- logical/error.go | 24 +++ logical/response_util.go | 111 +++++++++++++ scripts/cross/Dockerfile | 2 +- vault/cluster_test.go | 3 +- vault/core.go | 22 +-- vault/core_test.go | 5 +- vault/generate_root.go | 21 +-- vault/rekey.go | 41 ++--- vault/request_handling.go | 5 +- vault/wrapping.go | 5 +- .../source/docs/internals/replication.html.md | 149 ++++++++++++++++++ website/source/layouts/docs.erb | 4 + 28 files changed, 456 insertions(+), 142 deletions(-) create mode 100644 helper/consts/error.go create mode 100644 logical/response_util.go create mode 100644 website/source/docs/internals/replication.html.md diff --git a/.travis.yml b/.travis.yml index ab3c2d599..8fd2c16ce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ services: - docker go: - - 1.8rc2 + - 1.8 matrix: allow_failures: diff --git a/Makefile b/Makefile index f36e9dc48..732ba93a6 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,11 @@ dev-dynamic: generate test: generate CGO_ENABLED=0 VAULT_TOKEN= VAULT_ACC= go test -tags='$(BUILD_TAGS)' $(TEST) $(TESTARGS) -timeout=10m -parallel=4 +testcompile: generate + @for pkg in $(TEST) ; do \ + go test -v -c -tags='$(BUILD_TAGS)' $$pkg -parallel=4 ; \ + done + # testacc runs acceptance tests testacc: generate @if [ "$(TEST)" = "./..." ]; then \ diff --git a/README.md b/README.md index c7c46fca4..bf40cb75f 100644 --- a/README.md +++ b/README.md @@ -56,9 +56,9 @@ All documentation is available on the [Vault website](https://www.vaultproject.i Developing Vault -------------------- -If you wish to work on Vault itself or any of its built-in systems, -you'll first need [Go](https://www.golang.org) installed on your -machine (version 1.8+ is *required*). +If you wish to work on Vault itself or any of its built-in systems, you'll +first need [Go](https://www.golang.org) installed on your machine (version 1.8+ +is *required*). For local dev first make sure Go is properly installed, including setting up a [GOPATH](https://golang.org/doc/code.html#GOPATH). Next, clone this repository diff --git a/builtin/logical/cassandra/test-fixtures/cassandra.yaml b/builtin/logical/cassandra/test-fixtures/cassandra.yaml index 54f47d34a..5b12c8cf4 100644 --- a/builtin/logical/cassandra/test-fixtures/cassandra.yaml +++ b/builtin/logical/cassandra/test-fixtures/cassandra.yaml @@ -421,7 +421,7 @@ seed_provider: parameters: # seeds is actually a comma-delimited list of addresses. # Ex: ",," - - seeds: "172.17.0.2" + - seeds: "172.17.0.3" # For workloads with more data than can fit in memory, Cassandra's # bottleneck will be reads that need to fetch data from @@ -572,7 +572,7 @@ ssl_storage_port: 7001 # # Setting listen_address to 0.0.0.0 is always wrong. # -listen_address: 172.17.0.2 +listen_address: 172.17.0.3 # Set listen_address OR listen_interface, not both. Interfaces must correspond # to a single address, IP aliasing is not supported. @@ -586,7 +586,7 @@ listen_address: 172.17.0.2 # Address to broadcast to other Cassandra nodes # Leaving this blank will set it to the same value as listen_address -broadcast_address: 172.17.0.2 +broadcast_address: 172.17.0.3 # When using multiple physical network interfaces, set this # to true to listen on broadcast_address in addition to @@ -668,7 +668,7 @@ rpc_port: 9160 # be set to 0.0.0.0. If left blank, this will be set to the value of # rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must # be set. -broadcast_rpc_address: 172.17.0.2 +broadcast_rpc_address: 172.17.0.3 # enable or disable keepalive on rpc/native connections rpc_keepalive: true diff --git a/helper/consts/error.go b/helper/consts/error.go new file mode 100644 index 000000000..d96ba4fe8 --- /dev/null +++ b/helper/consts/error.go @@ -0,0 +1,13 @@ +package consts + +import "errors" + +var ( + // ErrSealed is returned if an operation is performed on a sealed barrier. + // No operation is expected to succeed before unsealing + ErrSealed = errors.New("Vault is sealed") + + // ErrStandby is returned if an operation is performed on a standby Vault. + // No operation is expected to succeed until active. + ErrStandby = errors.New("Vault is in standby mode") +) diff --git a/http/handler.go b/http/handler.go index 8a83af658..09533b539 100644 --- a/http/handler.go +++ b/http/handler.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/hashicorp/errwrap" + "github.com/hashicorp/vault/helper/consts" "github.com/hashicorp/vault/helper/duration" "github.com/hashicorp/vault/helper/jsonutil" "github.com/hashicorp/vault/logical" @@ -206,11 +207,11 @@ func handleRequestForwarding(core *vault.Core, handler http.Handler) http.Handle // case of an error. func request(core *vault.Core, w http.ResponseWriter, rawReq *http.Request, r *logical.Request) (*logical.Response, bool) { resp, err := core.HandleRequest(r) - if errwrap.Contains(err, vault.ErrStandby.Error()) { + if errwrap.Contains(err, consts.ErrStandby.Error()) { respondStandby(core, w, rawReq.URL) return resp, false } - if respondErrorCommon(w, resp, err) { + if respondErrorCommon(w, r, resp, err) { return resp, false } @@ -310,20 +311,7 @@ func requestWrapInfo(r *http.Request, req *logical.Request) (*logical.Request, e } func respondError(w http.ResponseWriter, status int, err error) { - // Adjust status code when sealed - if errwrap.Contains(err, vault.ErrSealed.Error()) { - status = http.StatusServiceUnavailable - } - - // Adjust status code on - if errwrap.Contains(err, "http: request body too large") { - status = http.StatusRequestEntityTooLarge - } - - // Allow HTTPCoded error passthrough to specify a code - if t, ok := err.(logical.HTTPCodedError); ok { - status = t.Code() - } + logical.AdjustErrorStatusCode(&status, err) w.Header().Add("Content-Type", "application/json") w.WriteHeader(status) @@ -337,42 +325,13 @@ func respondError(w http.ResponseWriter, status int, err error) { enc.Encode(resp) } -func respondErrorCommon(w http.ResponseWriter, resp *logical.Response, err error) bool { - // If there are no errors return - if err == nil && (resp == nil || !resp.IsError()) { +func respondErrorCommon(w http.ResponseWriter, req *logical.Request, resp *logical.Response, err error) bool { + statusCode, newErr := logical.RespondErrorCommon(req, resp, err) + if newErr == nil && statusCode == 0 { return false } - // Start out with internal server error since in most of these cases there - // won't be a response so this won't be overridden - statusCode := http.StatusInternalServerError - // If we actually have a response, start out with bad request - if resp != nil { - statusCode = http.StatusBadRequest - } - - // Now, check the error itself; if it has a specific logical error, set the - // appropriate code - if err != nil { - switch { - case errwrap.ContainsType(err, new(vault.StatusBadRequest)): - statusCode = http.StatusBadRequest - case errwrap.Contains(err, logical.ErrPermissionDenied.Error()): - statusCode = http.StatusForbidden - case errwrap.Contains(err, logical.ErrUnsupportedOperation.Error()): - statusCode = http.StatusMethodNotAllowed - case errwrap.Contains(err, logical.ErrUnsupportedPath.Error()): - statusCode = http.StatusNotFound - case errwrap.Contains(err, logical.ErrInvalidRequest.Error()): - statusCode = http.StatusBadRequest - } - } - - if resp != nil && resp.IsError() { - err = fmt.Errorf("%s", resp.Data["error"].(string)) - } - - respondError(w, statusCode, err) + respondError(w, statusCode, newErr) return true } diff --git a/http/handler_test.go b/http/handler_test.go index a7b1ad3d0..beab92281 100644 --- a/http/handler_test.go +++ b/http/handler_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/hashicorp/go-cleanhttp" + "github.com/hashicorp/vault/helper/consts" "github.com/hashicorp/vault/logical" "github.com/hashicorp/vault/vault" ) @@ -80,6 +81,7 @@ func TestSysMounts_headerAuth(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "sys/": map[string]interface{}{ "description": "system endpoints used for control, policy and debugging", @@ -88,6 +90,7 @@ func TestSysMounts_headerAuth(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "cubbyhole/": map[string]interface{}{ "description": "per-token private secret storage", @@ -96,6 +99,7 @@ func TestSysMounts_headerAuth(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, }, "secret/": map[string]interface{}{ @@ -105,6 +109,7 @@ func TestSysMounts_headerAuth(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "sys/": map[string]interface{}{ "description": "system endpoints used for control, policy and debugging", @@ -113,6 +118,7 @@ func TestSysMounts_headerAuth(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "cubbyhole/": map[string]interface{}{ "description": "per-token private secret storage", @@ -121,6 +127,7 @@ func TestSysMounts_headerAuth(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, } testResponseStatus(t, resp, 200) @@ -223,7 +230,7 @@ func TestHandler_error(t *testing.T) { // vault.ErrSealed is a special case w3 := httptest.NewRecorder() - respondError(w3, 400, vault.ErrSealed) + respondError(w3, 400, consts.ErrSealed) if w3.Code != 503 { t.Fatalf("expected 503, got %d", w3.Code) diff --git a/http/help.go b/http/help.go index b7191a929..f0ca8b170 100644 --- a/http/help.go +++ b/http/help.go @@ -35,7 +35,7 @@ func handleHelp(core *vault.Core, w http.ResponseWriter, req *http.Request) { resp, err := core.HandleRequest(lreq) if err != nil { - respondErrorCommon(w, resp, err) + respondErrorCommon(w, lreq, resp, err) return } diff --git a/http/logical.go b/http/logical.go index f350bf0c0..88bcb07fc 100644 --- a/http/logical.go +++ b/http/logical.go @@ -109,40 +109,13 @@ func handleLogical(core *vault.Core, dataOnly bool, prepareRequestCallback Prepa // Make the internal request. We attach the connection info // as well in case this is an authentication request that requires - // it. Vault core handles stripping this if we need to. + // it. Vault core handles stripping this if we need to. This also + // handles all error cases; if we hit respondLogical, the request is a + // success. resp, ok := request(core, w, r, req) if !ok { return } - switch { - case req.Operation == logical.ReadOperation: - if resp == nil { - respondError(w, http.StatusNotFound, nil) - return - } - - // Basically: if we have empty "keys" or no keys at all, 404. This - // provides consistency with GET. - case req.Operation == logical.ListOperation && resp.WrapInfo == nil: - if resp == nil || len(resp.Data) == 0 { - respondError(w, http.StatusNotFound, nil) - return - } - keysRaw, ok := resp.Data["keys"] - if !ok || keysRaw == nil { - respondError(w, http.StatusNotFound, nil) - return - } - keys, ok := keysRaw.([]string) - if !ok { - respondError(w, http.StatusInternalServerError, nil) - return - } - if len(keys) == 0 { - respondError(w, http.StatusNotFound, nil) - return - } - } // Build the proper response respondLogical(w, r, req, dataOnly, resp) diff --git a/http/logical_test.go b/http/logical_test.go index bc4cdfb04..c7b9bb6bd 100644 --- a/http/logical_test.go +++ b/http/logical_test.go @@ -101,7 +101,7 @@ func TestLogical_StandbyRedirect(t *testing.T) { // Attempt to fix raciness in this test by giving the first core a chance // to grab the lock - time.Sleep(time.Second) + time.Sleep(2 * time.Second) // Create a second HA Vault conf2 := &vault.CoreConfig{ diff --git a/http/sys_audit_test.go b/http/sys_audit_test.go index 499435a77..58873bfb1 100644 --- a/http/sys_audit_test.go +++ b/http/sys_audit_test.go @@ -35,6 +35,7 @@ func TestSysAudit(t *testing.T) { "type": "noop", "description": "", "options": map[string]interface{}{}, + "local": false, }, }, "noop/": map[string]interface{}{ @@ -42,6 +43,7 @@ func TestSysAudit(t *testing.T) { "type": "noop", "description": "", "options": map[string]interface{}{}, + "local": false, }, } testResponseStatus(t, resp, 200) diff --git a/http/sys_auth_test.go b/http/sys_auth_test.go index 3301e115e..9e193916f 100644 --- a/http/sys_auth_test.go +++ b/http/sys_auth_test.go @@ -32,6 +32,7 @@ func TestSysAuth(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, }, "token/": map[string]interface{}{ @@ -41,6 +42,7 @@ func TestSysAuth(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, } testResponseStatus(t, resp, 200) @@ -83,6 +85,7 @@ func TestSysEnableAuth(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "token/": map[string]interface{}{ "description": "token based credentials", @@ -91,6 +94,7 @@ func TestSysEnableAuth(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, }, "foo/": map[string]interface{}{ @@ -100,6 +104,7 @@ func TestSysEnableAuth(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "token/": map[string]interface{}{ "description": "token based credentials", @@ -108,6 +113,7 @@ func TestSysEnableAuth(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, } testResponseStatus(t, resp, 200) @@ -153,6 +159,7 @@ func TestSysDisableAuth(t *testing.T) { }, "description": "token based credentials", "type": "token", + "local": false, }, }, "token/": map[string]interface{}{ @@ -162,6 +169,7 @@ func TestSysDisableAuth(t *testing.T) { }, "description": "token based credentials", "type": "token", + "local": false, }, } testResponseStatus(t, resp, 200) diff --git a/http/sys_mount_test.go b/http/sys_mount_test.go index 513e5b941..f92229981 100644 --- a/http/sys_mount_test.go +++ b/http/sys_mount_test.go @@ -33,6 +33,7 @@ func TestSysMounts(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "sys/": map[string]interface{}{ "description": "system endpoints used for control, policy and debugging", @@ -41,6 +42,7 @@ func TestSysMounts(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "cubbyhole/": map[string]interface{}{ "description": "per-token private secret storage", @@ -49,6 +51,7 @@ func TestSysMounts(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, }, "secret/": map[string]interface{}{ @@ -58,6 +61,7 @@ func TestSysMounts(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "sys/": map[string]interface{}{ "description": "system endpoints used for control, policy and debugging", @@ -66,6 +70,7 @@ func TestSysMounts(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "cubbyhole/": map[string]interface{}{ "description": "per-token private secret storage", @@ -74,6 +79,7 @@ func TestSysMounts(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, } testResponseStatus(t, resp, 200) @@ -114,6 +120,7 @@ func TestSysMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "secret/": map[string]interface{}{ "description": "generic secret storage", @@ -122,6 +129,7 @@ func TestSysMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "sys/": map[string]interface{}{ "description": "system endpoints used for control, policy and debugging", @@ -130,6 +138,7 @@ func TestSysMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "cubbyhole/": map[string]interface{}{ "description": "per-token private secret storage", @@ -138,6 +147,7 @@ func TestSysMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, }, "foo/": map[string]interface{}{ @@ -147,6 +157,7 @@ func TestSysMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "secret/": map[string]interface{}{ "description": "generic secret storage", @@ -155,6 +166,7 @@ func TestSysMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "sys/": map[string]interface{}{ "description": "system endpoints used for control, policy and debugging", @@ -163,6 +175,7 @@ func TestSysMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "cubbyhole/": map[string]interface{}{ "description": "per-token private secret storage", @@ -171,6 +184,7 @@ func TestSysMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, } testResponseStatus(t, resp, 200) @@ -233,6 +247,7 @@ func TestSysRemount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "secret/": map[string]interface{}{ "description": "generic secret storage", @@ -241,6 +256,7 @@ func TestSysRemount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "sys/": map[string]interface{}{ "description": "system endpoints used for control, policy and debugging", @@ -249,6 +265,7 @@ func TestSysRemount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "cubbyhole/": map[string]interface{}{ "description": "per-token private secret storage", @@ -257,6 +274,7 @@ func TestSysRemount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, }, "bar/": map[string]interface{}{ @@ -266,6 +284,7 @@ func TestSysRemount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "secret/": map[string]interface{}{ "description": "generic secret storage", @@ -274,6 +293,7 @@ func TestSysRemount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "sys/": map[string]interface{}{ "description": "system endpoints used for control, policy and debugging", @@ -282,6 +302,7 @@ func TestSysRemount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "cubbyhole/": map[string]interface{}{ "description": "per-token private secret storage", @@ -290,6 +311,7 @@ func TestSysRemount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, } testResponseStatus(t, resp, 200) @@ -333,6 +355,7 @@ func TestSysUnmount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "sys/": map[string]interface{}{ "description": "system endpoints used for control, policy and debugging", @@ -341,6 +364,7 @@ func TestSysUnmount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "cubbyhole/": map[string]interface{}{ "description": "per-token private secret storage", @@ -349,6 +373,7 @@ func TestSysUnmount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, }, "secret/": map[string]interface{}{ @@ -358,6 +383,7 @@ func TestSysUnmount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "sys/": map[string]interface{}{ "description": "system endpoints used for control, policy and debugging", @@ -366,6 +392,7 @@ func TestSysUnmount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "cubbyhole/": map[string]interface{}{ "description": "per-token private secret storage", @@ -374,6 +401,7 @@ func TestSysUnmount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, } testResponseStatus(t, resp, 200) @@ -414,6 +442,7 @@ func TestSysTuneMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "secret/": map[string]interface{}{ "description": "generic secret storage", @@ -422,6 +451,7 @@ func TestSysTuneMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "sys/": map[string]interface{}{ "description": "system endpoints used for control, policy and debugging", @@ -430,6 +460,7 @@ func TestSysTuneMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "cubbyhole/": map[string]interface{}{ "description": "per-token private secret storage", @@ -438,6 +469,7 @@ func TestSysTuneMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, }, "foo/": map[string]interface{}{ @@ -447,6 +479,7 @@ func TestSysTuneMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "secret/": map[string]interface{}{ "description": "generic secret storage", @@ -455,6 +488,7 @@ func TestSysTuneMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "sys/": map[string]interface{}{ "description": "system endpoints used for control, policy and debugging", @@ -463,6 +497,7 @@ func TestSysTuneMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "cubbyhole/": map[string]interface{}{ "description": "per-token private secret storage", @@ -471,6 +506,7 @@ func TestSysTuneMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, } testResponseStatus(t, resp, 200) @@ -532,6 +568,7 @@ func TestSysTuneMount(t *testing.T) { "default_lease_ttl": json.Number("259196400"), "max_lease_ttl": json.Number("259200000"), }, + "local": false, }, "secret/": map[string]interface{}{ "description": "generic secret storage", @@ -540,6 +577,7 @@ func TestSysTuneMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "sys/": map[string]interface{}{ "description": "system endpoints used for control, policy and debugging", @@ -548,6 +586,7 @@ func TestSysTuneMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "cubbyhole/": map[string]interface{}{ "description": "per-token private secret storage", @@ -556,6 +595,7 @@ func TestSysTuneMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, }, "foo/": map[string]interface{}{ @@ -565,6 +605,7 @@ func TestSysTuneMount(t *testing.T) { "default_lease_ttl": json.Number("259196400"), "max_lease_ttl": json.Number("259200000"), }, + "local": false, }, "secret/": map[string]interface{}{ "description": "generic secret storage", @@ -573,6 +614,7 @@ func TestSysTuneMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "sys/": map[string]interface{}{ "description": "system endpoints used for control, policy and debugging", @@ -581,6 +623,7 @@ func TestSysTuneMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, "cubbyhole/": map[string]interface{}{ "description": "per-token private secret storage", @@ -589,6 +632,7 @@ func TestSysTuneMount(t *testing.T) { "default_lease_ttl": json.Number("0"), "max_lease_ttl": json.Number("0"), }, + "local": false, }, } diff --git a/http/sys_rekey.go b/http/sys_rekey.go index 2d6c97000..023452c18 100644 --- a/http/sys_rekey.go +++ b/http/sys_rekey.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" + "github.com/hashicorp/vault/helper/consts" "github.com/hashicorp/vault/helper/pgpkeys" "github.com/hashicorp/vault/vault" ) @@ -19,6 +20,13 @@ func handleSysRekeyInit(core *vault.Core, recovery bool) http.Handler { return } + repState := core.ReplicationState() + if repState == consts.ReplicationSecondary { + respondError(w, http.StatusBadRequest, + fmt.Errorf("rekeying can only be performed on the primary cluster when replication is activated")) + return + } + switch { case recovery && !core.SealAccess().RecoveryKeySupported(): respondError(w, http.StatusBadRequest, fmt.Errorf("recovery rekeying not supported")) @@ -108,7 +116,7 @@ func handleSysRekeyInitPut(core *vault.Core, recovery bool, w http.ResponseWrite // Right now we don't support this, but the rest of the code is ready for // when we do, hence the check below for this to be false if // StoredShares is greater than zero - if core.SealAccess().StoredKeysSupported() { + if core.SealAccess().StoredKeysSupported() && !recovery { respondError(w, http.StatusBadRequest, fmt.Errorf("rekeying of barrier not supported when stored key support is available")) return } diff --git a/http/sys_seal.go b/http/sys_seal.go index 27e3434ec..07ffbcd5b 100644 --- a/http/sys_seal.go +++ b/http/sys_seal.go @@ -8,6 +8,7 @@ import ( "net/http" "github.com/hashicorp/errwrap" + "github.com/hashicorp/vault/helper/consts" "github.com/hashicorp/vault/logical" "github.com/hashicorp/vault/vault" "github.com/hashicorp/vault/version" @@ -126,7 +127,7 @@ func handleSysUnseal(core *vault.Core) http.Handler { case errwrap.Contains(err, vault.ErrBarrierInvalidKey.Error()): case errwrap.Contains(err, vault.ErrBarrierNotInit.Error()): case errwrap.Contains(err, vault.ErrBarrierSealed.Error()): - case errwrap.Contains(err, vault.ErrStandby.Error()): + case errwrap.Contains(err, consts.ErrStandby.Error()): default: respondError(w, http.StatusInternalServerError, err) return diff --git a/logical/connection.go b/logical/connection.go index f14f65567..d54a0f532 100644 --- a/logical/connection.go +++ b/logical/connection.go @@ -8,7 +8,7 @@ import ( // is present on the Request structure for credential backends. type Connection struct { // RemoteAddr is the network address that sent the request. - RemoteAddr string + RemoteAddr string `json:"remote_addr"` // ConnState is the TLS connection state if applicable. ConnState *tls.ConnectionState diff --git a/logical/error.go b/logical/error.go index 9d082013f..19e3e2dea 100644 --- a/logical/error.go +++ b/logical/error.go @@ -21,3 +21,27 @@ func (e *codedError) Error() string { func (e *codedError) Code() int { return e.code } + +// Struct to identify user input errors. This is helpful in responding the +// appropriate status codes to clients from the HTTP endpoints. +type StatusBadRequest struct { + Err string +} + +// Implementing error interface +func (s *StatusBadRequest) Error() string { + return s.Err +} + +// This is a new type declared to not cause potential compatibility problems if +// the logic around the HTTPCodedError interface changes; in particular for +// logical request paths it is basically ignored, and changing that behavior +// might cause unforseen issues. +type ReplicationCodedError struct { + Msg string + Code int +} + +func (r *ReplicationCodedError) Error() string { + return r.Msg +} diff --git a/logical/response_util.go b/logical/response_util.go new file mode 100644 index 000000000..a3fd2bfd1 --- /dev/null +++ b/logical/response_util.go @@ -0,0 +1,111 @@ +package logical + +import ( + "errors" + "fmt" + "net/http" + + "github.com/hashicorp/errwrap" + multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/vault/helper/consts" +) + +// RespondErrorCommon pulls most of the functionality from http's +// respondErrorCommon and some of http's handleLogical and makes it available +// to both the http package and elsewhere. +func RespondErrorCommon(req *Request, resp *Response, err error) (int, error) { + if err == nil && (resp == nil || !resp.IsError()) { + switch { + case req.Operation == ReadOperation: + if resp == nil { + return http.StatusNotFound, nil + } + + // Basically: if we have empty "keys" or no keys at all, 404. This + // provides consistency with GET. + case req.Operation == ListOperation && resp.WrapInfo == nil: + if resp == nil || len(resp.Data) == 0 { + return http.StatusNotFound, nil + } + keysRaw, ok := resp.Data["keys"] + if !ok || keysRaw == nil { + return http.StatusNotFound, nil + } + keys, ok := keysRaw.([]string) + if !ok { + return http.StatusInternalServerError, nil + } + if len(keys) == 0 { + return http.StatusNotFound, nil + } + } + + return 0, nil + } + + if errwrap.ContainsType(err, new(ReplicationCodedError)) { + var allErrors error + codedErr := errwrap.GetType(err, new(ReplicationCodedError)).(*ReplicationCodedError) + errwrap.Walk(err, func(inErr error) { + newErr, ok := inErr.(*ReplicationCodedError) + if !ok { + allErrors = multierror.Append(allErrors, newErr) + } + }) + if allErrors != nil { + return codedErr.Code, multierror.Append(errors.New(fmt.Sprintf("errors from both primary and secondary; primary error was %v; secondary errors follow", codedErr.Msg)), allErrors) + } + return codedErr.Code, errors.New(codedErr.Msg) + } + + // Start out with internal server error since in most of these cases there + // won't be a response so this won't be overridden + statusCode := http.StatusInternalServerError + // If we actually have a response, start out with bad request + if resp != nil { + statusCode = http.StatusBadRequest + } + + // Now, check the error itself; if it has a specific logical error, set the + // appropriate code + if err != nil { + switch { + case errwrap.ContainsType(err, new(StatusBadRequest)): + statusCode = http.StatusBadRequest + case errwrap.Contains(err, ErrPermissionDenied.Error()): + statusCode = http.StatusForbidden + case errwrap.Contains(err, ErrUnsupportedOperation.Error()): + statusCode = http.StatusMethodNotAllowed + case errwrap.Contains(err, ErrUnsupportedPath.Error()): + statusCode = http.StatusNotFound + case errwrap.Contains(err, ErrInvalidRequest.Error()): + statusCode = http.StatusBadRequest + } + } + + if resp != nil && resp.IsError() { + err = fmt.Errorf("%s", resp.Data["error"].(string)) + } + + return statusCode, err +} + +// AdjustErrorStatusCode adjusts the status that will be sent in error +// conditions in a way that can be shared across http's respondError and other +// locations. +func AdjustErrorStatusCode(status *int, err error) { + // Adjust status code when sealed + if errwrap.Contains(err, consts.ErrSealed.Error()) { + *status = http.StatusServiceUnavailable + } + + // Adjust status code on + if errwrap.Contains(err, "http: request body too large") { + *status = http.StatusRequestEntityTooLarge + } + + // Allow HTTPCoded error passthrough to specify a code + if t, ok := err.(HTTPCodedError); ok { + *status = t.Code() + } +} diff --git a/scripts/cross/Dockerfile b/scripts/cross/Dockerfile index 7d9638b0a..1194fb02e 100644 --- a/scripts/cross/Dockerfile +++ b/scripts/cross/Dockerfile @@ -10,7 +10,7 @@ RUN apt-get update -y && apt-get install --no-install-recommends -y -q \ git mercurial bzr \ && rm -rf /var/lib/apt/lists/* -ENV GOVERSION 1.8rc3 +ENV GOVERSION 1.8 RUN mkdir /goroot && mkdir /gopath RUN curl https://storage.googleapis.com/golang/go${GOVERSION}.linux-amd64.tar.gz \ | tar xvzf - -C /goroot --strip-components=1 diff --git a/vault/cluster_test.go b/vault/cluster_test.go index 204d76d46..d3ee5126f 100644 --- a/vault/cluster_test.go +++ b/vault/cluster_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/hashicorp/vault/helper/consts" "github.com/hashicorp/vault/helper/logformat" "github.com/hashicorp/vault/logical" "github.com/hashicorp/vault/physical" @@ -100,7 +101,7 @@ func TestCluster_ListenForRequests(t *testing.T) { checkListenersFunc := func(expectFail bool) { tlsConfig, err := cores[0].ClusterTLSConfig() if err != nil { - if err.Error() != ErrSealed.Error() { + if err.Error() != consts.ErrSealed.Error() { t.Fatal(err) } tlsConfig = lastTLSConfig diff --git a/vault/core.go b/vault/core.go index f3ffcf5a8..f3b9bf696 100644 --- a/vault/core.go +++ b/vault/core.go @@ -60,14 +60,6 @@ const ( ) var ( - // ErrSealed is returned if an operation is performed on - // a sealed barrier. No operation is expected to succeed before unsealing - ErrSealed = errors.New("Vault is sealed") - - // ErrStandby is returned if an operation is performed on - // a standby Vault. No operation is expected to succeed until active. - ErrStandby = errors.New("Vault is in standby mode") - // ErrAlreadyInit is returned if the core is already // initialized. This prevents a re-initialization. ErrAlreadyInit = errors.New("Vault is already initialized") @@ -519,10 +511,10 @@ func (c *Core) LookupToken(token string) (*TokenEntry, error) { c.stateLock.RLock() defer c.stateLock.RUnlock() if c.sealed { - return nil, ErrSealed + return nil, consts.ErrSealed } if c.standby { - return nil, ErrStandby + return nil, consts.ErrStandby } // Many tests don't have a token store running @@ -657,7 +649,7 @@ func (c *Core) Leader() (isLeader bool, leaderAddr string, err error) { // Check if sealed if c.sealed { - return false, "", ErrSealed + return false, "", consts.ErrSealed } // Check if HA enabled @@ -1600,6 +1592,14 @@ func (c *Core) emitMetrics(stopCh chan struct{}) { } } +func (c *Core) ReplicationState() consts.ReplicationState { + var state consts.ReplicationState + c.clusterParamsLock.RLock() + state = c.replicationState + c.clusterParamsLock.RUnlock() + return state +} + func (c *Core) SealAccess() *SealAccess { sa := &SealAccess{} sa.SetSeal(c.seal) diff --git a/vault/core_test.go b/vault/core_test.go index 6b9eab04d..eb991d17b 100644 --- a/vault/core_test.go +++ b/vault/core_test.go @@ -8,6 +8,7 @@ import ( "github.com/hashicorp/errwrap" "github.com/hashicorp/go-uuid" "github.com/hashicorp/vault/audit" + "github.com/hashicorp/vault/helper/consts" "github.com/hashicorp/vault/helper/logformat" "github.com/hashicorp/vault/logical" "github.com/hashicorp/vault/physical" @@ -198,7 +199,7 @@ func TestCore_Route_Sealed(t *testing.T) { Path: "sys/mounts", } _, err := c.HandleRequest(req) - if err != ErrSealed { + if err != consts.ErrSealed { t.Fatalf("err: %v", err) } @@ -1541,7 +1542,7 @@ func testCore_Standby_Common(t *testing.T, inm physical.Backend, inmha physical. // Request should fail in standby mode _, err = core2.HandleRequest(req) - if err != ErrStandby { + if err != consts.ErrStandby { t.Fatalf("err: %v", err) } diff --git a/vault/generate_root.go b/vault/generate_root.go index 0966e9602..4278b022f 100644 --- a/vault/generate_root.go +++ b/vault/generate_root.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/hashicorp/go-uuid" + "github.com/hashicorp/vault/helper/consts" "github.com/hashicorp/vault/helper/pgpkeys" "github.com/hashicorp/vault/helper/xor" "github.com/hashicorp/vault/shamir" @@ -34,10 +35,10 @@ func (c *Core) GenerateRootProgress() (int, error) { c.stateLock.RLock() defer c.stateLock.RUnlock() if c.sealed { - return 0, ErrSealed + return 0, consts.ErrSealed } if c.standby { - return 0, ErrStandby + return 0, consts.ErrStandby } c.generateRootLock.Lock() @@ -52,10 +53,10 @@ func (c *Core) GenerateRootConfiguration() (*GenerateRootConfig, error) { c.stateLock.RLock() defer c.stateLock.RUnlock() if c.sealed { - return nil, ErrSealed + return nil, consts.ErrSealed } if c.standby { - return nil, ErrStandby + return nil, consts.ErrStandby } c.generateRootLock.Lock() @@ -101,10 +102,10 @@ func (c *Core) GenerateRootInit(otp, pgpKey string) error { c.stateLock.RLock() defer c.stateLock.RUnlock() if c.sealed { - return ErrSealed + return consts.ErrSealed } if c.standby { - return ErrStandby + return consts.ErrStandby } c.generateRootLock.Lock() @@ -170,10 +171,10 @@ func (c *Core) GenerateRootUpdate(key []byte, nonce string) (*GenerateRootResult c.stateLock.RLock() defer c.stateLock.RUnlock() if c.sealed { - return nil, ErrSealed + return nil, consts.ErrSealed } if c.standby { - return nil, ErrStandby + return nil, consts.ErrStandby } c.generateRootLock.Lock() @@ -308,10 +309,10 @@ func (c *Core) GenerateRootCancel() error { c.stateLock.RLock() defer c.stateLock.RUnlock() if c.sealed { - return ErrSealed + return consts.ErrSealed } if c.standby { - return ErrStandby + return consts.ErrStandby } c.generateRootLock.Lock() diff --git a/vault/rekey.go b/vault/rekey.go index 964abef4f..50f683b6b 100644 --- a/vault/rekey.go +++ b/vault/rekey.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/hashicorp/go-uuid" + "github.com/hashicorp/vault/helper/consts" "github.com/hashicorp/vault/helper/jsonutil" "github.com/hashicorp/vault/helper/pgpkeys" "github.com/hashicorp/vault/physical" @@ -44,10 +45,10 @@ func (c *Core) RekeyThreshold(recovery bool) (int, error) { c.stateLock.RLock() defer c.stateLock.RUnlock() if c.sealed { - return 0, ErrSealed + return 0, consts.ErrSealed } if c.standby { - return 0, ErrStandby + return 0, consts.ErrStandby } c.rekeyLock.RLock() @@ -72,10 +73,10 @@ func (c *Core) RekeyProgress(recovery bool) (int, error) { c.stateLock.RLock() defer c.stateLock.RUnlock() if c.sealed { - return 0, ErrSealed + return 0, consts.ErrSealed } if c.standby { - return 0, ErrStandby + return 0, consts.ErrStandby } c.rekeyLock.RLock() @@ -92,10 +93,10 @@ func (c *Core) RekeyConfig(recovery bool) (*SealConfig, error) { c.stateLock.RLock() defer c.stateLock.RUnlock() if c.sealed { - return nil, ErrSealed + return nil, consts.ErrSealed } if c.standby { - return nil, ErrStandby + return nil, consts.ErrStandby } c.rekeyLock.Lock() @@ -146,10 +147,10 @@ func (c *Core) BarrierRekeyInit(config *SealConfig) error { c.stateLock.RLock() defer c.stateLock.RUnlock() if c.sealed { - return ErrSealed + return consts.ErrSealed } if c.standby { - return ErrStandby + return consts.ErrStandby } c.rekeyLock.Lock() @@ -196,10 +197,10 @@ func (c *Core) RecoveryRekeyInit(config *SealConfig) error { c.stateLock.RLock() defer c.stateLock.RUnlock() if c.sealed { - return ErrSealed + return consts.ErrSealed } if c.standby { - return ErrStandby + return consts.ErrStandby } c.rekeyLock.Lock() @@ -240,10 +241,10 @@ func (c *Core) BarrierRekeyUpdate(key []byte, nonce string) (*RekeyResult, error c.stateLock.RLock() defer c.stateLock.RUnlock() if c.sealed { - return nil, ErrSealed + return nil, consts.ErrSealed } if c.standby { - return nil, ErrStandby + return nil, consts.ErrStandby } // Verify the key length @@ -422,10 +423,10 @@ func (c *Core) RecoveryRekeyUpdate(key []byte, nonce string) (*RekeyResult, erro c.stateLock.RLock() defer c.stateLock.RUnlock() if c.sealed { - return nil, ErrSealed + return nil, consts.ErrSealed } if c.standby { - return nil, ErrStandby + return nil, consts.ErrStandby } // Verify the key length @@ -589,10 +590,10 @@ func (c *Core) RekeyCancel(recovery bool) error { c.stateLock.RLock() defer c.stateLock.RUnlock() if c.sealed { - return ErrSealed + return consts.ErrSealed } if c.standby { - return ErrStandby + return consts.ErrStandby } c.rekeyLock.Lock() @@ -615,10 +616,10 @@ func (c *Core) RekeyRetrieveBackup(recovery bool) (*RekeyBackup, error) { c.stateLock.RLock() defer c.stateLock.RUnlock() if c.sealed { - return nil, ErrSealed + return nil, consts.ErrSealed } if c.standby { - return nil, ErrStandby + return nil, consts.ErrStandby } c.rekeyLock.RLock() @@ -652,10 +653,10 @@ func (c *Core) RekeyDeleteBackup(recovery bool) error { c.stateLock.RLock() defer c.stateLock.RUnlock() if c.sealed { - return ErrSealed + return consts.ErrSealed } if c.standby { - return ErrStandby + return consts.ErrStandby } c.rekeyLock.Lock() diff --git a/vault/request_handling.go b/vault/request_handling.go index 44b812550..2c7c45434 100644 --- a/vault/request_handling.go +++ b/vault/request_handling.go @@ -7,6 +7,7 @@ import ( "github.com/armon/go-metrics" "github.com/hashicorp/go-multierror" + "github.com/hashicorp/vault/helper/consts" "github.com/hashicorp/vault/helper/jsonutil" "github.com/hashicorp/vault/helper/policyutil" "github.com/hashicorp/vault/helper/strutil" @@ -18,10 +19,10 @@ func (c *Core) HandleRequest(req *logical.Request) (resp *logical.Response, err c.stateLock.RLock() defer c.stateLock.RUnlock() if c.sealed { - return nil, ErrSealed + return nil, consts.ErrSealed } if c.standby { - return nil, ErrStandby + return nil, consts.ErrStandby } // Allowing writing to a path ending in / makes it extremely difficult to diff --git a/vault/wrapping.go b/vault/wrapping.go index 0dc2e9b59..46409c39b 100644 --- a/vault/wrapping.go +++ b/vault/wrapping.go @@ -13,6 +13,7 @@ import ( "github.com/SermoDigital/jose/jws" "github.com/SermoDigital/jose/jwt" "github.com/hashicorp/errwrap" + "github.com/hashicorp/vault/helper/consts" "github.com/hashicorp/vault/helper/jsonutil" "github.com/hashicorp/vault/logical" ) @@ -284,10 +285,10 @@ func (c *Core) ValidateWrappingToken(req *logical.Request) (bool, error) { c.stateLock.RLock() defer c.stateLock.RUnlock() if c.sealed { - return false, ErrSealed + return false, consts.ErrSealed } if c.standby { - return false, ErrStandby + return false, consts.ErrStandby } te, err := c.tokenStore.Lookup(token) diff --git a/website/source/docs/internals/replication.html.md b/website/source/docs/internals/replication.html.md new file mode 100644 index 000000000..437997436 --- /dev/null +++ b/website/source/docs/internals/replication.html.md @@ -0,0 +1,149 @@ +--- +layout: "docs" +page_title: "Replication" +sidebar_current: "docs-internals-replication" +description: |- + Learn about the details of multi-datacenter replication within Vault. +--- + +# Replication (Vault Enterprise) + +Vault Enterprise 0.7 adds support for multi-datacenter replication. Before +using this feature, it is useful to understand the intended use cases, design +goals, and high level architecture. + +Replication is based on a primary/secondary (1:N) model with asynchronous +replication, focusing on high availability for global deployments. The +trade-offs made in the design and implementation of replication reflect these +high level goals. + +# Use Cases + +Vault replication is based on a number of common use cases: + +* **Multi-Datacenter Deployments**: A common challenge is providing Vault to + applications across many datacenters in a highly-available manner. Running a + single Vault cluster imposes high latency of access for remote clients, + availability loss or outages during connectivity failures, and limits + scalability. + +* **Backup Sites**: Implementing a robust business continuity plan around the + loss of a primary datacenter requires the ability to quickly and easily fail + to a hot backup site. + +* **Scaling Throughput**: Applications that use Vault for + Encryption-as-a-Service or cryptographic offload may generate a very high + volume of requests for Vault. Replicating keys between multiple clusters + allows load to be distributed across additional servers to scale request + throughput. + +# Design Goals + +Based on the use cases for Vault Replication, we had a number of design goals +for the implementation: + +* **Availability**: Global deployments of Vault require high levels of + availability, and can tolerate reduced consistency. During full connectivity, + replication is nearly real-time between the primary and secondary clusters. + Degraded connectivity between a primary and secondary does not impact the + primary's ability to service requests, and the secondary will continue to + service reads on last-known data. + +* **Conflict Free**: Certain replication techniques allow for potential write + conflicts to take place. Particularly, any active/active configuration where + writes are allowed to multiple sites require a conflict resolution strategy. + This varies from techniques that allow for data loss like last-write-wins, or + techniques that require manual operator resolution like allowing multiple + values per key. We avoid the possibility of conflicts to ensure there is no + data loss or manual intervention required. + +* **Transparent to Clients**: Vault replication should be transparent to + clients of Vault, so that existing thin clients work unmodified. The Vault + servers handle the logic of request forwarding to the primary when necessary, + and multi-hop routing is performed internally to ensure requests are + processed. + +* **Simple to Operate**: Operating a replicated cluster should be simple to + avoid administrative overhead and potentially introducing security gaps. + Setup of replication is very simple, and secondaries can handle being + arbitrarily behind the primary, avoiding the need for operator intervention + to copy data or snapshot the primary. + +# Architecture + +The architecture of Vault replication is based on the design goals, focusing on +the intended use cases. When replication is enabled, a cluster is set as either +a _primary_ or _secondary_. The primary cluster is authoritative, and is the +only cluster allowed to perform actions that write to the underlying data +storage, such as modifying policies or secrets. Secondary clusters can service +all other operations, such as reading secrets or sending data through +`transit`, and forward any writes to the primary cluster. Disallowing multiple +primaries ensures the cluster is conflict free and has an authoritative state. + +The primary cluster uses log shipping to replicate changes to all of the +secondaries. This ensures writes are visible globally in near real-time when +there is full network connectivity. If a secondary is down or unable to +communicate with the primary, writes are not blocked on the primary and reads +are still serviced on the secondary. This ensures the availability of Vault. +When the secondary is initialized or recovers from degraded connectivity it +will automatically reconcile with the primary. + +Lastly, clients can speak to any Vault server without a thick client. If a +client is communicating with a standby instance, the request is automatically +forwarded to a active instance. Secondary clusters will service reads locally +and forward any write requests to the primary cluster. The primary cluster is +able to service all request types. + +An important optimization Vault makes is to avoid replication of tokens or +leases between clusters. Policies and secrets are the minority of data managed +by Vault and tend to be relatively stable. Tokens and leases are much more +dynamic, as they are created and expire rapidly. Keeping tokens and leases +locally reduces the amount of data that needs to be replicated, and distributes +the work of TTL management between the clusters. The caveat is that clients +will need to re-authenticate if they switch the Vault cluster they are +communicating with. + +# Implementation Details + +It is important to understand the high-level architecture of replication to +ensure the trade-offs are appropriate for your use case. The implementation +details may be useful for those who are curious or want to understand more +about the performance characteristics or failure scenarios. + +Using replication requires a storage backend that supports transactional +updates, such as Consul. This allows multiple key/value updates to be +performed atomically. Replication uses this to maintain a +[Write-Ahead-Log][wal] (WAL) of all updates, so that the key update happens +atomically with the WAL entry creation. The WALs are then used to perform log +shipping between the Vault clusters. When a secondary is closely synchronized +with a primary, Vault directly streams new WALs to be applied, providing near +real-time replication. A bounded set of WALs are maintained for the +secondaries, and older WALs are garbage collected automatically. + +When a secondary is initialized or is too far behind the primary there may not +be enough WALs to synchronize. To handle this scenario, Vault maintains a +[merkle index][merkle] of the encrypted keys. Any time a key is updated or +deleted, the merkle index is updated to reflect the change. When a secondary +needs to reconcile with a primary, they compare their merkle indexes to +determine which keys are out of sync. The structure of the index allows this to +be done very efficiently, usually requiring only two round trips and a small +amount of data. The secondary uses this information to reconcile and then +switches back into WAL streaming mode. + +Performance is an important concern for Vault, so WAL entries are batched and +the merkle index is not flushed to disk with every operation. Instead, the +index is updated in memory for every operation and asynchronously flushed to +disk. As a result, a crash or power loss may cause the merkle index to become +out of sync with the underlying keys. Vault uses the [ARIES][aries] recovery +algorithm to ensure the consistency of the index under those failure +conditions. + +Log shipping traditionally requires the WAL stream to be synchronized, which +can introduce additional complexity when a new primary cluster is promoted. +Vault uses the merkle index as the source of truth, allowing the WAL streams to +be completely distinct and unsynchronized. This simplifies administration of +Vault Replication for operators. + +[wal]: https://en.wikipedia.org/wiki/Write-ahead_logging +[merkle]: https://en.wikipedia.org/wiki/Merkle_tree +[aries]: https://en.wikipedia.org/wiki/Algorithms_for_Recovery_and_Isolation_Exploiting_Semantics diff --git a/website/source/layouts/docs.erb b/website/source/layouts/docs.erb index 2a06df33c..ab40d4f6e 100644 --- a/website/source/layouts/docs.erb +++ b/website/source/layouts/docs.erb @@ -32,6 +32,10 @@ > Key Rotation + + > + Replication +