From 35674ae6b91f1d1096535c6eea4ce95e6c6cb158 Mon Sep 17 00:00:00 2001 From: Dan Upton Date: Mon, 15 May 2023 12:34:36 +0100 Subject: [PATCH] docs: initial documentation for the new State Store (#17315) --- docs/README.md | 1 + docs/persistence/README.md | 5 + docs/resources/README.md | 111 +++++++++++++++++++++++ docs/resources/architecture-overview.png | 3 + docs/resources/raft-backend.png | 3 + internal/storage/raft/backend.go | 3 + proto-public/pbresource/resource.pb.go | 2 + proto-public/pbresource/resource.proto | 1 + 8 files changed, 129 insertions(+) create mode 100644 docs/resources/README.md create mode 100644 docs/resources/architecture-overview.png create mode 100644 docs/resources/raft-backend.png diff --git a/docs/README.md b/docs/README.md index 4a7523fd1..d3483710b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -25,6 +25,7 @@ be found in the public [user documentation]. 1. [Agent Configuration](./config) 1. [RPC](./rpc) 1. [Cluster Persistence](./persistence) +1. [Resources and Controllers](./resources) 1. [Client Agent](./client-agent) 1. [Service Discovery](./service-discovery) 1. [Service Mesh (Connect)](./service-mesh) diff --git a/docs/persistence/README.md b/docs/persistence/README.md index bb907d7d1..f705539fa 100644 --- a/docs/persistence/README.md +++ b/docs/persistence/README.md @@ -1,5 +1,10 @@ # Cluster Persistence +> **Note** +> While the content of this document is still accurate, it doesn't cover the new +> generic resource-oriented storage layer introduced in Consul 1.16. Please see +> [Resources](../resources) for more information. + The cluser persistence subsystem runs entirely in Server Agents. It handles both read and write requests from the [RPC] subsystem. See the [Consul Architecture Guide] for an introduction to the Consul deployment architecture and the [Consensus Protocol] used by diff --git a/docs/resources/README.md b/docs/resources/README.md new file mode 100644 index 000000000..1356da207 --- /dev/null +++ b/docs/resources/README.md @@ -0,0 +1,111 @@ +# Resources + +Consul 1.16 introduced a set of [generic APIs] for managing resources, and a +[controller runtime] for building functionality on top of them. + +[generic APIs]: ../../proto-public/pbresource/resource.proto +[controller runtime]: ../../internal/controller + +Previously, adding features to Consul involved making changes at every layer of +the stack, including: HTTP handlers, RPC handlers, MemDB tables, Raft +operations, and CLI commands. + +This architecture made sense when the product was maintained by a small core +group who could keep the entire system in their heads, but presented significant +collaboration, ownership, and onboarding challenges when our contributor base +expanded to many engineers, across several teams, and the product grew in +complexity. + +In the new model, teams can work with much greater autonomy by building on top +of a shared platform and own their resource types and controllers. + +## Architecture Overview + +![architecture diagram](./architecture-overview.png) + +[source](https://whimsical.com/state-store-v2-UKE6SaEPXNc4UrZBrZj4Kg) + +Our resource-oriented architecture comprises the following components: + +#### Resource Service + +[Resource Service](../../proto-public/pbresource/resource.proto) is a gRPC +service that contains the shared logic for creating, reading, updating, +deleting, and watching resources. It will be consumed by controllers, our +Kubernetes integration, the CLI, and mapped to an HTTP+JSON API. + +#### Type Registry + +[Type Registry](../../internal/resource/registry.go) is where teams register +their resource types, along with hooks for performing structural validation, +authorization, etc. + +#### Storage Backend + +[Storage Backend](../../internal/storage/storage.go) is an abstraction over +low-level storage primitives. Today, there are two implementations (Raft and +an in-memory backend for tests) but in the future, we envisage external storage +systems such as the Kubernetes API or an RDBMS could be supported which would +reduce operational complexity for our customers. + +#### Controllers + +[Controllers](../../internal/controller/api.go) implement Consul's business +logic using asynchronous control loops that respond to changes in resources. + +## Raft Storage Backend + +Our [Raft Storage Backend](../../internal/storage/raft/backend.go) integrates +with the existing Raft machinery (e.g. FSM) used by the [old state store]. It +also transparently forwards writes and strongly consistent reads to the leader +over gRPC. + +There's quite a lot going on here, so to dig into the details, let's take a look +at how a write operation is handled. + +[old state store]: ../persistence/ + +![raft storage backend diagram](./raft-backend.png) + +[source](https://whimsical.com/state-store-v2-UKE6SaEPXNc4UrZBrZj4Kg) + +#### Steps 1 & 2 + +User calls the resource service's `Write` endpoint, on a Raft follower, which +in-turn calls the storage backend's `WriteCAS` method. + +#### Steps 3 & 4 + +The storage backend determines that the current server is a Raft follower, and +forwards the operation to the leader via a gRPC [forwarding service] listening +on the multiplexed RPC port ([`ports.server`]). + +[forwarding service]: ../../proto/private/pbstorage/raft.proto +[`ports.server`]: https://developer.hashicorp.com/consul/docs/agent/config/config-files#server_rpc_port + +#### Step 5 + +The leader's storage backend serializes the operation to protobuf and applies it +to the Raft log. As we need to share the Raft log with the old state store, we go +through the [`consul.raftHandle`](../../agent/consul/raft_handle.go) and +[`consul.Server`](../../agent/consul/server/server.go) which applies a msgpack +envelope and type byte prefix. + +#### Step 6 + +Raft consensus happens! Once the log has been committed, it is applied to the +[FSM](../../agent/consul/fsm/fsm.go) which calls the storage backend's `Apply` +method to apply the protobuf-encoded operation to the [`inmem.Store`]. + +[`inmem.Store`]: ../../internal/storage/inmem/store.go + +#### Steps 7, 8, 9 + +At this point, the operation is complete. The forwarding service returns a +successful response, as does the follower's storage backend, and the user +gets a successful response too. + +#### Steps 10 & 11 + +Asynchronously, the log is replicated to followers and applied to their storage +backends. diff --git a/docs/resources/architecture-overview.png b/docs/resources/architecture-overview.png new file mode 100644 index 000000000..9ec3be530 --- /dev/null +++ b/docs/resources/architecture-overview.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dce815f3cf56748303ad5c59adbf2c59f5cbb63f12ce0c04a26b4fd762fe59f4 +size 125060 diff --git a/docs/resources/raft-backend.png b/docs/resources/raft-backend.png new file mode 100644 index 000000000..c03631231 --- /dev/null +++ b/docs/resources/raft-backend.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17b14c6314e672e5cd15a2b772ce1819e64bc354b409f22a10bb89af68c9fe4b +size 307481 diff --git a/internal/storage/raft/backend.go b/internal/storage/raft/backend.go index 387e35973..4e1cd05bb 100644 --- a/internal/storage/raft/backend.go +++ b/internal/storage/raft/backend.go @@ -44,6 +44,9 @@ import ( // intended to communicate over Consul's multiplexed server port (which handles // TLS). // +// For more information, see here: +// https://github.com/hashicorp/consul/tree/main/docs/resources#raft-storage-backend +// // You must call Run before using the backend. func NewBackend(h Handle, l hclog.Logger) (*Backend, error) { s, err := inmem.NewStore() diff --git a/proto-public/pbresource/resource.pb.go b/proto-public/pbresource/resource.pb.go index 76bf1be45..4a46d1ea8 100644 --- a/proto-public/pbresource/resource.pb.go +++ b/proto-public/pbresource/resource.pb.go @@ -7,6 +7,8 @@ // protoc (unknown) // source: pbresource/resource.proto +// For more information, see: https://github.com/hashicorp/consul/tree/main/docs/resources + package pbresource import ( diff --git a/proto-public/pbresource/resource.proto b/proto-public/pbresource/resource.proto index 038f96c7a..9e6564758 100644 --- a/proto-public/pbresource/resource.proto +++ b/proto-public/pbresource/resource.proto @@ -3,6 +3,7 @@ syntax = "proto3"; +// For more information, see: https://github.com/hashicorp/consul/tree/main/docs/resources package hashicorp.consul.resource; import "annotations/ratelimit/ratelimit.proto";