diff --git a/contributing/README.md b/contributing/README.md index d84e41824..19e4fc5b2 100644 --- a/contributing/README.md +++ b/contributing/README.md @@ -5,10 +5,25 @@ See [our contributing guide](../.github/CONTRIBUTING.md) to get started. This directory contains documentation intended for anyone interested in understanding, and contributing changes to, the Consul codebase. -## Contents +## Overview + +This documentation is organized into the following categories. Each category is +either a significant architectural layer, or major functional area of Consul. + +![Overview](./overview.svg) + +[source](./overview.mmd) + +## Contents 1. [Overview](./INTERNALS.md) -2. [Configuration](./checklist-adding-config-fields.md) -3. [Streaming](./streaming) -4. [Network Areas](./network-areas) -5. [Service Discovery](./service-discovery) +1. [Agent Configuration](./config) +1. [RPC](./rpc) +1. [Cluster Persistence](./persistence) +1. [Client Agent](./client-agent) +1. [Service Discovery](./service-discovery) +1. [Service Mesh (Connect)](./service-mesh) +1. [Cluster Membership](./cluster-membership) +1. [Key/Value Store](./kv) +1. [ACL](./acl) +1. [Multi-Cluster Federation](./cluster-federation) diff --git a/contributing/acl/acl-entity-relationship.mmd b/contributing/acl/acl-entity-relationship.mmd new file mode 100644 index 000000000..87031aabe --- /dev/null +++ b/contributing/acl/acl-entity-relationship.mmd @@ -0,0 +1,28 @@ +erDiagram + + Token + Policy + Role + ServiceIdentity + NodeIdentity + AuthMethod + BindingRule + Rule + // TODO: rules are made up of resources and x (enforcement decision or permission?) + // TODO: add Authorizer and Enforcement Decision + + Policy ||--|{ Rule: grants + Role ||--|{ Policy: includes + Role }|--|{ ServiceIdentity: includes + Role }|--|{ NodeIdentity: includes + + Token }|--|{ Policy: includes + Token }|--|{ Role: includes + Token }|--|{ ServiceIdentity: includes + Token }|--|{ NodeIdentity: includes + + AuthMethod ||--|{ BindingRule: defines + AuthMethod ||--|{ Token: creates + + ServiceIdentity ||--|{ Rule: implies + NodeIdentity ||--|{ Rule: implies diff --git a/contributing/acl/acl-entity-relationship.svg b/contributing/acl/acl-entity-relationship.svg new file mode 100644 index 000000000..8b9f57c72 --- /dev/null +++ b/contributing/acl/acl-entity-relationship.svg @@ -0,0 +1 @@ +TokenPolicyRoleServiceIdentityNodeIdentityAuthMethodBindingRuleRulegrantsincludesincludesincludesincludesincludesincludesincludesdefinescreatesimpliesimplies \ No newline at end of file diff --git a/contributing/client-agent/README.md b/contributing/client-agent/README.md new file mode 100644 index 000000000..5f16a031a --- /dev/null +++ b/contributing/client-agent/README.md @@ -0,0 +1,5 @@ +# Client Agent + +- agent/cache +- agent/local (local state) +- anti-entropy sync diff --git a/contributing/cluster-federation/README.md b/contributing/cluster-federation/README.md new file mode 100644 index 000000000..964796d64 --- /dev/null +++ b/contributing/cluster-federation/README.md @@ -0,0 +1,4 @@ +# Multi-Cluster Federation + +1. [Network Areas](./network-areas) + diff --git a/contributing/network-areas/README.md b/contributing/cluster-federation/network-areas/README.md similarity index 100% rename from contributing/network-areas/README.md rename to contributing/cluster-federation/network-areas/README.md diff --git a/contributing/cluster-membership/README.md b/contributing/cluster-membership/README.md new file mode 100644 index 000000000..9c285b77c --- /dev/null +++ b/contributing/cluster-membership/README.md @@ -0,0 +1,7 @@ +# Cluster membership + - hashicorp/serf + - hashicorp/memberlist + - network coordinates + - consul events + - consul exec + diff --git a/contributing/config/README.md b/contributing/config/README.md new file mode 100644 index 000000000..c3a51a45c --- /dev/null +++ b/contributing/config/README.md @@ -0,0 +1,3 @@ +# Agent Configuration + +- [Checklist for adding a new field](./checklist-adding-config-fields.md) diff --git a/contributing/checklist-adding-config-fields.md b/contributing/config/checklist-adding-config-fields.md similarity index 100% rename from contributing/checklist-adding-config-fields.md rename to contributing/config/checklist-adding-config-fields.md diff --git a/contributing/consul-mermaid-theme.json b/contributing/consul-mermaid-theme.json new file mode 100644 index 000000000..e9e078fc3 --- /dev/null +++ b/contributing/consul-mermaid-theme.json @@ -0,0 +1,4 @@ +{ + "theme": "default", + "themeCSS": ".node rect { fill: rgb(220, 71, 125); stroke-width: 1; stroke: black; } .node .label { color: white; } " +} diff --git a/contributing/overview.mmd b/contributing/overview.mmd new file mode 100644 index 000000000..ae210a919 --- /dev/null +++ b/contributing/overview.mmd @@ -0,0 +1,24 @@ +graph TD + + ServiceMesh[Sercice Mesh] + ServiceDiscovery[Service Discovery] + ClusterMembership[Cluster Membership] + KV[Key/Value Store] + MultiClusterFederation[Multi-Cluster Federation] + + ACL + AgentConfiguration[Agent Configuration] + ClientAgent[Client Agent] + RPC + ClusterPersistence[Cluster Persistence] + + AgentConfiguration --> ClientAgent + ClientAgent --> RPC + ClientAgent --> ACL + RPC --> ClusterPersistence + RPC --> ACL + + MultiClusterFederation --> ClusterMembership + MultiClusterFederation --> RPC + ServiceMesh --> ServiceDiscovery + diff --git a/contributing/overview.svg b/contributing/overview.svg new file mode 100644 index 000000000..44d77c94e --- /dev/null +++ b/contributing/overview.svg @@ -0,0 +1 @@ +
Sercice Mesh
Service Discovery
Cluster Membership
Key/Value Store
Multi-Cluster Federation
ACL
Agent Configuration
Client Agent
RPC
Cluster Persistence
 \ No newline at end of file diff --git a/contributing/persistence/README.md b/contributing/persistence/README.md new file mode 100644 index 000000000..12e1c0e7f --- /dev/null +++ b/contributing/persistence/README.md @@ -0,0 +1,8 @@ +# Persistence + +- hashicorp/raft +- state.Store - hashicorp/go-memdb +- FSM +- boltdb - https://github.com/boltdb/bolt (https://github.com/etcd-io/bbolt) +- snapshot and restore + diff --git a/contributing/rpc/README.md b/contributing/rpc/README.md new file mode 100644 index 000000000..a177f7f91 --- /dev/null +++ b/contributing/rpc/README.md @@ -0,0 +1,6 @@ +# RPC + +- net/rpc - (in the stdlib) +- [Streaming](./streaming) +- routing of "RPC" requests + diff --git a/contributing/rpc/rpc-routing.mmd b/contributing/rpc/rpc-routing.mmd new file mode 100644 index 000000000..bed74c024 --- /dev/null +++ b/contributing/rpc/rpc-routing.mmd @@ -0,0 +1,33 @@ +graph LR + + handleConn + + handleConn -->|RPCConsul| handleConsulConn + handleConn -->|RPCRaft| raftLayer + handleConn -->|RPCTLS| handleConn + handleConn -->|RPCMultiplexV2| handleMultiplexV2 + handleConn -->|RPCSnapshot| handleSnapshotConn + handleConn -->|RPCTLSInsecure| handleInsecureConn + handleConn -->|RPCGossip| handleGossipConn + + handleConsulConn --> RPCServer + handleMultiplexV2 --> handleConsulConn + + %% new after 1.6.9 + + handleConn -->|PeekForTLS| handleNativeTLS + + handleNativeTLS -->|ALPN_RPCConsul| handleConsulConn + handleNativeTLS -->|ALPN_RPCRaft| raftLayer + handleNativeTLS -->|ALPN_RPCMultiplexV2| handleMultiplexV2 + handleNativeTLS -->|ALPN_RPCSnapshot| handleSnapshotConn + handleNativeTLS -->|ALPN_RPCGRPC| grpcHandler + handleNativeTLS -->|ALPN_WANGossipPacket| handleWANGossipPacket + handleNativeTLS -->|ALPN_WANGossipStream | handleWANGossipStream + handleNativeTLS -->|ALPN_RPCGossip| handleGossipConn + + handleMultiplexV2 -->|RPCGossip| handleGossipConn + handleConn -->|RPCGRPC| grpcHandler + + + diff --git a/contributing/rpc/rpc-routing.svg b/contributing/rpc/rpc-routing.svg new file mode 100644 index 000000000..21c4f913d --- /dev/null +++ b/contributing/rpc/rpc-routing.svg @@ -0,0 +1 @@ +
RPCConsul
RPCRaft
RPCTLS
RPCMultiplexV2
RPCSnapshot
RPCTLSInsecure
RPCGossip
PeekForTLS
ALPN_RPCConsul
ALPN_RPCRaft
ALPN_RPCMultiplexV2
ALPN_RPCSnapshot
ALPN_RPCGRPC
ALPN_WANGossipPacket
ALPN_WANGossipStream
ALPN_RPCGossip
RPCGossip
RPCGRPC
handleConn
handleConsulConn
raftLayer
handleMultiplexV2
handleSnapshotConn
handleInsecureConn
handleGossipConn
RPCServer
handleNativeTLS
grpcHandler
handleWANGossipPacket
handleWANGossipStream
 \ No newline at end of file diff --git a/contributing/streaming/README.md b/contributing/rpc/streaming/README.md similarity index 100% rename from contributing/streaming/README.md rename to contributing/rpc/streaming/README.md diff --git a/contributing/streaming/adding-a-topic.md b/contributing/rpc/streaming/adding-a-topic.md similarity index 100% rename from contributing/streaming/adding-a-topic.md rename to contributing/rpc/streaming/adding-a-topic.md diff --git a/contributing/streaming/event-filtering.mmd b/contributing/rpc/streaming/event-filtering.mmd similarity index 100% rename from contributing/streaming/event-filtering.mmd rename to contributing/rpc/streaming/event-filtering.mmd diff --git a/contributing/streaming/event-filtering.svg b/contributing/rpc/streaming/event-filtering.svg similarity index 100% rename from contributing/streaming/event-filtering.svg rename to contributing/rpc/streaming/event-filtering.svg diff --git a/contributing/streaming/event-publisher-layout.mmd b/contributing/rpc/streaming/event-publisher-layout.mmd similarity index 100% rename from contributing/streaming/event-publisher-layout.mmd rename to contributing/rpc/streaming/event-publisher-layout.mmd diff --git a/contributing/streaming/event-publisher-layout.svg b/contributing/rpc/streaming/event-publisher-layout.svg similarity index 100% rename from contributing/streaming/event-publisher-layout.svg rename to contributing/rpc/streaming/event-publisher-layout.svg diff --git a/contributing/streaming/framing-events.mmd b/contributing/rpc/streaming/framing-events.mmd similarity index 100% rename from contributing/streaming/framing-events.mmd rename to contributing/rpc/streaming/framing-events.mmd diff --git a/contributing/streaming/framing-events.svg b/contributing/rpc/streaming/framing-events.svg similarity index 100% rename from contributing/streaming/framing-events.svg rename to contributing/rpc/streaming/framing-events.svg diff --git a/contributing/streaming/overview.mmd b/contributing/rpc/streaming/overview.mmd similarity index 100% rename from contributing/streaming/overview.mmd rename to contributing/rpc/streaming/overview.mmd diff --git a/contributing/streaming/overview.svg b/contributing/rpc/streaming/overview.svg similarity index 100% rename from contributing/streaming/overview.svg rename to contributing/rpc/streaming/overview.svg diff --git a/contributing/service-discovery/README.md b/contributing/service-discovery/README.md index 50e8e2c0b..6e9483491 100644 --- a/contributing/service-discovery/README.md +++ b/contributing/service-discovery/README.md @@ -2,4 +2,6 @@ This section is still a work in progress. +1. [catalog](./catalog.md) 1. [DNS Interface](./dns.md) +1. health checking diff --git a/contributing/service-discovery/catalog-2.mmd b/contributing/service-discovery/catalog-2.mmd new file mode 100644 index 000000000..2db8092d7 --- /dev/null +++ b/contributing/service-discovery/catalog-2.mmd @@ -0,0 +1,36 @@ +erDiagram + + CheckServiceNode + Node + NodeService + ServiceNode + HealthCheck + + CheckServiceNode ||--|| Node: has + CheckServiceNode ||--|| NodeService: has + CheckServiceNode ||--o{ HealthCheck: has + + Store ||--o{ Node: "stored in the node table" + Store ||--o{ ServiceNode: "stored in the service table" + Store ||--o{ HealthCheck: "stored in the checks table" + + ServiceNode ||--|| Node: references + HealthCheck ||--o| Node: references + HealthCheck ||--o| Service: references + + RegisterRequest ||--o| Node: has + RegisterRequest ||--o| NodeService: has + RegisterRequest ||--o{ HealthCheck: has + + + CheckDefinition + HealthCheckDefinition + CheckType + + HealthCheck ||--|| HealthCheckDefinition: has + + ServiceDefinition ||--|| NodeService: "is essentially a" + ServiceDefinition ||--o{ CheckType: "has" + + Config ||--o{ CheckDefinition: "has" + Config ||--o{ ServiceDefinition: "has" diff --git a/contributing/service-discovery/catalog.md b/contributing/service-discovery/catalog.md new file mode 100644 index 000000000..3aa2a80bd --- /dev/null +++ b/contributing/service-discovery/catalog.md @@ -0,0 +1,6 @@ +# Catalog + +This section is a work in progress. + +The catalog is at the core of both Service Discovery and Service Mesh. It accepts +registrations and deregistrations of Services, Nodes, and Checks. diff --git a/contributing/service-discovery/catalog.mmd b/contributing/service-discovery/catalog.mmd new file mode 100644 index 000000000..b1a85c3b6 --- /dev/null +++ b/contributing/service-discovery/catalog.mmd @@ -0,0 +1,24 @@ +erDiagram + + CheckServiceNode + Node + NodeService + ServiceNode + HealthCheck + + CheckServiceNode ||--|| Node: has + CheckServiceNode ||--|| NodeService: has + CheckServiceNode ||--o{ HealthCheck: has + + Store ||--o{ Node: "stored in the node table" + Store ||--o{ ServiceNode: "stored in the service table" + Store ||--o{ HealthCheck: "stored in the checks table" + + ServiceNode ||--|| Node: references + HealthCheck ||--o| Node: references + HealthCheck ||--o| Service: references + + RegisterRequest ||--o| Node: has + RegisterRequest ||--o| NodeService: has + RegisterRequest ||--o{ HealthCheck: has + diff --git a/contributing/service-mesh/README.md b/contributing/service-mesh/README.md new file mode 100644 index 000000000..3c770a88c --- /dev/null +++ b/contributing/service-mesh/README.md @@ -0,0 +1,12 @@ +# Service Mesh (Connect) + +- call out: envoy/proxy is the data plane, Consul is the control plane +- agent/xds - gRPC service that implements + [xDS](https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol) +- [agent/proxycfg](https://github.com/hashicorp/consul/blob/master/agent/proxycfg/proxycfg.go) +- CA Manager - certificate authority +- command/connect/envoy - bootstrapping and running envoy +- command/connect/proxy - built-in proxy that is dev-only and not supported + for production. +- `connect/` - "Native" service mesh +