Merge pull request #8630 from hashicorp/streaming/add-protobuf-types

streaming: add protobuf for subscribe service and events

build-support/scripts/proto-gen.sh

@@ -17,14 +17,13 @@ cat <<-EOF
 Usage: ${SCRIPT_NAME} [<options ...>] <proto filepath>
 
 Description:
-   This script will build generate the Go files from protobuf files. In addition to
+   Generate the Go files from protobuf definitions. In addition to
-   just running the correct protoc generator it will also fixup build tags in the
+   running the protoc generator it will also fixup build tags in the
    generated code.
 
 Options:
    --import-replace         Replace imports of google types with those from the gogo/protobuf repo.
    --grpc                   Enable the gRPC plugin
-
    -h | --help              Print this help text.
 EOF
 }
@@ -74,6 +73,7 @@ function main {
    local gogo_proto_imp_replace="Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types"
    gogo_proto_imp_replace="${gogo_proto_imp_replace},Mgoogle/protobuf/duration.proto=github.com/gogo/protobuf/types"
    gogo_proto_imp_replace="${gogo_proto_imp_replace},Mgoogle/protobuf/empty.proto=github.com/gogo/protobuf/types"
+   gogo_proto_imp_replace="${gogo_proto_imp_replace},Mgoogle/protobuf/struct.proto=github.com/gogo/protobuf/types"
    gogo_proto_imp_replace="${gogo_proto_imp_replace},Mgoogle/api/annotations.proto=github.com/gogo/googleapis/google/api"
    gogo_proto_imp_replace="${gogo_proto_imp_replace},Mgoogle/protobuf/field_mask.proto=github.com/gogo/protobuf/types"
 

proto/pbservice/healthcheck.pb.binary.go

@@ -0,0 +1,48 @@
+// Code generated by protoc-gen-go-binary. DO NOT EDIT.
+// source: proto/pbservice/healthcheck.proto
+
+package pbservice
+
+import (
+	"github.com/golang/protobuf/proto"
+)
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *HealthCheck) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *HealthCheck) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *HeaderValue) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *HeaderValue) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *HealthCheckDefinition) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *HealthCheckDefinition) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *CheckType) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *CheckType) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}

proto/pbservice/healthcheck.proto

@@ -0,0 +1,144 @@
+syntax = "proto3";
+
+package pbservice;
+
+option go_package = "github.com/hashicorp/consul/proto/pbservice";
+
+import "google/protobuf/duration.proto";
+import "proto/pbcommon/common.proto";
+import "proto/pbcommon/common_oss.proto";
+
+// This fake import path is replaced by the build script with a versioned path
+import "gogoproto/gogo.proto";
+
+option (gogoproto.goproto_unkeyed_all) = false;
+option (gogoproto.goproto_unrecognized_all) = false;
+option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.goproto_sizecache_all) = false;
+
+// HealthCheck represents a single check on a given node
+//
+// mog annotation:
+//
+// target=github.com/hashicorp/consul/agent/structs.HealthCheck
+// output=healthcheck.gen.go
+// name=Structs
+message HealthCheck {
+    string Node = 1;
+    string CheckID = 2 [(gogoproto.casttype) = "github.com/hashicorp/consul/types.CheckID"];
+    string Name = 3;
+    string Status = 4;                // The current check status
+    string Notes = 5;                 // Additional notes with the status
+    string Output = 6;                // Holds output of script runs
+    string ServiceID = 7;             // optional associated service
+    string ServiceName = 8;           // optional service name
+    repeated string ServiceTags = 9;  // optional service tags
+    string Type = 12;                 // Check type: http/ttl/tcp/etc
+
+    HealthCheckDefinition Definition = 10 [(gogoproto.nullable) = false];
+
+    // mog: func-to=RaftIndexToStructs func-from=NewRaftIndexFromStructs
+    common.RaftIndex RaftIndex = 11 [(gogoproto.embed) = true, (gogoproto.nullable) = false];
+
+    // mog: func-to=EnterpriseMetaToStructs func-from=NewEnterpriseMetaFromStructs
+    common.EnterpriseMeta EnterpriseMeta = 13  [(gogoproto.nullable) = false];
+}
+
+message HeaderValue {
+    repeated string Value = 1;
+}
+
+// HealthCheckDefinition of a single HealthCheck.
+//
+// mog annotation:
+//
+// target=github.com/hashicorp/consul/agent/structs.HealthCheckDefinition
+// output=healthcheck.gen.go
+// name=Structs
+message HealthCheckDefinition {
+    string HTTP = 1;
+    bool TLSSkipVerify = 2;
+
+    // mog: func-to=MapHeadersToStructs func-from=NewMapHeadersFromStructs
+    map<string, HeaderValue> Header = 3 [(gogoproto.nullable) = false];
+    string Method = 4;
+    string Body = 18;
+    string TCP = 5;
+    google.protobuf.Duration Interval = 6
+    [(gogoproto.stdduration) = true, (gogoproto.nullable) = false];
+
+    // mog: func-to=uint func-from=uint32
+    uint32 OutputMaxSize = 9;
+    google.protobuf.Duration Timeout = 7
+    [(gogoproto.stdduration) = true, (gogoproto.nullable) = false];
+    google.protobuf.Duration DeregisterCriticalServiceAfter = 8
+    [(gogoproto.stdduration) = true, (gogoproto.nullable) = false];
+    repeated string ScriptArgs = 10;
+    string DockerContainerID = 11;
+    string Shell = 12;
+    string GRPC = 13;
+    bool GRPCUseTLS = 14;
+    string AliasNode = 15;
+    string AliasService = 16;
+    google.protobuf.Duration TTL = 17
+    [(gogoproto.stdduration) = true, (gogoproto.nullable) = false];
+}
+
+// CheckType is used to create either the CheckMonitor or the CheckTTL.
+// The following types are supported: Script, HTTP, TCP, Docker, TTL, GRPC,
+// Alias. Script,
+// HTTP, Docker, TCP and GRPC all require Interval. Only one of the types may
+// to be provided: TTL or Script/Interval or HTTP/Interval or TCP/Interval or
+// Docker/Interval or GRPC/Interval or AliasService.
+//
+// mog annotation:
+//
+// target=github.com/hashicorp/consul/agent/structs.CheckType
+// output=healthcheck.gen.go
+// name=Structs
+message CheckType {
+    string CheckID = 1 [(gogoproto.casttype) = "github.com/hashicorp/consul/types.CheckID"];
+    string Name = 2;
+    string Status = 3;
+    string Notes = 4;
+
+    repeated string ScriptArgs = 5;
+    string HTTP = 6;
+    // mog: func-to=MapHeadersToStructs func-from=NewMapHeadersFromStructs
+    map<string, HeaderValue> Header = 20 [(gogoproto.nullable) = false];
+    string Method = 7;
+    string Body = 26;
+    string TCP = 8;
+    google.protobuf.Duration Interval = 9
+    [(gogoproto.stdduration) = true, (gogoproto.nullable) = false];
+
+    string AliasNode = 10;
+    string AliasService = 11;
+    string DockerContainerID = 12;
+    string Shell = 13;
+    string GRPC = 14;
+    bool GRPCUseTLS = 15;
+    bool TLSSkipVerify = 16;
+    google.protobuf.Duration Timeout = 17
+    [(gogoproto.stdduration) = true, (gogoproto.nullable) = false];
+    google.protobuf.Duration TTL = 18
+    [(gogoproto.stdduration) = true, (gogoproto.nullable) = false];
+
+    // mog: func-to=int func-from=int32
+    int32 SuccessBeforePassing = 21;
+    // mog: func-to=int func-from=int32
+    int32 FailuresBeforeCritical = 22;
+
+    // Definition fields used when exposing checks through a proxy
+    string ProxyHTTP = 23;
+    string ProxyGRPC = 24;
+
+    // DeregisterCriticalServiceAfter, if >0, will cause the associated
+    // service, if any, to be deregistered if this check is critical for
+    // longer than this duration.
+    google.protobuf.Duration DeregisterCriticalServiceAfter = 19
+    [(gogoproto.stdduration) = true, (gogoproto.nullable) = false];
+
+    // mog: func-to=int func-from=int32
+    int32 OutputMaxSize = 25;
+}

proto/pbservice/node.pb.binary.go

@@ -0,0 +1,38 @@
+// Code generated by protoc-gen-go-binary. DO NOT EDIT.
+// source: proto/pbservice/node.proto
+
+package pbservice
+
+import (
+	"github.com/golang/protobuf/proto"
+)
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *CheckServiceNode) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *CheckServiceNode) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *Node) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *Node) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *NodeService) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *NodeService) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}

proto/pbservice/node.proto

@@ -0,0 +1,115 @@
+syntax = "proto3";
+
+package pbservice;
+
+option go_package = "github.com/hashicorp/consul/proto/pbservice";
+
+import "proto/pbcommon/common.proto";
+import "proto/pbcommon/common_oss.proto";
+import "proto/pbservice/healthcheck.proto";
+import "proto/pbservice/service.proto";
+
+// This fake import path is replaced by the build script with a versioned path
+import "gogoproto/gogo.proto";
+
+option (gogoproto.goproto_unkeyed_all) = false;
+option (gogoproto.goproto_unrecognized_all) = false;
+option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.goproto_sizecache_all) = false;
+
+// CheckServiceNode is used to provide the node, its service
+// definition, as well as a HealthCheck that is associated.
+message CheckServiceNode {
+  Node Node = 1;
+  NodeService Service = 2;
+  repeated HealthCheck Checks = 3;
+}
+
+// Node contains information about a node.
+//
+// mog annotation:
+//
+// target=github.com/hashicorp/consul/agent/structs.Node
+// output=node.gen.go
+// name=Structs
+message Node {
+  string ID = 1 [(gogoproto.casttype) = "github.com/hashicorp/consul/types.NodeID"];
+
+  string Node = 2;
+  string Address = 3;
+  string Datacenter = 4;
+  map<string, string> TaggedAddresses = 5;
+  map<string, string> Meta = 6;
+
+  // mog: func-to=RaftIndexToStructs func-from=NewRaftIndexFromStructs
+  common.RaftIndex RaftIndex = 7 [(gogoproto.embed) = true, (gogoproto.nullable) = false];
+}
+
+// NodeService is a service provided by a node
+//
+// mog annotation:
+//
+// target=github.com/hashicorp/consul/agent/structs.NodeService
+// output=node.gen.go
+// name=Structs
+message NodeService {
+  // Kind is the kind of service this is. Different kinds of services may
+  // have differing validation, DNS behavior, etc. An empty kind will default
+  // to the Default kind. See ServiceKind for the full list of kinds.
+  string Kind = 1 [(gogoproto.casttype) = "github.com/hashicorp/consul/agent/structs.ServiceKind"];
+
+  string ID = 2;
+  string Service = 3;
+  repeated string Tags = 4;
+  string Address = 5;
+  // mog: func-to=MapStringServiceAddressToStructs func-from=NewMapStringServiceAddressFromStructs
+  map<string, ServiceAddress> TaggedAddresses = 15 [(gogoproto.nullable) = false];
+  map<string, string> Meta = 6;
+  // mog: func-to=int func-from=int32
+  int32 Port = 7;
+
+  // mog: func-to=WeightsPtrToStructs func-from=NewWeightsPtrFromStructs
+  Weights Weights = 8;
+  bool EnableTagOverride = 9;
+
+  // Proxy is the configuration set for Kind = connect-proxy. It is mandatory in
+  // that case and an error to be set for any other kind. This config is part of
+  // a proxy service definition and is distinct from but shares some fields with
+  // the Connect.Proxy which configures a managed proxy as part of the actual
+  // service's definition. This duplication is ugly but seemed better than the
+  // alternative which was to re-use the same struct fields for both cases even
+  // though the semantics are different and the non-shred fields make no sense
+  // in the other case. ProxyConfig may be a more natural name here, but it's
+  // confusing for the UX because one of the fields in ConnectProxyConfig is
+  // also called just "Config"
+  ConnectProxyConfig Proxy = 11 [(gogoproto.nullable) = false];
+
+  // Connect are the Connect settings for a service. This is purposely NOT
+  // a pointer so that we never have to nil-check this.
+  ServiceConnect Connect = 12 [(gogoproto.nullable) = false];
+
+  // LocallyRegisteredAsSidecar is private as it is only used by a local agent
+  // state to track if the service was registered from a nested sidecar_service
+  // block. We need to track that so we can know whether we need to deregister
+  // it automatically too if it's removed from the service definition or if the
+  // parent service is deregistered. Relying only on ID would cause us to
+  // deregister regular services if they happen to be registered using the same
+  // ID scheme as our sidecars do by default. We could use meta but that gets
+  // unpleasant because we can't use the consul- prefix from an agent (reserved
+  // for use internally but in practice that means within the state store or in
+  // responses only), and it leaks the detail publicly which people might rely
+  // on which is a bit unpleasant for something that is meant to be config-file
+  // syntax sugar. Note this is not translated to ServiceNode and friends and
+  // may not be set on a NodeService that isn't the one the agent registered and
+  // keeps in it's local state. We never want this rendered in JSON as it's
+  // internal only. Right now our agent endpoints return api structs which don't
+  // include it but this is a safety net incase we change that or there is
+  // somewhere this is used in API output.
+  bool LocallyRegisteredAsSidecar = 13;
+
+  // mog: func-to=EnterpriseMetaToStructs func-from=NewEnterpriseMetaFromStructs
+  common.EnterpriseMeta EnterpriseMeta = 16 [(gogoproto.nullable) = false];
+
+  // mog: func-to=RaftIndexToStructs func-from=NewRaftIndexFromStructs
+  common.RaftIndex RaftIndex = 14 [(gogoproto.embed) = true, (gogoproto.nullable) = false];
+}

proto/pbservice/service.pb.binary.go

@@ -0,0 +1,98 @@
+// Code generated by protoc-gen-go-binary. DO NOT EDIT.
+// source: proto/pbservice/service.proto
+
+package pbservice
+
+import (
+	"github.com/golang/protobuf/proto"
+)
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *ConnectProxyConfig) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *ConnectProxyConfig) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *Upstream) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *Upstream) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *ServiceConnect) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *ServiceConnect) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *ExposeConfig) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *ExposeConfig) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *ExposePath) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *ExposePath) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *MeshGatewayConfig) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *MeshGatewayConfig) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *ServiceDefinition) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *ServiceDefinition) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *ServiceAddress) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *ServiceAddress) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *Weights) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *Weights) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}

proto/pbservice/service.proto

@@ -0,0 +1,256 @@
+syntax = "proto3";
+
+package pbservice;
+
+option go_package = "github.com/hashicorp/consul/proto/pbservice";
+
+import "google/protobuf/struct.proto";
+import "proto/pbcommon/common_oss.proto";
+import "proto/pbservice/healthcheck.proto";
+
+// This fake import path is replaced by the build script with a versioned path
+import "gogoproto/gogo.proto";
+
+option (gogoproto.goproto_unkeyed_all) = false;
+option (gogoproto.goproto_unrecognized_all) = false;
+option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.goproto_sizecache_all) = false;
+
+
+// ConnectProxyConfig describes the configuration needed for any proxy managed
+// or unmanaged. It describes a single logical service's listener and optionally
+// upstreams and sidecar-related config for a single instance. To describe a
+// centralized proxy that routed traffic for multiple services, a different one
+// of these would be needed for each, sharing the same LogicalProxyID.
+//
+// mog annotation:
+//
+// target=github.com/hashicorp/consul/agent/structs.ConnectProxyConfig
+// output=service.gen.go
+// name=Structs
+message ConnectProxyConfig {
+    // DestinationServiceName is required and is the name of the service to accept
+    // traffic for.
+    string DestinationServiceName = 1;
+
+    // DestinationServiceID is optional and should only be specified for
+    // "side-car" style proxies where the proxy is in front of just a single
+    // instance of the service. It should be set to the service ID of the instance
+    // being represented which must be registered to the same agent. It's valid to
+    // provide a service ID that does not yet exist to avoid timing issues when
+    // bootstrapping a service with a proxy.
+    string DestinationServiceID = 2;
+
+    // LocalServiceAddress is the address of the local service instance. It is
+    // optional and should only be specified for "side-car" style proxies. It will
+    // default to 127.0.0.1 if the proxy is a "side-car" (DestinationServiceID is
+    // set) but otherwise will be ignored.
+    string LocalServiceAddress = 3;
+
+    // LocalServicePort is the port of the local service instance. It is optional
+    // and should only be specified for "side-car" style proxies. It will default
+    // to the registered port for the instance if the proxy is a "side-car"
+    // (DestinationServiceID is set) but otherwise will be ignored.
+    // mog: func-to=int func-from=int32
+    int32 LocalServicePort = 4;
+
+    // Config is the arbitrary configuration data provided with the proxy
+    // registration.
+    // mog: func-to=MapStringInterfaceToStructs func-from=NewMapStringInterfaceFromStructs
+    google.protobuf.Struct Config = 5 [(gogoproto.nullable) = true];
+
+    // Upstreams describes any upstream dependencies the proxy instance should
+    // setup.
+    // mog: func-to=UpstreamsToStructs func-from=NewUpstreamsFromStructs
+    repeated Upstream Upstreams = 6 [(gogoproto.nullable) = false];
+
+    // MeshGateway defines the mesh gateway configuration for this upstream
+    MeshGatewayConfig MeshGateway = 7 [(gogoproto.nullable) = false];
+
+    // Expose defines whether checks or paths are exposed through the proxy
+    ExposeConfig Expose = 8 [(gogoproto.nullable) = false];
+}
+
+// Upstream represents a single upstream dependency for a service or proxy. It
+// describes the mechanism used to discover instances to communicate with (the
+// Target) as well as any potential client configuration that may be useful such
+// as load balancer options, timeouts etc.
+//
+// mog annotation:
+//
+// target=github.com/hashicorp/consul/agent/structs.Upstream
+// output=service.gen.go
+// name=Structs
+// ignore-fields=IngressHosts
+message Upstream {
+    // Destination fields are the required ones for determining what this upstream
+    // points to. Depending on DestinationType some other fields below might
+    // further restrict the set of instances allowable.
+    //
+    // DestinationType would be better as an int constant but even with custom
+    // JSON marshallers it causes havoc with all the mapstructure mangling we do
+    // on service definitions in various places.
+    string DestinationType = 1;
+    string DestinationNamespace = 2;
+    string DestinationName = 3;
+
+    // Datacenter that the service discovery request should be run against. Note
+    // for prepared queries, the actual results might be from a different
+    // datacenter.
+    string Datacenter = 4;
+
+    // LocalBindAddress is the ip address a side-car proxy should listen on for
+    // traffic destined for this upstream service. Default if empty is 127.0.0.1.
+    string LocalBindAddress = 5;
+
+    // LocalBindPort is the ip address a side-car proxy should listen on for
+    // traffic destined for this upstream service. Required.
+    // mog: func-to=int func-from=int32
+    int32 LocalBindPort = 6;
+
+    // Config is an opaque config that is specific to the proxy process being run.
+    // It can be used to pass arbitrary configuration for this specific upstream
+    // to the proxy.
+    // mog: func-to=MapStringInterfaceToStructs func-from=NewMapStringInterfaceFromStructs
+    google.protobuf.Struct Config = 7 [(gogoproto.nullable) = true];
+
+    // MeshGateway is the configuration for mesh gateway usage of this upstream
+    MeshGatewayConfig MeshGateway = 8  [(gogoproto.nullable) = false];
+}
+
+// ServiceConnect are the shared Connect settings between all service
+// definitions from the agent to the state store.
+// mog annotation:
+//
+// target=github.com/hashicorp/consul/agent/structs.ServiceConnect
+// output=service.gen.go
+// name=Structs
+message ServiceConnect {
+    // Native is true when this service can natively understand Connect.
+    bool Native = 1;
+
+    // SidecarService is a nested Service Definition to register at the same time.
+    // It's purely a convenience mechanism to allow specifying a sidecar service
+    // along with the application service definition. It's nested nature allows
+    // all of the fields to be defaulted which can reduce the amount of
+    // boilerplate needed to register a sidecar service separately, but the end
+    // result is identical to just making a second service registration via any
+    // other means.
+    // mog: func-to=ServiceDefinitionPtrToStructs func-from=NewServiceDefinitionPtrFromStructs
+    ServiceDefinition SidecarService = 3;
+}
+
+// ExposeConfig describes HTTP paths to expose through Envoy outside of Connect.
+// Users can expose individual paths and/or all HTTP/GRPC paths for checks.
+//
+// mog annotation:
+//
+// target=github.com/hashicorp/consul/agent/structs.ExposeConfig
+// output=service.gen.go
+// name=Structs
+message ExposeConfig {
+    // Checks defines whether paths associated with Consul checks will be exposed.
+    // This flag triggers exposing all HTTP and GRPC check paths registered for the service.
+    bool Checks = 1;
+
+    // Paths is the list of paths exposed through the proxy.
+    // mog: func-to=ExposePathSliceToStructs func-from=NewExposePathSliceFromStructs
+    repeated ExposePath Paths = 2 [(gogoproto.nullable) = false];
+}
+
+// mog annotation:
+//
+// target=github.com/hashicorp/consul/agent/structs.ExposePath
+// output=service.gen.go
+// name=Structs
+message ExposePath {
+    // ListenerPort defines the port of the proxy's listener for exposed paths.
+    // mog: func-to=int func-from=int32
+    int32 ListenerPort = 1;
+
+    // ExposePath is the path to expose through the proxy, ie. "/metrics."
+    string Path = 2;
+
+    // LocalPathPort is the port that the service is listening on for the given path.
+    // mog: func-to=int func-from=int32
+    int32 LocalPathPort = 3;
+
+    // Protocol describes the upstream's service protocol.
+    // Valid values are "http" and "http2", defaults to "http"
+    string Protocol = 4;
+
+    // ParsedFromCheck is set if this path was parsed from a registered check
+    bool ParsedFromCheck = 5;
+}
+
+// mog annotation:
+//
+// target=github.com/hashicorp/consul/agent/structs.MeshGatewayConfig
+// output=service.gen.go
+// name=Structs
+message MeshGatewayConfig {
+    string Mode = 1 [(gogoproto.casttype) = "github.com/hashicorp/consul/agent/structs.MeshGatewayMode"];
+}
+
+// ServiceDefinition is used to JSON decode the Service definitions. For
+// documentation on specific fields see NodeService which is better documented.
+//
+// mog annotation:
+//
+// target=github.com/hashicorp/consul/agent/structs.ServiceDefinition
+// output=service.gen.go
+// name=Structs
+message ServiceDefinition {
+    string Kind = 1 [(gogoproto.casttype) = "github.com/hashicorp/consul/agent/structs.ServiceKind"];
+    string ID = 2;
+    string Name = 3;
+    repeated string Tags = 4;
+    string Address = 5;
+    // mog: func-to=MapStringServiceAddressToStructs func-from=NewMapStringServiceAddressFromStructs
+    map<string, ServiceAddress> TaggedAddresses = 16 [(gogoproto.nullable) = false];
+    map<string, string> Meta = 6;
+    // mog: func-to=int func-from=int32
+    int32 Port = 7;
+    CheckType Check = 8 [(gogoproto.nullable) = false];
+    // mog: func-to=CheckTypesToStructs func-from=NewCheckTypesFromStructs
+    repeated CheckType Checks = 9;
+    // mog: func-to=WeightsPtrToStructs func-from=NewWeightsPtrFromStructs
+    Weights Weights = 10;
+    string Token = 11;
+    bool EnableTagOverride = 12;
+
+    // Proxy is the configuration set for Kind = connect-proxy. It is mandatory in
+    // that case and an error to be set for any other kind. This config is part of
+    // a proxy service definition and is distinct from but shares some fields with
+    // the Connect.Proxy which configures a managed proxy as part of the actual
+    // service's definition. This duplication is ugly but seemed better than the
+    // alternative which was to re-use the same struct fields for both cases even
+    // though the semantics are different and the non-shared fields make no sense
+    // in the other case. ProxyConfig may be a more natural name here, but it's
+    // confusing for the UX because one of the fields in ConnectProxyConfig is
+    // also called just "Config"
+    // mog: func-to=ConnectProxyConfigPtrToStructs func-from=NewConnectProxyConfigPtrFromStructs
+    ConnectProxyConfig Proxy = 14;
+
+    // mog: func-to=EnterpriseMetaToStructs func-from=NewEnterpriseMetaFromStructs
+    common.EnterpriseMeta EnterpriseMeta = 17 [(gogoproto.nullable) = false];
+
+    // mog: func-to=ServiceConnectPtrToStructs func-from=NewServiceConnectPtrFromStructs
+    ServiceConnect Connect = 15;
+}
+
+// Type to hold an address and port of a service
+message ServiceAddress {
+    string Address = 1;
+    // mog: func-to=int func-from=int32
+    int32 Port = 2;
+}
+
+
+// Weights represent the weight used by DNS for a given status
+message Weights {
+    // mog: func-to=int func-from=int32
+    int32 Passing = 1;
+    // mog: func-to=int func-from=int32
+    int32 Warning = 2;
+}

proto/pbsubscribe/subscribe.pb.binary.go

@@ -0,0 +1,48 @@
+// Code generated by protoc-gen-go-binary. DO NOT EDIT.
+// source: proto/pbsubscribe/subscribe.proto
+
+package pbsubscribe
+
+import (
+	"github.com/golang/protobuf/proto"
+)
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *SubscribeRequest) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *SubscribeRequest) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *Event) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *Event) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *EventBatch) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *EventBatch) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *ServiceHealthUpdate) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *ServiceHealthUpdate) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}

proto/pbsubscribe/subscribe.proto

@@ -0,0 +1,126 @@
+/*
+Package event provides a service for subscribing to state change events.
+*/
+syntax = "proto3";
+
+package subscribe;
+
+option go_package = "github.com/hashicorp/consul/proto/pbsubscribe";
+
+import "proto/pbservice/node.proto";
+
+// StateChangeSubscription service allows consumers to subscribe to topics of
+// state change events. Events are streamed as they happen.
+service StateChangeSubscription {
+    // Subscribe to a topic to receive events when there are changes to the topic.
+    // TODO: document how to handle framing events
+    //
+    //
+    // Subscribe may return an ABORTED status error to indicate the client must
+    // re-start the Subscribe call.
+    // This error is used when the server can no longer correctly maintain the
+    // stream, for example because the ACL permissions for the token changed
+    // and the server doesn't know which previously delivered events should
+    // now not be visible. Clients when receiving this must reset their
+    // local copy of the state to empty and start over from index 0 to get a
+    // valid snapshot again. Servers may also send this if their state store
+    // is restored from a snapshot.
+    rpc Subscribe(SubscribeRequest) returns (stream Event) {}
+}
+
+// Topic enumerates the supported event topics.
+enum Topic {
+    Unknown = 0;
+    // ServiceHealth topic contains events for any changes to service health.
+    ServiceHealth = 1;
+    // ServiceHealthConnect topic contains events for any changes to service
+    // health for connect-enabled services.
+    ServiceHealthConnect = 2;
+}
+
+// SubscribeRequest used to subscribe to a topic.
+message SubscribeRequest {
+    // Topic identifies the set of events the subscriber is interested in.
+    Topic Topic = 1;
+
+    // Key is a topic-specific identifier that restricts the scope of the
+    // subscription to only events pertaining to that identifier. For example,
+    // to receive events for a single service, the service's name is
+    // specified as the key. An empty key indicates that all events in the topic
+    // are of interest.
+    string Key = 2;
+
+    // Token is the ACL token to authenticate the request. The token must have
+    // sufficient privileges to read the requested information otherwise events
+    // will be filtered, possibly resulting in an empty snapshot and no further
+    // updates sent.
+    string Token = 3;
+
+    // Index is the raft index the subscriber has already observed up to. This
+    // is zero on an initial streaming call, but then can be provided by a
+    // client on subsequent re-connections such that the full snapshot doesn't
+    // need to be resent if the client is up to date.
+    uint64 Index = 4;
+
+    // Datacenter specifies the Consul datacenter the request is targeted at.
+    // If it's not the local DC the server will forward the request to
+    // the remote DC and proxy the results back  to the subscriber. An empty
+    // string defaults to the local datacenter.
+    string Datacenter = 5;
+}
+
+// Event describes a streaming update on a subscription. Events are used both to
+// describe the current "snapshot" of the result as well as ongoing mutations to
+// that snapshot.
+message Event {
+    // Topic the event was published to
+    Topic Topic = 1;
+
+    // Key is the logical identifier for the entity that was mutated.
+    string Key = 2;
+
+    // Index is the raft index at which the mutation took place. At the top
+    // level of a subscription there will always be at most one Event per index.
+    // If multiple events are published to the same topic in a single raft
+    // transaction then the batch of events will be encoded inside a single
+    // top-level event to ensure they are delivered atomically to clients.
+    uint64 Index = 3;
+
+    // Payload is the actual event content.
+    oneof Payload {
+        // EndOfSnapshot indicates the event stream for the initial snapshot has
+        // ended. Subsequent Events delivered will be mutations to that result.
+        bool EndOfSnapshot = 5;
+
+        // EndOfEmptySnapshot indicates that the client is still up-to-date.
+        // The snapshot has ended, and was empty. The rest of the stream will be
+        // individual update events. It distinguishes between "up to date, no snapshot"
+        // and "snapshot contains zero events but you should reset any old state to be blank".
+        bool EndOfEmptySnapshot = 6;
+
+        // EventBatch is a set of events. This is typically used as the payload
+        // type where multiple events are emitted in a single topic and raft
+        // index (e.g. transactional updates). In this case the Topic and Index
+        // values of all events will match and the whole set should be delivered
+        // and consumed atomically.
+        EventBatch EventBatch = 7;
+
+        // ServiceHealth is used for ServiceHealth and ServiceHealthConnect
+        // topics.
+        ServiceHealthUpdate ServiceHealth = 10;
+    }
+}
+
+message EventBatch {
+    repeated Event Events = 1;
+}
+
+enum CatalogOp {
+    Register = 0;
+    Deregister = 1;
+}
+
+message ServiceHealthUpdate {
+    CatalogOp Op = 1;
+    pbservice.CheckServiceNode CheckServiceNode = 2;
+}