From 90b9f87160c2939549bbead9f4711a4e9bc9a808 Mon Sep 17 00:00:00 2001 From: Matt Keeler Date: Fri, 24 Jan 2020 09:27:25 -0500 Subject: [PATCH] Add the v1/catalog/node-services/:node endpoint (#7115) The backing RPC already existed but the endpoint will be useful for other service syncing processes such as consul-k8s as this endpoint can return all services registered with a node regardless of namespacing. --- agent/catalog_endpoint.go | 50 +++++++++++ agent/catalog_endpoint_test.go | 50 +++++++++++ agent/consul/catalog_endpoint.go | 2 +- agent/http_register.go | 1 + agent/translate_addr.go | 8 ++ api/catalog.go | 29 +++++++ api/catalog_test.go | 59 +++++++++++++ website/source/api/catalog.html.md | 132 ++++++++++++++++++++++++++++- 8 files changed, 329 insertions(+), 2 deletions(-) diff --git a/agent/catalog_endpoint.go b/agent/catalog_endpoint.go index dba5f4871..74c3a7810 100644 --- a/agent/catalog_endpoint.go +++ b/agent/catalog_endpoint.go @@ -364,3 +364,53 @@ RETRY_ONCE: []metrics.Label{{Name: "node", Value: s.nodeName()}}) return out.NodeServices, nil } + +func (s *HTTPServer) CatalogNodeServiceList(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + metrics.IncrCounterWithLabels([]string{"client", "api", "catalog_node_service_list"}, 1, + []metrics.Label{{Name: "node", Value: s.nodeName()}}) + + // Set default Datacenter + args := structs.NodeSpecificRequest{} + if err := s.parseEntMeta(req, &args.EnterpriseMeta); err != nil { + return nil, err + } + + if done := s.parse(resp, req, &args.Datacenter, &args.QueryOptions); done { + return nil, nil + } + + // Pull out the node name + args.Node = strings.TrimPrefix(req.URL.Path, "/v1/catalog/node-services/") + if args.Node == "" { + resp.WriteHeader(http.StatusBadRequest) + fmt.Fprint(resp, "Missing node name") + return nil, nil + } + + // Make the RPC request + var out structs.IndexedNodeServiceList + defer setMeta(resp, &out.QueryMeta) +RETRY_ONCE: + if err := s.agent.RPC("Catalog.NodeServiceList", &args, &out); err != nil { + metrics.IncrCounterWithLabels([]string{"client", "rpc", "error", "catalog_node_service_list"}, 1, + []metrics.Label{{Name: "node", Value: s.nodeName()}}) + return nil, err + } + if args.QueryOptions.AllowStale && args.MaxStaleDuration > 0 && args.MaxStaleDuration < out.LastContact { + args.AllowStale = false + args.MaxStaleDuration = 0 + goto RETRY_ONCE + } + out.ConsistencyLevel = args.QueryOptions.ConsistencyLevel() + s.agent.TranslateAddresses(args.Datacenter, &out.NodeServices, TranslateAddressAcceptAny) + + // Use empty list instead of nil + for _, s := range out.NodeServices.Services { + if s.Tags == nil { + s.Tags = make([]string, 0) + } + } + metrics.IncrCounterWithLabels([]string{"client", "api", "success", "catalog_node_service_list"}, 1, + []metrics.Label{{Name: "node", Value: s.nodeName()}}) + return &out.NodeServices, nil +} diff --git a/agent/catalog_endpoint_test.go b/agent/catalog_endpoint_test.go index fa64456b0..3b13c98fd 100644 --- a/agent/catalog_endpoint_test.go +++ b/agent/catalog_endpoint_test.go @@ -1118,6 +1118,56 @@ func TestCatalogNodeServices(t *testing.T) { require.Equal(t, args.Service.Proxy, services.Services["web-proxy"].Proxy) } +func TestCatalogNodeServiceList(t *testing.T) { + t.Parallel() + a := NewTestAgent(t, t.Name(), "") + defer a.Shutdown() + testrpc.WaitForTestAgent(t, a.RPC, "dc1") + + // Register node with a regular service and connect proxy + args := &structs.RegisterRequest{ + Datacenter: "dc1", + Node: "foo", + Address: "127.0.0.1", + Service: &structs.NodeService{ + Service: "api", + Tags: []string{"a"}, + }, + } + + var out struct{} + if err := a.RPC("Catalog.Register", args, &out); err != nil { + t.Fatalf("err: %v", err) + } + + // Register a connect proxy + args.Service = structs.TestNodeServiceProxy(t) + require.NoError(t, a.RPC("Catalog.Register", args, &out)) + + req, _ := http.NewRequest("GET", "/v1/catalog/node-services/foo?dc=dc1", nil) + resp := httptest.NewRecorder() + obj, err := a.srv.CatalogNodeServiceList(resp, req) + if err != nil { + t.Fatalf("err: %v", err) + } + assertIndex(t, resp) + + services := obj.(*structs.NodeServiceList) + if len(services.Services) != 2 { + t.Fatalf("bad: %v", obj) + } + + var proxySvc *structs.NodeService + for _, svc := range services.Services { + if svc.ID == "web-proxy" { + proxySvc = svc + } + } + require.NotNil(t, proxySvc, "Missing proxy service registration") + // Proxy service should have it's config intact + require.Equal(t, args.Service.Proxy, proxySvc.Proxy) +} + func TestCatalogNodeServices_Filter(t *testing.T) { t.Parallel() a := NewTestAgent(t, t.Name(), "") diff --git a/agent/consul/catalog_endpoint.go b/agent/consul/catalog_endpoint.go index 8019ed93a..1442def6c 100644 --- a/agent/consul/catalog_endpoint.go +++ b/agent/consul/catalog_endpoint.go @@ -520,7 +520,7 @@ func (c *Catalog) NodeServiceList(args *structs.NodeSpecificRequest, reply *stru return fmt.Errorf("Must provide node") } - var filterType map[string]*structs.NodeService + var filterType []*structs.NodeService filter, err := bexpr.CreateFilter(args.Filter, nil, filterType) if err != nil { return err diff --git a/agent/http_register.go b/agent/http_register.go index 096e32cf6..b498969ce 100644 --- a/agent/http_register.go +++ b/agent/http_register.go @@ -66,6 +66,7 @@ func init() { registerEndpoint("/v1/catalog/services", []string{"GET"}, (*HTTPServer).CatalogServices) registerEndpoint("/v1/catalog/service/", []string{"GET"}, (*HTTPServer).CatalogServiceNodes) registerEndpoint("/v1/catalog/node/", []string{"GET"}, (*HTTPServer).CatalogNodeServices) + registerEndpoint("/v1/catalog/node-services/", []string{"GET"}, (*HTTPServer).CatalogNodeServiceList) registerEndpoint("/v1/config/", []string{"GET", "DELETE"}, (*HTTPServer).Config) registerEndpoint("/v1/config", []string{"PUT"}, (*HTTPServer).ConfigApply) registerEndpoint("/v1/connect/ca/configuration", []string{"GET", "PUT"}, (*HTTPServer).ConnectCAConfiguration) diff --git a/agent/translate_addr.go b/agent/translate_addr.go index 45bf8c912..7967347ec 100644 --- a/agent/translate_addr.go +++ b/agent/translate_addr.go @@ -149,6 +149,14 @@ func (a *Agent) TranslateAddresses(dc string, subj interface{}, accept Translate entry.Address = a.TranslateServiceAddress(dc, entry.Address, entry.TaggedAddresses, accept) entry.Port = a.TranslateServicePort(dc, entry.Port, entry.TaggedAddresses) } + case *structs.NodeServiceList: + if v.Node != nil { + v.Node.Address = a.TranslateAddress(dc, v.Node.Address, v.Node.TaggedAddresses, accept) + } + for _, entry := range v.Services { + entry.Address = a.TranslateServiceAddress(dc, entry.Address, entry.TaggedAddresses, accept) + entry.Port = a.TranslateServicePort(dc, entry.Port, entry.TaggedAddresses) + } default: panic(fmt.Errorf("Unhandled type passed to address translator: %#v", subj)) } diff --git a/api/catalog.go b/api/catalog.go index 5647dc637..dd34c17db 100644 --- a/api/catalog.go +++ b/api/catalog.go @@ -54,6 +54,11 @@ type CatalogNode struct { Services map[string]*AgentService } +type CatalogNodeServiceList struct { + Node *Node + Services []*AgentService +} + type CatalogRegistration struct { ID string Node string @@ -254,6 +259,30 @@ func (c *Catalog) Node(node string, q *QueryOptions) (*CatalogNode, *QueryMeta, return out, qm, nil } +// NodeServiceList is used to query for service information about a single node. It differs from +// the Node function only in its return type which will contain a list of services as opposed to +// a map of service ids to services. This different structure allows for using the wildcard specifier +// '*' for the Namespace in the QueryOptions. +func (c *Catalog) NodeServiceList(node string, q *QueryOptions) (*CatalogNodeServiceList, *QueryMeta, error) { + r := c.c.newRequest("GET", "/v1/catalog/node-services/"+node) + r.setQueryOptions(q) + rtt, resp, err := requireOK(c.c.doRequest(r)) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + qm := &QueryMeta{} + parseQueryMeta(resp, qm) + qm.RequestTime = rtt + + var out *CatalogNodeServiceList + if err := decodeBody(resp, &out); err != nil { + return nil, nil, err + } + return out, qm, nil +} + func ParseServiceAddr(addrPort string) (ServiceAddress, error) { port := 0 host, portStr, err := net.SplitHostPort(addrPort) diff --git a/api/catalog_test.go b/api/catalog_test.go index e0c4f9073..8b74c6fe8 100644 --- a/api/catalog_test.go +++ b/api/catalog_test.go @@ -726,6 +726,65 @@ func TestAPI_CatalogNode(t *testing.T) { }) } +func TestAPI_CatalogNodeServiceList(t *testing.T) { + t.Parallel() + c, s := makeClient(t) + defer s.Stop() + + catalog := c.Catalog() + + name, err := c.Agent().NodeName() + require.NoError(t, err) + + proxyReg := testUnmanagedProxyRegistration(t) + proxyReg.Node = name + proxyReg.SkipNodeUpdate = true + + retry.Run(t, func(r *retry.R) { + // Register a connect proxy to ensure all it's config fields are returned + _, err := catalog.Register(proxyReg, nil) + r.Check(err) + + info, meta, err := catalog.NodeServiceList(name, nil) + if err != nil { + r.Fatal(err) + } + + if meta.LastIndex == 0 { + r.Fatalf("Bad: %v", meta) + } + + if len(info.Services) != 2 { + r.Fatalf("Bad: %v (len %d)", info, len(info.Services)) + } + + if _, ok := info.Node.TaggedAddresses["wan"]; !ok { + r.Fatalf("Bad: %v", info.Node.TaggedAddresses) + } + + if info.Node.Datacenter != "dc1" { + r.Fatalf("Bad datacenter: %v", info) + } + + var proxySvc *AgentService + for _, svc := range info.Services { + if svc.ID == "web-proxy1" { + proxySvc = svc + break + } + } + + if proxySvc == nil { + r.Fatalf("Missing proxy service: %v", info.Services) + } + + if !reflect.DeepEqual(proxyReg.Service.Proxy, proxySvc.Proxy) { + r.Fatalf("Bad proxy config:\nwant %v\n got: %v", proxyReg.Service.Proxy, + proxySvc.Proxy) + } + }) +} + func TestAPI_CatalogNode_Filter(t *testing.T) { t.Parallel() c, s := makeClient(t) diff --git a/website/source/api/catalog.html.md b/website/source/api/catalog.html.md index 1af47298b..72565777d 100644 --- a/website/source/api/catalog.html.md +++ b/website/source/api/catalog.html.md @@ -655,7 +655,7 @@ so this endpoint may be used to filter only the Connect-capable endpoints. Parameters and response format are the same as [`/catalog/service/:service`](/api/catalog.html#list-nodes-for-service). -## List Services for Node +## Retrieve Map of Services for a Node This endpoint returns the node's registered services. @@ -783,3 +783,133 @@ top level Node object. The following selectors and filter operations are support | `Tags` | In, Not In, Is Empty, Is Not Empty | | `Weights.Passing` | Equal, Not Equal | | `Weights.Warning` | Equal, Not Equal | + +## List Services for Node + +This endpoint returns the node's registered services. + +| Method | Path | Produces | +| ------ | ------------------------------ | -------------------------- | +| `GET` | `/catalog/node-services/:node` | `application/json` | + +The table below shows this endpoint's support for +[blocking queries](/api/features/blocking.html), +[consistency modes](/api/features/consistency.html), +[agent caching](/api/features/caching.html), and +[required ACLs](/api/index.html#authentication). + +| Blocking Queries | Consistency Modes | Agent Caching | ACL Required | +| ---------------- | ----------------- | ------------- | ------------------------ | +| `YES` | `all` | `none` | `node:read,service:read` | + +### Parameters + +- `node` `(string: )` - Specifies the name of the node for which + to list services. This is specified as part of the URL. + +- `dc` `(string: "")` - Specifies the datacenter to query. This will default to + the datacenter of the agent being queried. This is specified as part of the + URL as a query parameter. + +- `filter` `(string: "")` - Specifies the expression used to filter the + queries results prior to returning the data. + +- `ns` `(string: "")` - **(Enterprise Only)** Specifies the namespace to list services. + This value may be provided by either the `ns` URL query parameter or in the + `X-Consul-Namespace` header. If not provided at all, the namespace will be inherited + from the request's ACL token or will default to the `default` namespace. The `*` + wildcard may be used and then services from all namespaces will be returned. Added in Consul 1.7.0. + +### Sample Request + +```text +$ curl \ + http://127.0.0.1:8500/v1/catalog/node-services/my-node +``` + +### Sample Response + +```json +{ + "Node": { + "ID": "40e4a748-2192-161a-0510-9bf59fe950b5", + "Node": "foobar", + "Address": "10.1.10.12", + "Datacenter": "dc1", + "TaggedAddresses": { + "lan": "10.1.10.12", + "wan": "10.1.10.12" + }, + "Meta": { + "instance_type": "t2.medium" + } + }, + "Services": [ + { + "ID": "consul", + "Service": "consul", + "Tags": null, + "Meta": {}, + "Port": 8300 + }, + { + "ID": "redis", + "Service": "redis", + "TaggedAddresses": { + "lan": { + "address": "10.1.10.12", + "port": 8000, + }, + "wan": { + "address": "198.18.1.2", + "port": 80 + } + }, + "Tags": [ + "v1" + ], + "Meta": { + "redis_version": "4.0" + }, + "Port": 8000, + "Namespace": "default" + } + } +} +``` + +### Filtering + +The filter will be executed against each value in the `Services` list within the +top level object. The following selectors and filter operations are supported: + +| Selector | Supported Operations | +| -------------------------------------- | -------------------------------------------------- | +| `Address` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `Connect.Native` | Equal, Not Equal | +| `EnableTagOverride` | Equal, Not Equal | +| `ID` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `Kind` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `Meta` | Is Empty, Is Not Empty, In, Not In | +| `Meta.` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `Port` | Equal, Not Equal | +| `Proxy.DestinationServiceID` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `Proxy.DestinationServiceName` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `Proxy.LocalServiceAddress` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `Proxy.LocalServicePort` | Equal, Not Equal | +| `Proxy.MeshGateway.Mode` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `Proxy.Upstreams` | Is Empty, Is Not Empty | +| `Proxy.Upstreams.Datacenter` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `Proxy.Upstreams.DestinationName` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `Proxy.Upstreams.DestinationNamespace` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `Proxy.Upstreams.DestinationType` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `Proxy.Upstreams.LocalBindAddress` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `Proxy.Upstreams.LocalBindPort` | Equal, Not Equal | +| `Proxy.Upstreams.MeshGateway.Mode` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `Service` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `TaggedAddresses` | Is Empty, Is Not Empty, In, Not In | +| `TaggedAddresses..Address` | Equal, Not Equal, In, Not In, Matches, Not Matches | +| `TaggedAddresses..Port` | Equal, Not Equal | +| `Tags` | In, Not In, Is Empty, Is Not Empty | +| `Weights.Passing` | Equal, Not Equal | +| `Weights.Warning` | Equal, Not Equal |