luxolus
/
open-consul
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Add agent.segment interpolation to prepared queries

This commit is contained in:
Kyle Havlovitz 2017-08-29 17:02:50 -07:00
parent 107d7f6c5a
commit 1c04f1537a
No known key found for this signature in database
GPG Key ID: 8A5E6B173056AD6C
12 changed files with 74 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
agent/config.go
View File

@ -358,7 +358,6 @@ type NetworkSegment struct {
// Bind is the bind address for this segment.
Bind string `mapstructure:"bind"`
BindAddrs []string `mapstructure:"-"`
// Port is the port for this segment.
Port int `mapstructure:"port"`
@ -1467,7 +1466,7 @@ func DecodeConfig(r io.Reader) (*Config, error) {
}
// Validate node meta fields
if err := structs.ValidateMetadata(result.Meta); err != nil {
if err := structs.ValidateMetadata(result.Meta, false); err != nil {
return nil, fmt.Errorf("Failed to parse node metadata: %v", err)
}

5
agent/config_test.go
View File

@ -597,11 +597,10 @@ func TestDecodeConfig(t *testing.T) {
c: &Config{Segment: "thing"},
},
{
in: `{"server": true, "segments":[{"name": "alpha", "bind": "127.0.0.1", "port": 1234, "rpc_listener": true, "advertise": "1.1.1.1"}]}`,
c: &Config{Server: true, Segments: []NetworkSegment{{
in: `{"segments":[{"name": "alpha", "bind": "127.0.0.1", "port": 1234, "rpc_listener": true, "advertise": "1.1.1.1"}]}`,
c: &Config{Segments: []NetworkSegment{{
Name: "alpha",
Bind: "127.0.0.1",
BindAddrs: []string{"127.0.0.1"},
Port: 1234,
RPCListener: true,
Advertise: "1.1.1.1",

8
agent/consul/prepared_query/template.go
View File

@ -89,7 +89,7 @@ func Compile(query *structs.PreparedQuery) (*CompiledTemplate, error) {
// prefix it will be expected to run with. The results might not make
// sense and create a valid service to lookup, but it should render
// without any errors.
if _, err = ct.Render(ct.query.Name); err != nil {
if _, err = ct.Render(ct.query.Name, structs.QuerySource{}); err != nil {
return nil, err
}
@ -99,7 +99,7 @@ func Compile(query *structs.PreparedQuery) (*CompiledTemplate, error) {
// Render takes a compiled template and renders it for the given name. For
// example, if the user looks up foobar.query.consul via DNS then we will call
// this function with "foobar" on the compiled template.
func (ct *CompiledTemplate) Render(name string) (*structs.PreparedQuery, error) {
func (ct *CompiledTemplate) Render(name string, source structs.QuerySource) (*structs.PreparedQuery, error) {
// Make it "safe" to render a default structure.
if ct == nil {
return nil, fmt.Errorf("Cannot render an uncompiled template")
@ -156,6 +156,10 @@ func (ct *CompiledTemplate) Render(name string) (*structs.PreparedQuery, error)
Type: ast.TypeString,
Value: strings.TrimPrefix(name, query.Name),
},
"agent.segment": ast.Variable{
Type: ast.TypeString,
Value: source.Segment,
},
},
FuncMap: map[string]ast.Function{
"match": match,

32
agent/consul/prepared_query/template_test.go
View File

@ -29,6 +29,7 @@ var (
"${match(0)}",
"${match(1)}",
"${match(2)}",
"${agent.segment}",
},
},
Tags: []string{
@ -38,11 +39,13 @@ var (
"${match(0)}",
"${match(1)}",
"${match(2)}",
"${agent.segment}",
},
NodeMeta: map[string]string{
"foo": "${name.prefix}",
"bar": "${match(0)}",
"baz": "${match(1)}",
"zoo": "${agent.segment}",
},
},
}
@ -83,7 +86,7 @@ func renderBench(b *testing.B, query *structs.PreparedQuery) {
}
for i := 0; i < b.N; i++ {
_, err := compiled.Render("hello-bench-mark")
_, err := compiled.Render("hello-bench-mark", structs.QuerySource{})
if err != nil {
b.Fatalf("err: %v", err)
}
@ -121,7 +124,7 @@ func TestTemplate_Compile(t *testing.T) {
query.Template.Type = structs.QueryTemplateTypeNamePrefixMatch
query.Template.Regexp = "^(hello)there$"
query.Service.Service = "${name.full}"
query.Service.Tags = []string{"${match(1)}"}
query.Service.Tags = []string{"${match(1)}", "${agent.segment}"}
backup, err := copystructure.Copy(query)
if err != nil {
t.Fatalf("err: %v", err)
@ -135,7 +138,7 @@ func TestTemplate_Compile(t *testing.T) {
}
// Do a sanity check render on it.
actual, err := ct.Render("hellothere")
actual, err := ct.Render("hellothere", structs.QuerySource{Segment: "segment-foo"})
if err != nil {
t.Fatalf("err: %v", err)
}
@ -150,6 +153,7 @@ func TestTemplate_Compile(t *testing.T) {
Service: "hellothere",
Tags: []string{
"hello",
"segment-foo",
},
},
}
@ -201,7 +205,7 @@ func TestTemplate_Render(t *testing.T) {
t.Fatalf("err: %v", err)
}
actual, err := ct.Render("unused")
actual, err := ct.Render("unused", structs.QuerySource{})
if err != nil {
t.Fatalf("err: %v", err)
}
@ -218,7 +222,7 @@ func TestTemplate_Render(t *testing.T) {
Regexp: "^(.*?)-(.*?)-(.*)$",
},
Service: structs.ServiceQuery{
Service: "${name.prefix} xxx ${name.full} xxx ${name.suffix}",
Service: "${name.prefix} xxx ${name.full} xxx ${name.suffix} xxx ${agent.segment}",
Tags: []string{
"${match(-1)}",
"${match(0)}",
@ -238,7 +242,7 @@ func TestTemplate_Render(t *testing.T) {
// Run a case that matches the regexp.
{
actual, err := ct.Render("hello-foo-bar-none")
actual, err := ct.Render("hello-foo-bar-none", structs.QuerySource{Segment: "segment-bar"})
if err != nil {
t.Fatalf("err: %v", err)
}
@ -249,7 +253,7 @@ func TestTemplate_Render(t *testing.T) {
Regexp: "^(.*?)-(.*?)-(.*)$",
},
Service: structs.ServiceQuery{
Service: "hello- xxx hello-foo-bar-none xxx foo-bar-none",
Service: "hello- xxx hello-foo-bar-none xxx foo-bar-none xxx segment-bar",
Tags: []string{
"",
"hello-foo-bar-none",
@ -269,7 +273,7 @@ func TestTemplate_Render(t *testing.T) {
// Run a case that doesn't match the regexp
{
actual, err := ct.Render("hello-nope")
actual, err := ct.Render("hello-nope", structs.QuerySource{Segment: "segment-bar"})
if err != nil {
t.Fatalf("err: %v", err)
}
@ -280,7 +284,7 @@ func TestTemplate_Render(t *testing.T) {
Regexp: "^(.*?)-(.*?)-(.*)$",
},
Service: structs.ServiceQuery{
Service: "hello- xxx hello-nope xxx nope",
Service: "hello- xxx hello-nope xxx nope xxx segment-bar",
Tags: []string{
"",
"",
@ -307,7 +311,7 @@ func TestTemplate_Render(t *testing.T) {
RemoveEmptyTags: true,
},
Service: structs.ServiceQuery{
Service: "${name.prefix} xxx ${name.full} xxx ${name.suffix}",
Service: "${name.prefix} xxx ${name.full} xxx ${name.suffix} xxx ${agent.segment}",
Tags: []string{
"${match(-1)}",
"${match(0)}",
@ -326,7 +330,7 @@ func TestTemplate_Render(t *testing.T) {
// Run a case that matches the regexp, removing empty tags.
{
actual, err := ct.Render("hello-foo-bar-none")
actual, err := ct.Render("hello-foo-bar-none", structs.QuerySource{Segment: "segment-baz"})
if err != nil {
t.Fatalf("err: %v", err)
}
@ -338,7 +342,7 @@ func TestTemplate_Render(t *testing.T) {
RemoveEmptyTags: true,
},
Service: structs.ServiceQuery{
Service: "hello- xxx hello-foo-bar-none xxx foo-bar-none",
Service: "hello- xxx hello-foo-bar-none xxx foo-bar-none xxx segment-baz",
Tags: []string{
"hello-foo-bar-none",
"hello",
@ -355,7 +359,7 @@ func TestTemplate_Render(t *testing.T) {
// Run a case that doesn't match the regexp, removing empty tags.
{
actual, err := ct.Render("hello-nope")
actual, err := ct.Render("hello-nope", structs.QuerySource{Segment: "segment-baz"})
if err != nil {
t.Fatalf("err: %v", err)
}
@ -367,7 +371,7 @@ func TestTemplate_Render(t *testing.T) {
RemoveEmptyTags: true,
},
Service: structs.ServiceQuery{
Service: "hello- xxx hello-nope xxx nope",
Service: "hello- xxx hello-nope xxx nope xxx segment-baz",
Tags: []string{
"42",
},

6
agent/consul/prepared_query_endpoint.go
View File

@ -182,7 +182,7 @@ func parseService(svc *structs.ServiceQuery) error {
}
// Make sure the metadata filters are valid
if err := structs.ValidateMetadata(svc.NodeMeta); err != nil {
if err := structs.ValidateMetadata(svc.NodeMeta, true); err != nil {
return err
}
@ -298,7 +298,7 @@ func (p *PreparedQuery) Explain(args *structs.PreparedQueryExecuteRequest,
// Try to locate the query.
state := p.srv.fsm.State()
_, query, err := state.PreparedQueryResolve(args.QueryIDOrName)
_, query, err := state.PreparedQueryResolve(args.QueryIDOrName, args.Agent)
if err != nil {
return err
}
@ -345,7 +345,7 @@ func (p *PreparedQuery) Execute(args *structs.PreparedQueryExecuteRequest,
// Try to locate the query.
state := p.srv.fsm.State()
_, query, err := state.PreparedQueryResolve(args.QueryIDOrName)
_, query, err := state.PreparedQueryResolve(args.QueryIDOrName, args.Agent)
if err != nil {
return err
}

4
agent/consul/state/prepared_query.go
View File

@ -256,7 +256,7 @@ func (s *Store) PreparedQueryGet(ws memdb.WatchSet, queryID string) (uint64, *st
// PreparedQueryResolve returns the given prepared query by looking up an ID or
// Name. If the query was looked up by name and it's a template, then the
// template will be rendered before it is returned.
func (s *Store) PreparedQueryResolve(queryIDOrName string) (uint64, *structs.PreparedQuery, error) {
func (s *Store) PreparedQueryResolve(queryIDOrName string, source structs.QuerySource) (uint64, *structs.PreparedQuery, error) {
tx := s.db.Txn(false)
defer tx.Abort()
@ -293,7 +293,7 @@ func (s *Store) PreparedQueryResolve(queryIDOrName string) (uint64, *structs.Pre
prep := func(wrapped interface{}) (uint64, *structs.PreparedQuery, error) {
wrapper := wrapped.(*queryWrapper)
if prepared_query.IsTemplate(wrapper.PreparedQuery) {
render, err := wrapper.ct.Render(queryIDOrName)
render, err := wrapper.ct.Render(queryIDOrName, source)
if err != nil {
return idx, nil, err
}

20
agent/consul/state/prepared_query_test.go
View File

@ -554,7 +554,7 @@ func TestStateStore_PreparedQueryResolve(t *testing.T) {
// Try to lookup a query that's not there using something that looks
// like a real ID.
idx, actual, err := s.PreparedQueryResolve(query.ID)
idx, actual, err := s.PreparedQueryResolve(query.ID, structs.QuerySource{})
if err != nil {
t.Fatalf("err: %s", err)
}
@ -567,7 +567,7 @@ func TestStateStore_PreparedQueryResolve(t *testing.T) {
// Try to lookup a query that's not there using something that looks
// like a name
idx, actual, err = s.PreparedQueryResolve(query.Name)
idx, actual, err = s.PreparedQueryResolve(query.Name, structs.QuerySource{})
if err != nil {
t.Fatalf("err: %s", err)
}
@ -600,7 +600,7 @@ func TestStateStore_PreparedQueryResolve(t *testing.T) {
ModifyIndex: 3,
},
}
idx, actual, err = s.PreparedQueryResolve(query.ID)
idx, actual, err = s.PreparedQueryResolve(query.ID, structs.QuerySource{})
if err != nil {
t.Fatalf("err: %s", err)
}
@ -612,7 +612,7 @@ func TestStateStore_PreparedQueryResolve(t *testing.T) {
}
// Read it back using the name and verify it again.
idx, actual, err = s.PreparedQueryResolve(query.Name)
idx, actual, err = s.PreparedQueryResolve(query.Name, structs.QuerySource{})
if err != nil {
t.Fatalf("err: %s", err)
}
@ -625,7 +625,7 @@ func TestStateStore_PreparedQueryResolve(t *testing.T) {
// Make sure an empty lookup is well-behaved if there are actual queries
// in the state store.
idx, actual, err = s.PreparedQueryResolve("")
idx, actual, err = s.PreparedQueryResolve("", structs.QuerySource{})
if err != ErrMissingQueryID {
t.Fatalf("bad: %v ", err)
}
@ -681,7 +681,7 @@ func TestStateStore_PreparedQueryResolve(t *testing.T) {
ModifyIndex: 4,
},
}
idx, actual, err = s.PreparedQueryResolve("prod-mongodb")
idx, actual, err = s.PreparedQueryResolve("prod-mongodb", structs.QuerySource{})
if err != nil {
t.Fatalf("err: %s", err)
}
@ -708,7 +708,7 @@ func TestStateStore_PreparedQueryResolve(t *testing.T) {
ModifyIndex: 5,
},
}
idx, actual, err = s.PreparedQueryResolve("prod-redis-foobar")
idx, actual, err = s.PreparedQueryResolve("prod-redis-foobar", structs.QuerySource{})
if err != nil {
t.Fatalf("err: %s", err)
}
@ -735,7 +735,7 @@ func TestStateStore_PreparedQueryResolve(t *testing.T) {
ModifyIndex: 4,
},
}
idx, actual, err = s.PreparedQueryResolve("prod-")
idx, actual, err = s.PreparedQueryResolve("prod-", structs.QuerySource{})
if err != nil {
t.Fatalf("err: %s", err)
}
@ -748,7 +748,7 @@ func TestStateStore_PreparedQueryResolve(t *testing.T) {
// Make sure you can't run a prepared query template by ID, since that
// makes no sense.
_, _, err = s.PreparedQueryResolve(tmpl1.ID)
_, _, err = s.PreparedQueryResolve(tmpl1.ID, structs.QuerySource{})
if err == nil || !strings.Contains(err.Error(), "prepared query templates can only be resolved up by name") {
t.Fatalf("bad: %v", err)
}
@ -960,7 +960,7 @@ func TestStateStore_PreparedQuery_Snapshot_Restore(t *testing.T) {
// Make sure the second query, which is a template, was compiled
// and can be resolved.
_, query, err := s.PreparedQueryResolve("bob-backwards-is-bob")
_, query, err := s.PreparedQueryResolve("bob-backwards-is-bob", structs.QuerySource{})
if err != nil {
t.Fatalf("err: %s", err)
}

2
agent/prepared_query_endpoint.go
View File

@ -96,6 +96,7 @@ func (s *HTTPServer) preparedQueryExecute(id string, resp http.ResponseWriter, r
Agent: structs.QuerySource{
Node: s.agent.config.NodeName,
Datacenter: s.agent.config.Datacenter,
Segment: s.agent.config.Segment,
},
}
s.parseSource(req, &args.Source)
@ -140,6 +141,7 @@ func (s *HTTPServer) preparedQueryExplain(id string, resp http.ResponseWriter, r
Agent: structs.QuerySource{
Node: s.agent.config.NodeName,
Datacenter: s.agent.config.Datacenter,
Segment: s.agent.config.Segment,
},
}
s.parseSource(req, &args.Source)

9
agent/structs/structs.go
View File

@ -242,6 +242,7 @@ func (r *DeregisterRequest) RequestDatacenter() string {
// coordinates.
type QuerySource struct {
Datacenter string
Segment string
Node string
}
@ -310,13 +311,13 @@ type Node struct {
type Nodes []*Node
// ValidateMeta validates a set of key/value pairs from the agent config
func ValidateMetadata(meta map[string]string) error {
func ValidateMetadata(meta map[string]string, allowConsulPrefix bool) error {
if len(meta) > metaMaxKeyPairs {
return fmt.Errorf("Node metadata cannot contain more than %d key/value pairs", metaMaxKeyPairs)
}
for key, value := range meta {
if err := validateMetaPair(key, value); err != nil {
if err := validateMetaPair(key, value, allowConsulPrefix); err != nil {
return fmt.Errorf("Couldn't load metadata pair ('%s', '%s'): %s", key, value, err)
}
}
@ -325,7 +326,7 @@ func ValidateMetadata(meta map[string]string) error {
}
// validateMetaPair checks that the given key/value pair is in a valid format
func validateMetaPair(key, value string) error {
func validateMetaPair(key, value string, allowConsulPrefix bool) error {
if key == "" {
return fmt.Errorf("Key cannot be blank")
}
@ -335,7 +336,7 @@ func validateMetaPair(key, value string) error {
if len(key) > metaKeyMaxLength {
return fmt.Errorf("Key is too long (limit: %d characters)", metaKeyMaxLength)
}
if strings.HasPrefix(key, metaKeyReservedPrefix) {
if strings.HasPrefix(key, metaKeyReservedPrefix) && !allowConsulPrefix {
return fmt.Errorf("Key prefix '%s' is reserved for internal use", metaKeyReservedPrefix)
}
if len(value) > metaValueMaxLength {

8
agent/structs/structs_test.go
View File

@ -482,7 +482,7 @@ func TestStructs_ValidateMetadata(t *testing.T) {
"key2": "value2",
}
// Should succeed
if err := ValidateMetadata(meta); err != nil {
if err := ValidateMetadata(meta, false); err != nil {
t.Fatalf("err: %s", err)
}
@ -490,7 +490,7 @@ func TestStructs_ValidateMetadata(t *testing.T) {
meta = map[string]string{
"": "value1",
}
if err := ValidateMetadata(meta); !strings.Contains(err.Error(), "Couldn't load metadata pair") {
if err := ValidateMetadata(meta, false); !strings.Contains(err.Error(), "Couldn't load metadata pair") {
t.Fatalf("should have failed")
}
@ -499,7 +499,7 @@ func TestStructs_ValidateMetadata(t *testing.T) {
for i := 0; i < metaMaxKeyPairs+1; i++ {
meta[string(i)] = "value"
}
if err := ValidateMetadata(meta); !strings.Contains(err.Error(), "cannot contain more than") {
if err := ValidateMetadata(meta, false); !strings.Contains(err.Error(), "cannot contain more than") {
t.Fatalf("should have failed")
}
}
@ -529,7 +529,7 @@ func TestStructs_validateMetaPair(t *testing.T) {
}
for _, pair := range pairs {
err := validateMetaPair(pair.Key, pair.Value)
err := validateMetaPair(pair.Key, pair.Value, false)
if pair.Error == "" && err != nil {
t.Fatalf("should have succeeded: %v, %v", pair, err)
} else if pair.Error != "" && !strings.Contains(err.Error(), pair.Error) {

2
command/agent.go
View File

@ -225,7 +225,7 @@ func (cmd *AgentCommand) readConfig() *agent.Config {
key, value := agent.ParseMetaPair(entry)
cmdCfg.Meta[key] = value
}
if err := structs.ValidateMetadata(cmdCfg.Meta); err != nil {
if err := structs.ValidateMetadata(cmdCfg.Meta, false); err != nil {
cmd.UI.Error(fmt.Sprintf("Failed to parse node metadata: %v", err))
return nil
}

19
website/source/api/query.html.md
View File

@ -87,6 +87,25 @@ populate the query before it is executed. All of the string fields inside the
doesn't match, or an invalid index is given, then `${match(N)}` will return an
empty string.
- `${agent.segment}` has the network segment (Enterprise only) of the agent that
initiated the query. This can be used with the `NodeMeta` field to limit the results
of a query to service instances within its own network segment:
```json
{
"Name": "",
"Template": {
"Type": "name_prefix_match"
},
"Service": {
"Service": "${name.full}",
"NodeMeta": {"consul-network-segment": "${agent.segment}"}
}
}
```
This will map all names of the form "&lt;service&gt;.query.consul" over DNS to a query
that will select an instance of the service in the agent's own network segment.
Using templates, it is possible to apply prepared query behaviors to many
services with a single template. Here's an example template that matches any
query and applies a failover policy to it: