luxolus
/
open-consul
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Wraps the prepared query to also store the compiled template.

This commit is contained in:
James Phillips 2016-02-26 00:25:44 -08:00
parent 98281be7df
commit 62405110dc
7 changed files with 113 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
consul/fsm.go
View File

@ -610,9 +610,9 @@ func (s *consulSnapshot) persistPreparedQueries(sink raft.SnapshotSink,
return err
}
for query := queries.Next(); query != nil; query = queries.Next() {
for _, query := range queries {
sink.Write([]byte{byte(structs.PreparedQueryRequestType)})
if err := encoder.Encode(query.(*structs.PreparedQuery)); err != nil {
if err := encoder.Encode(query); err != nil {
return err
}
}

4
consul/prepared_query_endpoint.go
View File

@ -134,6 +134,8 @@ func parseQuery(query *structs.PreparedQuery) error {
// transaction. Otherwise, people could "steal" queries that they don't
// have proper ACL rights to change.
// - Session is optional and checked for integrity during the transaction.
// - Template is checked during the transaction since that's where we
// compile it.
// Token is checked when the query is executed, but we do make sure the
// user hasn't accidentally pasted-in the special redacted token name,
@ -162,7 +164,7 @@ func parseQuery(query *structs.PreparedQuery) error {
func parseService(svc *structs.ServiceQuery) error {
// Service is required.
if svc.Service == "" {
return fmt.Errorf("Must provide a service name to query")
return fmt.Errorf("Must provide a Service name to query")
}
// NearestN can be 0 which means "don't fail over by RTT".

7
consul/prepared_query_endpoint_test.go
View File

@ -530,7 +530,7 @@ func TestPreparedQuery_parseQuery(t *testing.T) {
query := &structs.PreparedQuery{}
err := parseQuery(query)
if err == nil || !strings.Contains(err.Error(), "Must provide a service") {
if err == nil || !strings.Contains(err.Error(), "Must provide a Service") {
t.Fatalf("bad: %v", err)
}
@ -1910,6 +1910,11 @@ func TestPreparedQuery_tagFilter(t *testing.T) {
if ret != "node2|node6" {
t.Fatalf("bad: %s", ret)
}
ret = stringify(tagFilter([]string{""}, testNodes()))
if ret != "" {
t.Fatalf("bad: %s", ret)
}
}
func TestPreparedQuery_Wrapper(t *testing.T) {

102
consul/state/prepared_query.go
View File

@ -4,6 +4,7 @@ import (
"fmt"
"regexp"
"github.com/hashicorp/consul/consul/prepared_query"
"github.com/hashicorp/consul/consul/structs"
"github.com/hashicorp/go-memdb"
)
@ -16,22 +17,54 @@ func isUUID(str string) bool {
return validUUID.MatchString(str)
}
// queryWrapper is an internal structure that is used to store a query alongside
// its compiled template, which can be nil.
type queryWrapper struct {
*structs.PreparedQuery
ct *prepared_query.CompiledTemplate
}
// toPreparedQuery unwraps the internal form of a prepared query and returns
// the regular struct.
func toPreparedQuery(wrapped interface{}) *structs.PreparedQuery {
if wrapped == nil {
return nil
}
return wrapped.(*queryWrapper).PreparedQuery
}
// PreparedQueries is used to pull all the prepared queries from the snapshot.
func (s *StateSnapshot) PreparedQueries() (memdb.ResultIterator, error) {
iter, err := s.tx.Get("prepared-queries", "id")
func (s *StateSnapshot) PreparedQueries() (structs.PreparedQueries, error) {
queries, err := s.tx.Get("prepared-queries", "id")
if err != nil {
return nil, err
}
return iter, nil
var ret structs.PreparedQueries
for wrapped := queries.Next(); wrapped != nil; wrapped = queries.Next() {
ret = append(ret, toPreparedQuery(wrapped))
}
return ret, nil
}
// PrepparedQuery is used when restoring from a snapshot. For general inserts,
// use PreparedQuerySet.
func (s *StateRestore) PreparedQuery(query *structs.PreparedQuery) error {
if err := s.tx.Insert("prepared-queries", query); err != nil {
return fmt.Errorf("failed restoring prepared query: %s", err)
// If this is a template, compile it, otherwise leave the compiled
// template field nil.
var ct *prepared_query.CompiledTemplate
if prepared_query.IsTemplate(query) {
var err error
ct, err = prepared_query.Compile(query)
if err != nil {
return fmt.Errorf("failed compiling template: %s", err)
}
}
// Insert the wrapped query.
if err := s.tx.Insert("prepared-queries", &queryWrapper{query, ct}); err != nil {
return fmt.Errorf("failed restoring prepared query: %s", err)
}
if err := indexUpdateMaxTxn(s.tx, query.ModifyIndex, "prepared-queries"); err != nil {
return fmt.Errorf("failed updating index: %s", err)
}
@ -62,14 +95,15 @@ func (s *StateStore) preparedQuerySetTxn(tx *memdb.Txn, idx uint64, query *struc
}
// Check for an existing query.
existing, err := tx.First("prepared-queries", "id", query.ID)
wrapped, err := tx.First("prepared-queries", "id", query.ID)
if err != nil {
return fmt.Errorf("failed prepared query lookup: %s", err)
}
existing := toPreparedQuery(wrapped)
// Set the indexes.
if existing != nil {
query.CreateIndex = existing.(*structs.PreparedQuery).CreateIndex
query.CreateIndex = existing.CreateIndex
query.ModifyIndex = idx
} else {
query.CreateIndex = idx
@ -79,12 +113,12 @@ func (s *StateStore) preparedQuerySetTxn(tx *memdb.Txn, idx uint64, query *struc
// Verify that the query name doesn't already exist, or that we are
// updating the same instance that has this name.
if query.Name != "" {
alias, err := tx.First("prepared-queries", "name", query.Name)
wrapped, err := tx.First("prepared-queries", "name", query.Name)
if err != nil {
return fmt.Errorf("failed prepared query lookup: %s", err)
}
if alias != nil && (existing == nil ||
existing.(*structs.PreparedQuery).ID != alias.(*structs.PreparedQuery).ID) {
other := toPreparedQuery(wrapped)
if other != nil && (existing == nil || existing.ID != other.ID) {
return fmt.Errorf("name '%s' aliases an existing query name", query.Name)
}
}
@ -99,11 +133,11 @@ func (s *StateStore) preparedQuerySetTxn(tx *memdb.Txn, idx uint64, query *struc
// index will complain if we look up something that's not formatted
// like one.
if isUUID(query.Name) {
alias, err := tx.First("prepared-queries", "id", query.Name)
wrapped, err := tx.First("prepared-queries", "id", query.Name)
if err != nil {
return fmt.Errorf("failed prepared query lookup: %s", err)
}
if alias != nil {
if wrapped != nil {
return fmt.Errorf("name '%s' aliases an existing query ID", query.Name)
}
}
@ -123,8 +157,19 @@ func (s *StateStore) preparedQuerySetTxn(tx *memdb.Txn, idx uint64, query *struc
// checked at execute time and not doing integrity checking on them
// helps avoid bootstrapping chicken and egg problems.
// Insert the query.
if err := tx.Insert("prepared-queries", query); err != nil {
// If this is a template, compile it, otherwise leave the compiled
// template field nil.
var ct *prepared_query.CompiledTemplate
if prepared_query.IsTemplate(query) {
var err error
ct, err = prepared_query.Compile(query)
if err != nil {
return fmt.Errorf("failed compiling template: %s", err)
}
}
// Insert the wrapped query.
if err := tx.Insert("prepared-queries", &queryWrapper{query, ct}); err != nil {
return fmt.Errorf("failed inserting prepared query: %s", err)
}
if err := tx.Insert("index", &IndexEntry{"prepared-queries", idx}); err != nil {
@ -155,16 +200,16 @@ func (s *StateStore) PreparedQueryDelete(idx uint64, queryID string) error {
func (s *StateStore) preparedQueryDeleteTxn(tx *memdb.Txn, idx uint64, watches *DumbWatchManager,
queryID string) error {
// Pull the query.
query, err := tx.First("prepared-queries", "id", queryID)
wrapped, err := tx.First("prepared-queries", "id", queryID)
if err != nil {
return fmt.Errorf("failed prepared query lookup: %s", err)
}
if query == nil {
if wrapped == nil {
return nil
}
// Delete the query and update the index.
if err := tx.Delete("prepared-queries", query); err != nil {
if err := tx.Delete("prepared-queries", wrapped); err != nil {
return fmt.Errorf("failed prepared query delete: %s", err)
}
if err := tx.Insert("index", &IndexEntry{"prepared-queries", idx}); err != nil {
@ -184,14 +229,11 @@ func (s *StateStore) PreparedQueryGet(queryID string) (uint64, *structs.Prepared
idx := maxIndexTxn(tx, s.getWatchTables("PreparedQueryGet")...)
// Look up the query by its ID.
query, err := tx.First("prepared-queries", "id", queryID)
wrapped, err := tx.First("prepared-queries", "id", queryID)
if err != nil {
return 0, nil, fmt.Errorf("failed prepared query lookup: %s", err)
}
if query != nil {
return idx, query.(*structs.PreparedQuery), nil
}
return idx, nil, nil
return idx, toPreparedQuery(wrapped), nil
}
// PreparedQueryLookup returns the given prepared query by looking up an ID or
@ -215,22 +257,22 @@ func (s *StateStore) PreparedQueryLookup(queryIDOrName string) (uint64, *structs
// format before trying this because the UUID index will complain if
// we look up something that's not formatted like one.
if isUUID(queryIDOrName) {
query, err := tx.First("prepared-queries", "id", queryIDOrName)
wrapped, err := tx.First("prepared-queries", "id", queryIDOrName)
if err != nil {
return 0, nil, fmt.Errorf("failed prepared query lookup: %s", err)
}
if query != nil {
return idx, query.(*structs.PreparedQuery), nil
if wrapped != nil {
return idx, toPreparedQuery(wrapped), nil
}
}
// Then try by name.
query, err := tx.First("prepared-queries", "name", queryIDOrName)
wrapped, err := tx.First("prepared-queries", "name", queryIDOrName)
if err != nil {
return 0, nil, fmt.Errorf("failed prepared query lookup: %s", err)
}
if query != nil {
return idx, query.(*structs.PreparedQuery), nil
if wrapped != nil {
return idx, toPreparedQuery(wrapped), nil
}
return idx, nil, nil
@ -252,8 +294,8 @@ func (s *StateStore) PreparedQueryList() (uint64, structs.PreparedQueries, error
// Go over all of the queries and build the response.
var result structs.PreparedQueries
for query := queries.Next(); query != nil; query = queries.Next() {
result = append(result, query.(*structs.PreparedQuery))
for wrapped := queries.Next(); wrapped != nil; wrapped = queries.Next() {
result = append(result, toPreparedQuery(wrapped))
}
return idx, result, nil
}

6
consul/state/prepared_query_test.go
View File

@ -581,14 +581,10 @@ func TestStateStore_PreparedQuery_Snapshot_Restore(t *testing.T) {
},
},
}
iter, err := snap.PreparedQueries()
dump, err := snap.PreparedQueries()
if err != nil {
t.Fatalf("err: %s", err)
}
var dump structs.PreparedQueries
for query := iter.Next(); query != nil; query = iter.Next() {
dump = append(dump, query.(*structs.PreparedQuery))
}
if !reflect.DeepEqual(dump, expected) {
t.Fatalf("bad: %v", dump)
}

11
consul/state/state_store.go
View File

@ -2151,15 +2151,14 @@ func (s *StateStore) deleteSessionTxn(tx *memdb.Txn, idx uint64, watches *DumbWa
return fmt.Errorf("failed prepared query lookup: %s", err)
}
{
var objs []interface{}
for query := queries.Next(); query != nil; query = queries.Next() {
objs = append(objs, query)
var ids []string
for wrapped := queries.Next(); wrapped != nil; wrapped = queries.Next() {
ids = append(ids, toPreparedQuery(wrapped).ID)
}
// Do the delete in a separate loop so we don't trash the iterator.
for _, obj := range objs {
q := obj.(*structs.PreparedQuery)
if err := s.preparedQueryDeleteTxn(tx, idx, watches, q.ID); err != nil {
for _, id := range ids {
if err := s.preparedQueryDeleteTxn(tx, idx, watches, id); err != nil {
return fmt.Errorf("failed prepared query delete: %s", err)
}
}

24
consul/structs/prepared_query.go
View File

@ -40,6 +40,25 @@ type ServiceQuery struct {
Tags []string
}
const (
// QueryTemplateTypeNamePrefixMatch uses the Name field of the query as
// a prefix to select the template.
QueryTemplateTypeNamePrefixMatch = "name_prefix_match"
)
// QueryTemplateOptions controls settings if this query is a template.
type QueryTemplateOptions struct {
// Type, if non-empty, means that this query is a template. This is
// set to one of the QueryTemplateType* constants above.
Type string
// Regexp is an optional regular expression to use to parse the full
// name, once the prefix match has selected a template. This can be
// used to extract parts of the name and choose a service name, set
// tags, etc.
Regexp string
}
// PreparedQuery defines a complete prepared query, and is the structure we
// maintain in the state store.
type PreparedQuery struct {
@ -61,6 +80,11 @@ type PreparedQuery struct {
// with management privileges, must be used to change the query later.
Token string
// Template is used to configure this query as a template, which will
// respond to queries based on the Name, and then will be rendered
// before it is executed.
Template QueryTemplateOptions
// Service defines a service query (leaving things open for other types
// later).
Service ServiceQuery