open-consul/agent/consul/state/prepared_query.go

399 lines
12 KiB
Go
Raw Normal View History

package state
import (
"fmt"
"regexp"
pkg refactor command/agent/* -> agent/* command/consul/* -> agent/consul/* command/agent/command{,_test}.go -> command/agent{,_test}.go command/base/command.go -> command/base.go command/base/* -> command/* commands.go -> command/commands.go The script which did the refactor is: ( cd $GOPATH/src/github.com/hashicorp/consul git mv command/agent/command.go command/agent.go git mv command/agent/command_test.go command/agent_test.go git mv command/agent/flag_slice_value{,_test}.go command/ git mv command/agent . git mv command/base/command.go command/base.go git mv command/base/config_util{,_test}.go command/ git mv commands.go command/ git mv consul agent rmdir command/base/ gsed -i -e 's|package agent|package command|' command/agent{,_test}.go gsed -i -e 's|package agent|package command|' command/flag_slice_value{,_test}.go gsed -i -e 's|package base|package command|' command/base.go command/config_util{,_test}.go gsed -i -e 's|package main|package command|' command/commands.go gsed -i -e 's|base.Command|BaseCommand|' command/commands.go gsed -i -e 's|agent.Command|AgentCommand|' command/commands.go gsed -i -e 's|\tCommand:|\tBaseCommand:|' command/commands.go gsed -i -e 's|base\.||' command/commands.go gsed -i -e 's|command\.||' command/commands.go gsed -i -e 's|command|c|' main.go gsed -i -e 's|range Commands|range command.Commands|' main.go gsed -i -e 's|Commands: Commands|Commands: command.Commands|' main.go gsed -i -e 's|base\.BoolValue|BoolValue|' command/operator_autopilot_set.go gsed -i -e 's|base\.DurationValue|DurationValue|' command/operator_autopilot_set.go gsed -i -e 's|base\.StringValue|StringValue|' command/operator_autopilot_set.go gsed -i -e 's|base\.UintValue|UintValue|' command/operator_autopilot_set.go gsed -i -e 's|\bCommand\b|BaseCommand|' command/base.go gsed -i -e 's|BaseCommand Options|Command Options|' command/base.go gsed -i -e 's|base.Command|BaseCommand|' command/*.go gsed -i -e 's|c\.Command|c.BaseCommand|g' command/*.go gsed -i -e 's|\tCommand:|\tBaseCommand:|' command/*_test.go gsed -i -e 's|base\.||' command/*_test.go gsed -i -e 's|\bCommand\b|AgentCommand|' command/agent{,_test}.go gsed -i -e 's|cmd.AgentCommand|cmd.BaseCommand|' command/agent.go gsed -i -e 's|cli.AgentCommand = new(Command)|cli.Command = new(AgentCommand)|' command/agent_test.go gsed -i -e 's|exec.AgentCommand|exec.Command|' command/agent_test.go gsed -i -e 's|exec.BaseCommand|exec.Command|' command/agent_test.go gsed -i -e 's|NewTestAgent|agent.NewTestAgent|' command/agent_test.go gsed -i -e 's|= TestConfig|= agent.TestConfig|' command/agent_test.go gsed -i -e 's|: RetryJoin|: agent.RetryJoin|' command/agent_test.go gsed -i -e 's|\.\./\.\./|../|' command/config_util_test.go gsed -i -e 's|\bverifyUniqueListeners|VerifyUniqueListeners|' agent/config{,_test}.go command/agent.go gsed -i -e 's|\bserfLANKeyring\b|SerfLANKeyring|g' agent/{agent,keyring,testagent}.go command/agent.go gsed -i -e 's|\bserfWANKeyring\b|SerfWANKeyring|g' agent/{agent,keyring,testagent}.go command/agent.go gsed -i -e 's|\bNewAgent\b|agent.New|g' command/agent{,_test}.go gsed -i -e 's|\bNewAgent|New|' agent/{acl_test,agent,testagent}.go gsed -i -e 's|\bAgent\b|agent.&|g' command/agent{,_test}.go gsed -i -e 's|\bBool\b|agent.&|g' command/agent{,_test}.go gsed -i -e 's|\bConfig\b|agent.&|g' command/agent{,_test}.go gsed -i -e 's|\bDefaultConfig\b|agent.&|g' command/agent{,_test}.go gsed -i -e 's|\bDevConfig\b|agent.&|g' command/agent{,_test}.go gsed -i -e 's|\bMergeConfig\b|agent.&|g' command/agent{,_test}.go gsed -i -e 's|\bReadConfigPaths\b|agent.&|g' command/agent{,_test}.go gsed -i -e 's|\bParseMetaPair\b|agent.&|g' command/agent{,_test}.go gsed -i -e 's|\bSerfLANKeyring\b|agent.&|g' command/agent{,_test}.go gsed -i -e 's|\bSerfWANKeyring\b|agent.&|g' command/agent{,_test}.go gsed -i -e 's|circonus\.agent|circonus|g' command/agent{,_test}.go gsed -i -e 's|logger\.agent|logger|g' command/agent{,_test}.go gsed -i -e 's|metrics\.agent|metrics|g' command/agent{,_test}.go gsed -i -e 's|// agent.Agent|// agent|' command/agent{,_test}.go gsed -i -e 's|a\.agent\.Config|a.Config|' command/agent{,_test}.go gsed -i -e 's|agent\.AppendSliceValue|AppendSliceValue|' command/{configtest,validate}.go gsed -i -e 's|consul/consul|agent/consul|' GNUmakefile gsed -i -e 's|\.\./test|../../test|' agent/consul/server_test.go # fix imports f=$(grep -rl 'github.com/hashicorp/consul/command/agent' * | grep '\.go') gsed -i -e 's|github.com/hashicorp/consul/command/agent|github.com/hashicorp/consul/agent|' $f goimports -w $f f=$(grep -rl 'github.com/hashicorp/consul/consul' * | grep '\.go') gsed -i -e 's|github.com/hashicorp/consul/consul|github.com/hashicorp/consul/agent/consul|' $f goimports -w $f goimports -w command/*.go main.go )
2017-06-09 22:28:28 +00:00
"github.com/hashicorp/consul/agent/consul/prepared_query"
"github.com/hashicorp/consul/agent/structs"
"github.com/hashicorp/go-memdb"
)
// preparedQueriesTableSchema returns a new table schema used for storing
// prepared queries.
func preparedQueriesTableSchema() *memdb.TableSchema {
return &memdb.TableSchema{
Name: "prepared-queries",
Indexes: map[string]*memdb.IndexSchema{
"id": &memdb.IndexSchema{
Name: "id",
AllowMissing: false,
Unique: true,
Indexer: &memdb.UUIDFieldIndex{
Field: "ID",
},
},
"name": &memdb.IndexSchema{
Name: "name",
AllowMissing: true,
Unique: true,
Indexer: &memdb.StringFieldIndex{
Field: "Name",
Lowercase: true,
},
},
"template": &memdb.IndexSchema{
Name: "template",
AllowMissing: true,
Unique: true,
Indexer: &PreparedQueryIndex{},
},
"session": &memdb.IndexSchema{
Name: "session",
AllowMissing: true,
Unique: false,
Indexer: &memdb.UUIDFieldIndex{
Field: "Session",
},
},
},
}
}
func init() {
registerSchema(preparedQueriesTableSchema)
}
// validUUID is used to check if a given string looks like a UUID
2015-11-17 06:57:47 +00:00
var validUUID = regexp.MustCompile(`(?i)^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$`)
// isUUID returns true if the given string is a valid UUID.
func isUUID(str string) bool {
const uuidLen = 36
if len(str) != uuidLen {
return false
}
return validUUID.MatchString(str)
}
// queryWrapper is an internal structure that is used to store a query alongside
// its compiled template, which can be nil.
type queryWrapper struct {
// We embed the PreparedQuery structure so that the UUID field indexer
// can see the ID directly.
*structs.PreparedQuery
// ct is the compiled template, or nil if the query isn't a template. The
// state store manages this and keeps it up to date every time the query
// changes.
ct *prepared_query.CompiledTemplate
}
// toPreparedQuery unwraps the internal form of a prepared query and returns
// the regular struct.
func toPreparedQuery(wrapped interface{}) *structs.PreparedQuery {
if wrapped == nil {
return nil
}
return wrapped.(*queryWrapper).PreparedQuery
}
// PreparedQueries is used to pull all the prepared queries from the snapshot.
2017-04-21 00:46:29 +00:00
func (s *Snapshot) PreparedQueries() (structs.PreparedQueries, error) {
queries, err := s.tx.Get("prepared-queries", "id")
if err != nil {
return nil, err
}
var ret structs.PreparedQueries
for wrapped := queries.Next(); wrapped != nil; wrapped = queries.Next() {
ret = append(ret, toPreparedQuery(wrapped))
}
return ret, nil
}
2017-03-21 23:36:44 +00:00
// PreparedQuery is used when restoring from a snapshot. For general inserts,
// use PreparedQuerySet.
2017-04-21 00:46:29 +00:00
func (s *Restore) PreparedQuery(query *structs.PreparedQuery) error {
// If this is a template, compile it, otherwise leave the compiled
// template field nil.
var ct *prepared_query.CompiledTemplate
if prepared_query.IsTemplate(query) {
var err error
ct, err = prepared_query.Compile(query)
if err != nil {
return fmt.Errorf("failed compiling template: %s", err)
}
}
// Insert the wrapped query.
if err := s.tx.Insert("prepared-queries", &queryWrapper{query, ct}); err != nil {
return fmt.Errorf("failed restoring prepared query: %s", err)
}
if err := indexUpdateMaxTxn(s.tx, query.ModifyIndex, "prepared-queries"); err != nil {
return fmt.Errorf("failed updating index: %s", err)
}
return nil
}
// PreparedQuerySet is used to create or update a prepared query.
2017-04-21 00:46:29 +00:00
func (s *Store) PreparedQuerySet(idx uint64, query *structs.PreparedQuery) error {
tx := s.db.Txn(true)
defer tx.Abort()
if err := s.preparedQuerySetTxn(tx, idx, query); err != nil {
return err
}
tx.Commit()
return nil
}
// preparedQuerySetTxn is the inner method used to insert a prepared query with
// the proper indexes into the state store.
2017-04-21 00:46:29 +00:00
func (s *Store) preparedQuerySetTxn(tx *memdb.Txn, idx uint64, query *structs.PreparedQuery) error {
// Check that the ID is set.
if query.ID == "" {
return ErrMissingQueryID
}
// Check for an existing query.
wrapped, err := tx.First("prepared-queries", "id", query.ID)
if err != nil {
return fmt.Errorf("failed prepared query lookup: %s", err)
}
existing := toPreparedQuery(wrapped)
// Set the indexes.
if existing != nil {
query.CreateIndex = existing.CreateIndex
query.ModifyIndex = idx
} else {
query.CreateIndex = idx
query.ModifyIndex = idx
}
// Verify that the query name doesn't already exist, or that we are
// updating the same instance that has this name. If this is a template
// and the name is empty then we make sure there's not an empty template
// already registered.
if query.Name != "" {
wrapped, err := tx.First("prepared-queries", "name", query.Name)
if err != nil {
return fmt.Errorf("failed prepared query lookup: %s", err)
}
other := toPreparedQuery(wrapped)
if other != nil && (existing == nil || existing.ID != other.ID) {
return fmt.Errorf("name '%s' aliases an existing query name", query.Name)
}
} else if prepared_query.IsTemplate(query) {
wrapped, err := tx.First("prepared-queries", "template", query.Name)
if err != nil {
return fmt.Errorf("failed prepared query lookup: %s", err)
}
other := toPreparedQuery(wrapped)
if other != nil && (existing == nil || existing.ID != other.ID) {
return fmt.Errorf("a query template with an empty name already exists")
}
}
2015-11-10 05:15:55 +00:00
// Verify that the name doesn't alias any existing ID. We allow queries
// to be looked up by ID *or* name so we don't want anyone to try to
// register a query with a name equal to some other query's ID in an
// attempt to hijack it. We also look up by ID *then* name in order to
// prevent this, but it seems prudent to prevent these types of rogue
// queries from ever making it into the state store. Note that we have
// to see if the name looks like a UUID before checking since the UUID
// index will complain if we look up something that's not formatted
// like one.
if isUUID(query.Name) {
wrapped, err := tx.First("prepared-queries", "id", query.Name)
if err != nil {
return fmt.Errorf("failed prepared query lookup: %s", err)
}
if wrapped != nil {
return fmt.Errorf("name '%s' aliases an existing query ID", query.Name)
}
}
// Verify that the session exists.
if query.Session != "" {
sess, err := firstWithTxn(tx, "sessions", "id", query.Session, nil)
if err != nil {
return fmt.Errorf("invalid session: %v", err)
}
if sess == nil {
return fmt.Errorf("invalid session %#v", query.Session)
}
}
Creates new "prepared-query" ACL type and new token capture behavior. Prior to this change, prepared queries had the following behavior for ACLs, which will need to change to support templates: 1. A management token, or a token with read access to the service being queried needed to be provided in order to create a prepared query. 2. The token used to create the prepared query was stored with the query in the state store and used to execute the query. 3. A management token, or the token used to create the query needed to be supplied to perform and CRUD operations on an existing prepared query. This was pretty subtle and complicated behavior, and won't work for templates since the service name is computed at execution time. To solve this, we introduce a new "prepared-query" ACL type, where the prefix applies to the query name for static prepared query types and to the prefix for template prepared query types. With this change, the new behavior is: 1. A management token, or a token with "prepared-query" write access to the query name or (soon) the given template prefix is required to do any CRUD operations on a prepared query, or to list prepared queries (the list is filtered by this ACL). 2. You will no longer need a management token to list prepared queries, but you will only be able to see prepared queries that you have access to (you get an empty list instead of permission denied). 3. When listing or getting a query, because it was easy to capture management tokens given the past behavior, this will always blank out the "Token" field (replacing the contents as <hidden>) for all tokens unless a management token is supplied. Going forward, we should discourage people from binding tokens for execution unless strictly necessary. 4. No token will be captured by default when a prepared query is created. If the user wishes to supply an execution token then can pass it in via the "Token" field in the prepared query definition. Otherwise, this field will default to empty. 5. At execution time, we will use the captured token if it exists with the prepared query definition, otherwise we will use the token that's passed in with the request, just like we do for other RPCs (or you can use the agent's configured token for DNS). 6. Prepared queries with no name (accessible only by ID) will not require ACLs to create or modify (execution time will depend on the service ACL configuration). Our argument here is that these are designed to be ephemeral and the IDs are as good as an ACL. Management tokens will be able to list all of these. These changes enable templates, but also enable delegation of authority to manage the prepared query namespace.
2016-02-23 08:12:58 +00:00
// We do not verify the service here, nor the token, if any. These are
// checked at execute time and not doing integrity checking on them
// helps avoid bootstrapping chicken and egg problems.
// If this is a template, compile it, otherwise leave the compiled
// template field nil.
var ct *prepared_query.CompiledTemplate
if prepared_query.IsTemplate(query) {
var err error
ct, err = prepared_query.Compile(query)
if err != nil {
return fmt.Errorf("failed compiling template: %s", err)
}
}
// Insert the wrapped query.
if err := tx.Insert("prepared-queries", &queryWrapper{query, ct}); err != nil {
return fmt.Errorf("failed inserting prepared query: %s", err)
}
if err := tx.Insert("index", &IndexEntry{"prepared-queries", idx}); err != nil {
return fmt.Errorf("failed updating index: %s", err)
}
return nil
}
// PreparedQueryDelete deletes the given query by ID.
2017-04-21 00:46:29 +00:00
func (s *Store) PreparedQueryDelete(idx uint64, queryID string) error {
tx := s.db.Txn(true)
defer tx.Abort()
2017-01-24 19:53:02 +00:00
if err := s.preparedQueryDeleteTxn(tx, idx, queryID); err != nil {
return fmt.Errorf("failed prepared query delete: %s", err)
}
tx.Commit()
return nil
}
// preparedQueryDeleteTxn is the inner method used to delete a prepared query
// with the proper indexes into the state store.
2017-04-21 00:46:29 +00:00
func (s *Store) preparedQueryDeleteTxn(tx *memdb.Txn, idx uint64, queryID string) error {
// Pull the query.
wrapped, err := tx.First("prepared-queries", "id", queryID)
if err != nil {
return fmt.Errorf("failed prepared query lookup: %s", err)
}
if wrapped == nil {
return nil
}
// Delete the query and update the index.
if err := tx.Delete("prepared-queries", wrapped); err != nil {
return fmt.Errorf("failed prepared query delete: %s", err)
}
if err := tx.Insert("index", &IndexEntry{"prepared-queries", idx}); err != nil {
return fmt.Errorf("failed updating index: %s", err)
}
return nil
}
// PreparedQueryGet returns the given prepared query by ID.
2017-04-21 00:46:29 +00:00
func (s *Store) PreparedQueryGet(ws memdb.WatchSet, queryID string) (uint64, *structs.PreparedQuery, error) {
tx := s.db.Txn(false)
defer tx.Abort()
// Get the table index.
idx := maxIndexTxn(tx, "prepared-queries")
// Look up the query by its ID.
watchCh, wrapped, err := tx.FirstWatch("prepared-queries", "id", queryID)
if err != nil {
return 0, nil, fmt.Errorf("failed prepared query lookup: %s", err)
}
ws.Add(watchCh)
return idx, toPreparedQuery(wrapped), nil
}
// PreparedQueryResolve returns the given prepared query by looking up an ID or
// Name. If the query was looked up by name and it's a template, then the
// template will be rendered before it is returned.
func (s *Store) PreparedQueryResolve(queryIDOrName string, source structs.QuerySource) (uint64, *structs.PreparedQuery, error) {
tx := s.db.Txn(false)
defer tx.Abort()
// Get the table index.
2017-01-24 19:53:02 +00:00
idx := maxIndexTxn(tx, "prepared-queries")
// Explicitly ban an empty query. This will never match an ID and the
// schema is set up so it will never match a query with an empty name,
// but we check it here to be explicit about it (we'd never want to
// return the results from the first query w/o a name).
if queryIDOrName == "" {
return 0, nil, ErrMissingQueryID
}
// Try first by ID if it looks like they gave us an ID. We check the
// format before trying this because the UUID index will complain if
// we look up something that's not formatted like one.
if isUUID(queryIDOrName) {
wrapped, err := tx.First("prepared-queries", "id", queryIDOrName)
if err != nil {
return 0, nil, fmt.Errorf("failed prepared query lookup: %s", err)
}
if wrapped != nil {
query := toPreparedQuery(wrapped)
if prepared_query.IsTemplate(query) {
return idx, nil, fmt.Errorf("prepared query templates can only be resolved up by name, not by ID")
}
return idx, query, nil
}
}
// prep will check to see if the query is a template and render it
// first, otherwise it will just return a regular query.
prep := func(wrapped interface{}) (uint64, *structs.PreparedQuery, error) {
wrapper := wrapped.(*queryWrapper)
if prepared_query.IsTemplate(wrapper.PreparedQuery) {
render, err := wrapper.ct.Render(queryIDOrName, source)
if err != nil {
return idx, nil, err
}
return idx, render, nil
}
return idx, wrapper.PreparedQuery, nil
}
// Next, look for an exact name match. This is the common case for static
// prepared queries, and could also apply to templates.
{
wrapped, err := tx.First("prepared-queries", "name", queryIDOrName)
if err != nil {
return 0, nil, fmt.Errorf("failed prepared query lookup: %s", err)
}
if wrapped != nil {
return prep(wrapped)
}
}
// Next, look for the longest prefix match among the prepared query
// templates.
{
wrapped, err := tx.LongestPrefix("prepared-queries", "template_prefix", queryIDOrName)
if err != nil {
return 0, nil, fmt.Errorf("failed prepared query lookup: %s", err)
}
if wrapped != nil {
return prep(wrapped)
}
}
return idx, nil, nil
}
// PreparedQueryList returns all the prepared queries.
2017-04-21 00:46:29 +00:00
func (s *Store) PreparedQueryList(ws memdb.WatchSet) (uint64, structs.PreparedQueries, error) {
tx := s.db.Txn(false)
defer tx.Abort()
// Get the table index.
2017-01-24 19:53:02 +00:00
idx := maxIndexTxn(tx, "prepared-queries")
// Query all of the prepared queries in the state store.
queries, err := tx.Get("prepared-queries", "id")
if err != nil {
return 0, nil, fmt.Errorf("failed prepared query lookup: %s", err)
}
ws.Add(queries.WatchCh())
// Go over all of the queries and build the response.
var result structs.PreparedQueries
for wrapped := queries.Next(); wrapped != nil; wrapped = queries.Next() {
result = append(result, toPreparedQuery(wrapped))
}
return idx, result, nil
}