Merge pull request #336 from ryanuber/f-keyring

feature: gossip encryption key rotation

command/agent/agent.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"net"
 	"os"
+	"path/filepath"
 	"strconv"
 	"sync"
 
@@ -160,11 +161,6 @@ func (a *Agent) consulConfig() *consul.Config {
 	if a.config.DataDir != "" {
 		base.DataDir = a.config.DataDir
 	}
-	if a.config.EncryptKey != "" {
-		key, _ := a.config.EncryptBytes()
-		base.SerfLANConfig.MemberlistConfig.SecretKey = key
-		base.SerfWANConfig.MemberlistConfig.SecretKey = key
-	}
 	if a.config.NodeName != "" {
 		base.NodeName = a.config.NodeName
 	}
@@ -260,7 +256,13 @@ func (a *Agent) consulConfig() *consul.Config {
 
 // setupServer is used to initialize the Consul server
 func (a *Agent) setupServer() error {
-	server, err := consul.NewServer(a.consulConfig())
+	config := a.consulConfig()
+
+	if err := a.setupKeyrings(config); err != nil {
+		return fmt.Errorf("Failed to configure keyring: %v", err)
+	}
+
+	server, err := consul.NewServer(config)
 	if err != nil {
 		return fmt.Errorf("Failed to start Consul server: %v", err)
 	}
@@ -270,7 +272,13 @@ func (a *Agent) setupServer() error {
 
 // setupClient is used to initialize the Consul client
 func (a *Agent) setupClient() error {
-	client, err := consul.NewClient(a.consulConfig())
+	config := a.consulConfig()
+
+	if err := a.setupKeyrings(config); err != nil {
+		return fmt.Errorf("Failed to configure keyring: %v", err)
+	}
+
+	client, err := consul.NewClient(config)
 	if err != nil {
 		return fmt.Errorf("Failed to start Consul client: %v", err)
 	}
@@ -278,6 +286,47 @@ func (a *Agent) setupClient() error {
 	return nil
 }
 
+// setupKeyrings is used to initialize and load keyrings during agent startup
+func (a *Agent) setupKeyrings(config *consul.Config) error {
+	fileLAN := filepath.Join(a.config.DataDir, serfLANKeyring)
+	fileWAN := filepath.Join(a.config.DataDir, serfWANKeyring)
+
+	if a.config.EncryptKey == "" {
+		goto LOAD
+	}
+	if _, err := os.Stat(fileLAN); err != nil {
+		if err := initKeyring(fileLAN, a.config.EncryptKey); err != nil {
+			return err
+		}
+	}
+	if a.config.Server {
+		if _, err := os.Stat(fileWAN); err != nil {
+			if err := initKeyring(fileWAN, a.config.EncryptKey); err != nil {
+				return err
+			}
+		}
+	}
+
+LOAD:
+	if _, err := os.Stat(fileLAN); err == nil {
+		config.SerfLANConfig.KeyringFile = fileLAN
+	}
+	if err := loadKeyringFile(config.SerfLANConfig); err != nil {
+		return err
+	}
+	if a.config.Server {
+		if _, err := os.Stat(fileWAN); err == nil {
+			config.SerfWANConfig.KeyringFile = fileWAN
+		}
+		if err := loadKeyringFile(config.SerfWANConfig); err != nil {
+			return err
+		}
+	}
+
+	// Success!
+	return nil
+}
+
 // RPC is used to make an RPC call to the Consul servers
 // This allows the agent to implement the Consul.Interface
 func (a *Agent) RPC(method string, args interface{}, reply interface{}) error {

command/agent/agent_test.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"sync/atomic"
 	"testing"
 	"time"
@@ -71,6 +72,31 @@ func makeAgentLog(t *testing.T, conf *Config, l io.Writer) (string, *Agent) {
 	return dir, agent
 }
 
+func makeAgentKeyring(t *testing.T, conf *Config, key string) (string, *Agent) {
+	dir, err := ioutil.TempDir("", "agent")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	conf.DataDir = dir
+
+	fileLAN := filepath.Join(dir, serfLANKeyring)
+	if err := initKeyring(fileLAN, key); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	fileWAN := filepath.Join(dir, serfWANKeyring)
+	if err := initKeyring(fileWAN, key); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	agent, err := Create(conf, nil)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	return dir, agent
+}
+
 func makeAgent(t *testing.T, conf *Config) (string, *Agent) {
 	return makeAgentLog(t, conf, nil)
 }

command/agent/command.go

@@ -143,17 +143,27 @@ func (c *Command) readConfig() *Config {
 		config.NodeName = hostname
 	}
 
+	// Ensure we have a data directory
+	if config.DataDir == "" {
+		c.Ui.Error("Must specify data directory using -data-dir")
+		return nil
+	}
+
 	if config.EncryptKey != "" {
 		if _, err := config.EncryptBytes(); err != nil {
 			c.Ui.Error(fmt.Sprintf("Invalid encryption key: %s", err))
 			return nil
 		}
+		keyfileLAN := filepath.Join(config.DataDir, serfLANKeyring)
+		if _, err := os.Stat(keyfileLAN); err == nil {
+			c.Ui.Error("WARNING: LAN keyring exists but -encrypt given, ignoring")
+		}
+		if config.Server {
+			keyfileWAN := filepath.Join(config.DataDir, serfWANKeyring)
+			if _, err := os.Stat(keyfileWAN); err == nil {
+				c.Ui.Error("WARNING: WAN keyring exists but -encrypt given, ignoring")
+			}
 		}
-
-	// Ensure we have a data directory
-	if config.DataDir == "" {
-		c.Ui.Error("Must specify data directory using -data-dir")
-		return nil
 	}
 
 	// Verify data center is valid
@@ -459,6 +469,22 @@ func (c *Command) retryJoinWan(config *Config, errCh chan<- struct{}) {
 	}
 }
 
+// gossipEncrypted determines if the consul instance is using symmetric
+// encryption keys to protect gossip protocol messages.
+func (c *Command) gossipEncrypted() bool {
+	if c.agent.config.EncryptKey != "" {
+		return true
+	}
+
+	server := c.agent.server
+	if server != nil {
+		return server.KeyManagerLAN() != nil || server.KeyManagerWAN() != nil
+	}
+
+	client := c.agent.client
+	return client != nil && client.KeyManagerLAN() != nil
+}
+
 func (c *Command) Run(args []string) int {
 	c.Ui = &cli.PrefixedUi{
 		OutputPrefix: "==> ",
@@ -585,6 +611,14 @@ func (c *Command) Run(args []string) int {
 		}(wp)
 	}
 
+	// Figure out if gossip is encrypted
+	var gossipEncrypted bool
+	if config.Server {
+		gossipEncrypted = c.agent.server.Encrypted()
+	} else {
+		gossipEncrypted = c.agent.client.Encrypted()
+	}
+
 	// Let the agent know we've finished registration
 	c.agent.StartSync()
 
@@ -597,7 +631,7 @@ func (c *Command) Run(args []string) int {
 	c.Ui.Info(fmt.Sprintf("  Cluster Addr: %v (LAN: %d, WAN: %d)", config.AdvertiseAddr,
 		config.Ports.SerfLan, config.Ports.SerfWan))
 	c.Ui.Info(fmt.Sprintf("Gossip encrypt: %v, RPC-TLS: %v, TLS-Incoming: %v",
-		config.EncryptKey != "", config.VerifyOutgoing, config.VerifyIncoming))
+		gossipEncrypted, config.VerifyOutgoing, config.VerifyIncoming))
 
 	// Enable log streaming
 	c.Ui.Info("")

command/agent/keyring.go

@@ -0,0 +1,145 @@
+package agent
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/hashicorp/consul/consul/structs"
+	"github.com/hashicorp/memberlist"
+	"github.com/hashicorp/serf/serf"
+)
+
+const (
+	serfLANKeyring = "serf/local.keyring"
+	serfWANKeyring = "serf/remote.keyring"
+)
+
+// initKeyring will create a keyring file at a given path.
+func initKeyring(path, key string) error {
+	var keys []string
+
+	if _, err := base64.StdEncoding.DecodeString(key); err != nil {
+		return fmt.Errorf("Invalid key: %s", err)
+	}
+
+	// Just exit if the file already exists.
+	if _, err := os.Stat(path); err == nil {
+		return nil
+	}
+
+	keys = append(keys, key)
+	keyringBytes, err := json.Marshal(keys)
+	if err != nil {
+		return err
+	}
+
+	if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
+		return err
+	}
+
+	fh, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	defer fh.Close()
+
+	if _, err := fh.Write(keyringBytes); err != nil {
+		os.Remove(path)
+		return err
+	}
+
+	return nil
+}
+
+// loadKeyringFile will load a gossip encryption keyring out of a file. The file
+// must be in JSON format and contain a list of encryption key strings.
+func loadKeyringFile(c *serf.Config) error {
+	if c.KeyringFile == "" {
+		return nil
+	}
+
+	if _, err := os.Stat(c.KeyringFile); err != nil {
+		return err
+	}
+
+	// Read in the keyring file data
+	keyringData, err := ioutil.ReadFile(c.KeyringFile)
+	if err != nil {
+		return err
+	}
+
+	// Decode keyring JSON
+	keys := make([]string, 0)
+	if err := json.Unmarshal(keyringData, &keys); err != nil {
+		return err
+	}
+
+	// Decode base64 values
+	keysDecoded := make([][]byte, len(keys))
+	for i, key := range keys {
+		keyBytes, err := base64.StdEncoding.DecodeString(key)
+		if err != nil {
+			return err
+		}
+		keysDecoded[i] = keyBytes
+	}
+
+	// Guard against empty keyring
+	if len(keysDecoded) == 0 {
+		return fmt.Errorf("no keys present in keyring file: %s", c.KeyringFile)
+	}
+
+	// Create the keyring
+	keyring, err := memberlist.NewKeyring(keysDecoded, keysDecoded[0])
+	if err != nil {
+		return err
+	}
+
+	c.MemberlistConfig.Keyring = keyring
+
+	// Success!
+	return nil
+}
+
+// keyringProcess is used to abstract away the semantic similarities in
+// performing various operations on the encryption keyring.
+func (a *Agent) keyringProcess(args *structs.KeyringRequest) (*structs.KeyringResponses, error) {
+	var reply structs.KeyringResponses
+	if a.server == nil {
+		return nil, fmt.Errorf("keyring operations must run against a server node")
+	}
+	if err := a.RPC("Internal.KeyringOperation", args, &reply); err != nil {
+		return &reply, err
+	}
+
+	return &reply, nil
+}
+
+// ListKeys lists out all keys installed on the collective Consul cluster. This
+// includes both servers and clients in all DC's.
+func (a *Agent) ListKeys() (*structs.KeyringResponses, error) {
+	args := structs.KeyringRequest{Operation: structs.KeyringList}
+	return a.keyringProcess(&args)
+}
+
+// InstallKey installs a new gossip encryption key
+func (a *Agent) InstallKey(key string) (*structs.KeyringResponses, error) {
+	args := structs.KeyringRequest{Key: key, Operation: structs.KeyringInstall}
+	return a.keyringProcess(&args)
+}
+
+// UseKey changes the primary encryption key used to encrypt messages
+func (a *Agent) UseKey(key string) (*structs.KeyringResponses, error) {
+	args := structs.KeyringRequest{Key: key, Operation: structs.KeyringUse}
+	return a.keyringProcess(&args)
+}
+
+// RemoveKey will remove a gossip encryption key from the keyring
+func (a *Agent) RemoveKey(key string) (*structs.KeyringResponses, error) {
+	args := structs.KeyringRequest{Key: key, Operation: structs.KeyringRemove}
+	return a.keyringProcess(&args)
+}

command/agent/keyring_test.go

@@ -0,0 +1,115 @@
+package agent
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestAgent_LoadKeyrings(t *testing.T) {
+	key := "tbLJg26ZJyJ9pK3qhc9jig=="
+
+	// Should be no configured keyring file by default
+	conf1 := nextConfig()
+	dir1, agent1 := makeAgent(t, conf1)
+	defer os.RemoveAll(dir1)
+	defer agent1.Shutdown()
+
+	c := agent1.config.ConsulConfig
+	if c.SerfLANConfig.KeyringFile != "" {
+		t.Fatalf("bad: %#v", c.SerfLANConfig.KeyringFile)
+	}
+	if c.SerfLANConfig.MemberlistConfig.Keyring != nil {
+		t.Fatalf("keyring should not be loaded")
+	}
+	if c.SerfWANConfig.KeyringFile != "" {
+		t.Fatalf("bad: %#v", c.SerfLANConfig.KeyringFile)
+	}
+	if c.SerfWANConfig.MemberlistConfig.Keyring != nil {
+		t.Fatalf("keyring should not be loaded")
+	}
+
+	// Server should auto-load LAN and WAN keyring files
+	conf2 := nextConfig()
+	dir2, agent2 := makeAgentKeyring(t, conf2, key)
+	defer os.RemoveAll(dir2)
+	defer agent2.Shutdown()
+
+	c = agent2.config.ConsulConfig
+	if c.SerfLANConfig.KeyringFile == "" {
+		t.Fatalf("should have keyring file")
+	}
+	if c.SerfLANConfig.MemberlistConfig.Keyring == nil {
+		t.Fatalf("keyring should be loaded")
+	}
+	if c.SerfWANConfig.KeyringFile == "" {
+		t.Fatalf("should have keyring file")
+	}
+	if c.SerfWANConfig.MemberlistConfig.Keyring == nil {
+		t.Fatalf("keyring should be loaded")
+	}
+
+	// Client should auto-load only the LAN keyring file
+	conf3 := nextConfig()
+	conf3.Server = false
+	dir3, agent3 := makeAgentKeyring(t, conf3, key)
+	defer os.RemoveAll(dir3)
+	defer agent3.Shutdown()
+
+	c = agent3.config.ConsulConfig
+	if c.SerfLANConfig.KeyringFile == "" {
+		t.Fatalf("should have keyring file")
+	}
+	if c.SerfLANConfig.MemberlistConfig.Keyring == nil {
+		t.Fatalf("keyring should be loaded")
+	}
+	if c.SerfWANConfig.KeyringFile != "" {
+		t.Fatalf("bad: %#v", c.SerfWANConfig.KeyringFile)
+	}
+	if c.SerfWANConfig.MemberlistConfig.Keyring != nil {
+		t.Fatalf("keyring should not be loaded")
+	}
+}
+
+func TestAgent_InitKeyring(t *testing.T) {
+	key1 := "tbLJg26ZJyJ9pK3qhc9jig=="
+	key2 := "4leC33rgtXKIVUr9Nr0snQ=="
+	expected := fmt.Sprintf(`["%s"]`, key1)
+
+	dir, err := ioutil.TempDir("", "consul")
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer os.RemoveAll(dir)
+
+	file := filepath.Join(dir, "keyring")
+
+	// First initialize the keyring
+	if err := initKeyring(file, key1); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	content, err := ioutil.ReadFile(file)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if string(content) != expected {
+		t.Fatalf("bad: %s", content)
+	}
+
+	// Try initializing again with a different key
+	if err := initKeyring(file, key2); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Content should still be the same
+	content, err = ioutil.ReadFile(file)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if string(content) != expected {
+		t.Fatalf("bad: %s", content)
+	}
+}

command/agent/rpc.go

@@ -24,15 +24,17 @@ package agent
 import (
 	"bufio"
 	"fmt"
-	"github.com/hashicorp/go-msgpack/codec"
-	"github.com/hashicorp/logutils"
-	"github.com/hashicorp/serf/serf"
 	"io"
 	"log"
 	"net"
 	"os"
 	"strings"
 	"sync"
+
+	"github.com/hashicorp/consul/consul/structs"
+	"github.com/hashicorp/go-msgpack/codec"
+	"github.com/hashicorp/logutils"
+	"github.com/hashicorp/serf/serf"
 )
 
 const (
@@ -51,6 +53,10 @@ const (
 	leaveCommand      = "leave"
 	statsCommand      = "stats"
 	reloadCommand     = "reload"
+	installKeyCommand = "install-key"
+	useKeyCommand     = "use-key"
+	removeKeyCommand  = "remove-key"
+	listKeysCommand   = "list-keys"
 )
 
 const (
@@ -103,6 +109,37 @@ type joinResponse struct {
 	Num int32
 }
 
+type keyringRequest struct {
+	Key string
+}
+
+type KeyringEntry struct {
+	Datacenter string
+	Pool       string
+	Key        string
+	Count      int
+}
+
+type KeyringMessage struct {
+	Datacenter string
+	Pool       string
+	Node       string
+	Message    string
+}
+
+type KeyringInfo struct {
+	Datacenter string
+	Pool       string
+	NumNodes   int
+	Error      string
+}
+
+type keyringResponse struct {
+	Keys     []KeyringEntry
+	Messages []KeyringMessage
+	Info     []KeyringInfo
+}
+
 type membersResponse struct {
 	Members []Member
 }
@@ -373,6 +410,9 @@ func (i *AgentRPC) handleRequest(client *rpcClient, reqHeader *requestHeader) er
 	case reloadCommand:
 		return i.handleReload(client, seq)
 
+	case installKeyCommand, useKeyCommand, removeKeyCommand, listKeysCommand:
+		return i.handleKeyring(client, seq, command)
+
 	default:
 		respHeader := responseHeader{Seq: seq, Error: unsupportedCommand}
 		client.Send(&respHeader, nil)
@@ -583,6 +623,80 @@ func (i *AgentRPC) handleReload(client *rpcClient, seq uint64) error {
 	return client.Send(&resp, nil)
 }
 
+func (i *AgentRPC) handleKeyring(client *rpcClient, seq uint64, cmd string) error {
+	var req keyringRequest
+	var queryResp *structs.KeyringResponses
+	var r keyringResponse
+	var err error
+
+	if cmd != listKeysCommand {
+		if err = client.dec.Decode(&req); err != nil {
+			return fmt.Errorf("decode failed: %v", err)
+		}
+	}
+
+	switch cmd {
+	case listKeysCommand:
+		queryResp, err = i.agent.ListKeys()
+	case installKeyCommand:
+		queryResp, err = i.agent.InstallKey(req.Key)
+	case useKeyCommand:
+		queryResp, err = i.agent.UseKey(req.Key)
+	case removeKeyCommand:
+		queryResp, err = i.agent.RemoveKey(req.Key)
+	default:
+		respHeader := responseHeader{Seq: seq, Error: unsupportedCommand}
+		client.Send(&respHeader, nil)
+		return fmt.Errorf("command '%s' not recognized", cmd)
+	}
+
+	header := responseHeader{
+		Seq:   seq,
+		Error: errToString(err),
+	}
+
+	if queryResp == nil {
+		goto SEND
+	}
+
+	for _, kr := range queryResp.Responses {
+		var pool string
+		if kr.WAN {
+			pool = "WAN"
+		} else {
+			pool = "LAN"
+		}
+		for node, message := range kr.Messages {
+			msg := KeyringMessage{
+				Datacenter: kr.Datacenter,
+				Pool:       pool,
+				Node:       node,
+				Message:    message,
+			}
+			r.Messages = append(r.Messages, msg)
+		}
+		for key, qty := range kr.Keys {
+			k := KeyringEntry{
+				Datacenter: kr.Datacenter,
+				Pool:       pool,
+				Key:        key,
+				Count:      qty,
+			}
+			r.Keys = append(r.Keys, k)
+		}
+		info := KeyringInfo{
+			Datacenter: kr.Datacenter,
+			Pool:       pool,
+			NumNodes:   kr.NumNodes,
+			Error:      kr.Error,
+		}
+		r.Info = append(r.Info, info)
+	}
+
+SEND:
+	return client.Send(&header, r)
+}
+
 // Used to convert an error to a string representation
 func errToString(err error) string {
 	if err == nil {

command/agent/rpc_client.go

@@ -176,6 +176,49 @@ func (c *RPCClient) WANMembers() ([]Member, error) {
 	return resp.Members, err
 }
 
+func (c *RPCClient) ListKeys() (keyringResponse, error) {
+	header := requestHeader{
+		Command: listKeysCommand,
+		Seq:     c.getSeq(),
+	}
+	var resp keyringResponse
+	err := c.genericRPC(&header, nil, &resp)
+	return resp, err
+}
+
+func (c *RPCClient) InstallKey(key string) (keyringResponse, error) {
+	header := requestHeader{
+		Command: installKeyCommand,
+		Seq:     c.getSeq(),
+	}
+	req := keyringRequest{key}
+	var resp keyringResponse
+	err := c.genericRPC(&header, &req, &resp)
+	return resp, err
+}
+
+func (c *RPCClient) UseKey(key string) (keyringResponse, error) {
+	header := requestHeader{
+		Command: useKeyCommand,
+		Seq:     c.getSeq(),
+	}
+	req := keyringRequest{key}
+	var resp keyringResponse
+	err := c.genericRPC(&header, &req, &resp)
+	return resp, err
+}
+
+func (c *RPCClient) RemoveKey(key string) (keyringResponse, error) {
+	header := requestHeader{
+		Command: removeKeyCommand,
+		Seq:     c.getSeq(),
+	}
+	req := keyringRequest{key}
+	var resp keyringResponse
+	err := c.genericRPC(&header, &req, &resp)
+	return resp, err
+}
+
 // Leave is used to trigger a graceful leave and shutdown
 func (c *RPCClient) Leave() error {
 	header := requestHeader{

command/agent/rpc_client_test.go

@@ -30,6 +30,10 @@ func (r *rpcParts) Close() {
 // testRPCClient returns an RPCClient connected to an RPC server that
 // serves only this connection.
 func testRPCClient(t *testing.T) *rpcParts {
+	return testRPCClientWithConfig(t, func(c *Config) {})
+}
+
+func testRPCClientWithConfig(t *testing.T, cb func(c *Config)) *rpcParts {
 	l, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
 		t.Fatalf("err: %s", err)
@@ -39,6 +43,8 @@ func testRPCClient(t *testing.T) *rpcParts {
 	mult := io.MultiWriter(os.Stderr, lw)
 
 	conf := nextConfig()
+	cb(conf)
+
 	dir, agent := makeAgentLog(t, conf, mult)
 	rpc := NewAgentRPC(agent, l, mult, lw)
 
@@ -273,3 +279,159 @@ OUTER2:
 		t.Fatalf("should log joining")
 	}
 }
+
+func TestRPCClientListKeys(t *testing.T) {
+	key1 := "tbLJg26ZJyJ9pK3qhc9jig=="
+	p1 := testRPCClientWithConfig(t, func(c *Config) {
+		c.EncryptKey = key1
+		c.Datacenter = "dc1"
+	})
+	defer p1.Close()
+
+	// Key is initially installed to both wan/lan
+	keys := listKeys(t, p1.client)
+	if _, ok := keys["dc1"][key1]; !ok {
+		t.Fatalf("bad: %#v", keys)
+	}
+	if _, ok := keys["WAN"][key1]; !ok {
+		t.Fatalf("bad: %#v", keys)
+	}
+}
+
+func TestRPCClientInstallKey(t *testing.T) {
+	key1 := "tbLJg26ZJyJ9pK3qhc9jig=="
+	key2 := "xAEZ3uVHRMZD9GcYMZaRQw=="
+	p1 := testRPCClientWithConfig(t, func(c *Config) {
+		c.EncryptKey = key1
+	})
+	defer p1.Close()
+
+	// key2 is not installed yet
+	testutil.WaitForResult(func() (bool, error) {
+		keys := listKeys(t, p1.client)
+		if num, ok := keys["dc1"][key2]; ok || num != 0 {
+			return false, fmt.Errorf("bad: %#v", keys)
+		}
+		if num, ok := keys["WAN"][key2]; ok || num != 0 {
+			return false, fmt.Errorf("bad: %#v", keys)
+		}
+		return true, nil
+	}, func(err error) {
+		t.Fatal(err.Error())
+	})
+
+	// install key2
+	r, err := p1.client.InstallKey(key2)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	keyringSuccess(t, r)
+
+	// key2 should now be installed
+	testutil.WaitForResult(func() (bool, error) {
+		keys := listKeys(t, p1.client)
+		if num, ok := keys["dc1"][key2]; !ok || num != 1 {
+			return false, fmt.Errorf("bad: %#v", keys)
+		}
+		if num, ok := keys["WAN"][key2]; !ok || num != 1 {
+			return false, fmt.Errorf("bad: %#v", keys)
+		}
+		return true, nil
+	}, func(err error) {
+		t.Fatal(err.Error())
+	})
+}
+
+func TestRPCClientUseKey(t *testing.T) {
+	key1 := "tbLJg26ZJyJ9pK3qhc9jig=="
+	key2 := "xAEZ3uVHRMZD9GcYMZaRQw=="
+	p1 := testRPCClientWithConfig(t, func(c *Config) {
+		c.EncryptKey = key1
+	})
+	defer p1.Close()
+
+	// add a second key to the ring
+	r, err := p1.client.InstallKey(key2)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	keyringSuccess(t, r)
+
+	// key2 is installed
+	testutil.WaitForResult(func() (bool, error) {
+		keys := listKeys(t, p1.client)
+		if num, ok := keys["dc1"][key2]; !ok || num != 1 {
+			return false, fmt.Errorf("bad: %#v", keys)
+		}
+		if num, ok := keys["WAN"][key2]; !ok || num != 1 {
+			return false, fmt.Errorf("bad: %#v", keys)
+		}
+		return true, nil
+	}, func(err error) {
+		t.Fatal(err.Error())
+	})
+
+	// can't remove key1 yet
+	r, err = p1.client.RemoveKey(key1)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	keyringError(t, r)
+
+	// change primary key
+	r, err = p1.client.UseKey(key2)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	keyringSuccess(t, r)
+
+	// can remove key1 now
+	r, err = p1.client.RemoveKey(key1)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	keyringSuccess(t, r)
+}
+
+func TestRPCClientKeyOperation_encryptionDisabled(t *testing.T) {
+	p1 := testRPCClient(t)
+	defer p1.Close()
+
+	r, err := p1.client.ListKeys()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	keyringError(t, r)
+}
+
+func listKeys(t *testing.T, c *RPCClient) map[string]map[string]int {
+	resp, err := c.ListKeys()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	out := make(map[string]map[string]int)
+	for _, k := range resp.Keys {
+		respID := k.Datacenter
+		if k.Pool == "WAN" {
+			respID = k.Pool
+		}
+		out[respID] = map[string]int{k.Key: k.Count}
+	}
+	return out
+}
+
+func keyringError(t *testing.T, r keyringResponse) {
+	for _, i := range r.Info {
+		if i.Error == "" {
+			t.Fatalf("no error reported from %s (%s)", i.Datacenter, i.Pool)
+		}
+	}
+}
+
+func keyringSuccess(t *testing.T, r keyringResponse) {
+	for _, i := range r.Info {
+		if i.Error != "" {
+			t.Fatalf("error from %s (%s): %s", i.Datacenter, i.Pool, i.Error)
+		}
+	}
+}

command/keyring.go

@@ -0,0 +1,216 @@
+package command
+
+import (
+	"flag"
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/consul/command/agent"
+	"github.com/mitchellh/cli"
+)
+
+// KeyringCommand is a Command implementation that handles querying, installing,
+// and removing gossip encryption keys from a keyring.
+type KeyringCommand struct {
+	Ui cli.Ui
+}
+
+func (c *KeyringCommand) Run(args []string) int {
+	var installKey, useKey, removeKey string
+	var listKeys bool
+
+	cmdFlags := flag.NewFlagSet("keys", flag.ContinueOnError)
+	cmdFlags.Usage = func() { c.Ui.Output(c.Help()) }
+
+	cmdFlags.StringVar(&installKey, "install", "", "install key")
+	cmdFlags.StringVar(&useKey, "use", "", "use key")
+	cmdFlags.StringVar(&removeKey, "remove", "", "remove key")
+	cmdFlags.BoolVar(&listKeys, "list", false, "list keys")
+
+	rpcAddr := RPCAddrFlag(cmdFlags)
+	if err := cmdFlags.Parse(args); err != nil {
+		return 1
+	}
+
+	c.Ui = &cli.PrefixedUi{
+		OutputPrefix: "",
+		InfoPrefix:   "==> ",
+		ErrorPrefix:  "",
+		Ui:           c.Ui,
+	}
+
+	// Only accept a single argument
+	found := listKeys
+	for _, arg := range []string{installKey, useKey, removeKey} {
+		if found && len(arg) > 0 {
+			c.Ui.Error("Only a single action is allowed")
+			return 1
+		}
+		found = found || len(arg) > 0
+	}
+
+	// Fail fast if no actionable args were passed
+	if !found {
+		c.Ui.Error(c.Help())
+		return 1
+	}
+
+	// All other operations will require a client connection
+	client, err := RPCClient(*rpcAddr)
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err))
+		return 1
+	}
+	defer client.Close()
+
+	if listKeys {
+		c.Ui.Info("Gathering installed encryption keys...")
+		r, err := client.ListKeys()
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("error: %s", err))
+			return 1
+		}
+		if rval := c.handleResponse(r.Info, r.Messages); rval != 0 {
+			return rval
+		}
+		c.handleList(r.Info, r.Keys)
+		return 0
+	}
+
+	if installKey != "" {
+		c.Ui.Info("Installing new gossip encryption key...")
+		r, err := client.InstallKey(installKey)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("error: %s", err))
+			return 1
+		}
+		return c.handleResponse(r.Info, r.Messages)
+	}
+
+	if useKey != "" {
+		c.Ui.Info("Changing primary gossip encryption key...")
+		r, err := client.UseKey(useKey)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("error: %s", err))
+			return 1
+		}
+		return c.handleResponse(r.Info, r.Messages)
+	}
+
+	if removeKey != "" {
+		c.Ui.Info("Removing gossip encryption key...")
+		r, err := client.RemoveKey(removeKey)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("error: %s", err))
+			return 1
+		}
+		return c.handleResponse(r.Info, r.Messages)
+	}
+
+	// Should never make it here
+	return 0
+}
+
+func (c *KeyringCommand) handleResponse(
+	info []agent.KeyringInfo,
+	messages []agent.KeyringMessage) int {
+
+	var rval int
+
+	for _, i := range info {
+		if i.Error != "" {
+			pool := i.Pool
+			if pool != "WAN" {
+				pool = i.Datacenter + " (LAN)"
+			}
+
+			c.Ui.Error("")
+			c.Ui.Error(fmt.Sprintf("%s error: %s", pool, i.Error))
+
+			for _, msg := range messages {
+				if msg.Datacenter != i.Datacenter || msg.Pool != i.Pool {
+					continue
+				}
+				c.Ui.Error(fmt.Sprintf("  %s: %s", msg.Node, msg.Message))
+			}
+			rval = 1
+		}
+	}
+
+	if rval == 0 {
+		c.Ui.Info("Done!")
+	}
+
+	return rval
+}
+
+func (c *KeyringCommand) handleList(
+	info []agent.KeyringInfo,
+	keys []agent.KeyringEntry) {
+
+	installed := make(map[string]map[string][]int)
+	for _, key := range keys {
+		var nodes int
+		for _, i := range info {
+			if i.Datacenter == key.Datacenter && i.Pool == key.Pool {
+				nodes = i.NumNodes
+			}
+		}
+
+		pool := key.Pool
+		if pool != "WAN" {
+			pool = key.Datacenter + " (LAN)"
+		}
+
+		if _, ok := installed[pool]; !ok {
+			installed[pool] = map[string][]int{key.Key: []int{key.Count, nodes}}
+		} else {
+			installed[pool][key.Key] = []int{key.Count, nodes}
+		}
+	}
+
+	for pool, keys := range installed {
+		c.Ui.Output("")
+		c.Ui.Output(pool + ":")
+		for key, num := range keys {
+			c.Ui.Output(fmt.Sprintf("  %s [%d/%d]", key, num[0], num[1]))
+		}
+	}
+}
+
+func (c *KeyringCommand) Help() string {
+	helpText := `
+Usage: consul keyring [options]
+
+  Manages encryption keys used for gossip messages. Gossip encryption is
+  optional. When enabled, this command may be used to examine active encryption
+  keys in the cluster, add new keys, and remove old ones. When combined, this
+  functionality provides the ability to perform key rotation cluster-wide,
+  without disrupting the cluster.
+
+  All operations performed by this command can only be run against server nodes,
+  and affect both the LAN and WAN keyrings in lock-step.
+
+  All variations of the keyring command return 0 if all nodes reply and there
+  are no errors. If any node fails to reply or reports failure, the exit code
+  will be 1.
+
+Options:
+
+  -install=<key>            Install a new encryption key. This will broadcast
+                            the new key to all members in the cluster.
+  -use=<key>                Change the primary encryption key, which is used to
+                            encrypt messages. The key must already be installed
+                            before this operation can succeed.
+  -remove=<key>             Remove the given key from the cluster. This
+                            operation may only be performed on keys which are
+                            not currently the primary key.
+  -list                     List all keys currently in use within the cluster.
+  -rpc-addr=127.0.0.1:8400  RPC address of the Consul agent.
+`
+	return strings.TrimSpace(helpText)
+}
+
+func (c *KeyringCommand) Synopsis() string {
+	return "Manages gossip layer encryption keys"
+}

command/keyring_test.go

@@ -0,0 +1,136 @@
+package command
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/hashicorp/consul/command/agent"
+	"github.com/mitchellh/cli"
+)
+
+func TestKeyringCommand_implements(t *testing.T) {
+	var _ cli.Command = &KeyringCommand{}
+}
+
+func TestKeyringCommandRun(t *testing.T) {
+	key1 := "HS5lJ+XuTlYKWaeGYyG+/A=="
+	key2 := "kZyFABeAmc64UMTrm9XuKA=="
+
+	// Begin with a single key
+	a1 := testAgentWithConfig(t, func(c *agent.Config) {
+		c.EncryptKey = key1
+	})
+	defer a1.Shutdown()
+
+	// The LAN and WAN keyrings were initialized with key1
+	out := listKeys(t, a1.addr)
+	if !strings.Contains(out, "dc1 (LAN):\n  "+key1) {
+		t.Fatalf("bad: %#v", out)
+	}
+	if !strings.Contains(out, "WAN:\n  "+key1) {
+		t.Fatalf("bad: %#v", out)
+	}
+	if strings.Contains(out, key2) {
+		t.Fatalf("bad: %#v", out)
+	}
+
+	// Install the second key onto the keyring
+	installKey(t, a1.addr, key2)
+
+	// Both keys should be present
+	out = listKeys(t, a1.addr)
+	for _, key := range []string{key1, key2} {
+		if !strings.Contains(out, key) {
+			t.Fatalf("bad: %#v", out)
+		}
+	}
+
+	// Rotate to key2, remove key1
+	useKey(t, a1.addr, key2)
+	removeKey(t, a1.addr, key1)
+
+	// Only key2 is present now
+	out = listKeys(t, a1.addr)
+	if !strings.Contains(out, "dc1 (LAN):\n  "+key2) {
+		t.Fatalf("bad: %#v", out)
+	}
+	if !strings.Contains(out, "WAN:\n  "+key2) {
+		t.Fatalf("bad: %#v", out)
+	}
+	if strings.Contains(out, key1) {
+		t.Fatalf("bad: %#v", out)
+	}
+}
+
+func TestKeyringCommandRun_help(t *testing.T) {
+	ui := new(cli.MockUi)
+	c := &KeyringCommand{Ui: ui}
+	code := c.Run(nil)
+	if code != 1 {
+		t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
+	}
+
+	// Test that we didn't actually try to dial the RPC server.
+	if !strings.Contains(ui.ErrorWriter.String(), "Usage:") {
+		t.Fatalf("bad: %#v", ui.ErrorWriter.String())
+	}
+}
+
+func TestKeyringCommandRun_failedConnection(t *testing.T) {
+	ui := new(cli.MockUi)
+	c := &KeyringCommand{Ui: ui}
+	args := []string{"-list", "-rpc-addr=127.0.0.1:0"}
+	code := c.Run(args)
+	if code != 1 {
+		t.Fatalf("bad: %d, %#v", code, ui.ErrorWriter.String())
+	}
+	if !strings.Contains(ui.ErrorWriter.String(), "dial") {
+		t.Fatalf("bad: %#v", ui.OutputWriter.String())
+	}
+}
+
+func listKeys(t *testing.T, addr string) string {
+	ui := new(cli.MockUi)
+	c := &KeyringCommand{Ui: ui}
+
+	args := []string{"-list", "-rpc-addr=" + addr}
+	code := c.Run(args)
+	if code != 0 {
+		t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
+	}
+
+	return ui.OutputWriter.String()
+}
+
+func installKey(t *testing.T, addr string, key string) {
+	ui := new(cli.MockUi)
+	c := &KeyringCommand{Ui: ui}
+
+	args := []string{"-install=" + key, "-rpc-addr=" + addr}
+	code := c.Run(args)
+	if code != 0 {
+		t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
+	}
+}
+
+func useKey(t *testing.T, addr string, key string) {
+	ui := new(cli.MockUi)
+	c := &KeyringCommand{Ui: ui}
+
+	args := []string{"-use=" + key, "-rpc-addr=" + addr}
+	code := c.Run(args)
+	if code != 0 {
+		t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
+	}
+}
+
+func removeKey(t *testing.T, addr string, key string) {
+	ui := new(cli.MockUi)
+	c := &KeyringCommand{Ui: ui}
+
+	args := []string{"-remove=" + key, "-rpc-addr=" + addr}
+	code := c.Run(args)
+	if code != 0 {
+		t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
+	}
+}

command/util_test.go

@@ -39,6 +39,10 @@ func (a *agentWrapper) Shutdown() {
 }
 
 func testAgent(t *testing.T) *agentWrapper {
+	return testAgentWithConfig(t, func(c *agent.Config) {})
+}
+
+func testAgentWithConfig(t *testing.T, cb func(c *agent.Config)) *agentWrapper {
 	l, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
 		t.Fatalf("err: %s", err)
@@ -48,6 +52,7 @@ func testAgent(t *testing.T) *agentWrapper {
 	mult := io.MultiWriter(os.Stderr, lw)
 
 	conf := nextConfig()
+	cb(conf)
 
 	dir, err := ioutil.TempDir("", "agent")
 	if err != nil {

commands.go

@@ -56,6 +56,12 @@ func init() {
 			}, nil
 		},
 
+		"keyring": func() (cli.Command, error) {
+			return &command.KeyringCommand{
+				Ui: ui,
+			}, nil
+		},
+
 		"leave": func() (cli.Command, error) {
 			return &command.LeaveCommand{
 				Ui: ui,

consul/client.go

@@ -206,6 +206,16 @@ func (c *Client) UserEvent(name string, payload []byte) error {
 	return c.serf.UserEvent(userEventName(name), payload, false)
 }
 
+// KeyManagerLAN returns the LAN Serf keyring manager
+func (c *Client) KeyManagerLAN() *serf.KeyManager {
+	return c.serf.KeyManager()
+}
+
+// Encrypted determines if gossip is encrypted
+func (c *Client) Encrypted() bool {
+	return c.serf.EncryptionEnabled()
+}
+
 // lanEventHandler is used to handle events from the lan Serf cluster
 func (c *Client) lanEventHandler() {
 	for {

consul/client_test.go

@@ -269,3 +269,23 @@ func TestClientServer_UserEvent(t *testing.T) {
 		t.Fatalf("missing events")
 	}
 }
+
+func TestClient_Encrypted(t *testing.T) {
+	dir1, c1 := testClient(t)
+	defer os.RemoveAll(dir1)
+	defer c1.Shutdown()
+
+	key := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}
+	dir2, c2 := testClientWithConfig(t, func(c *Config) {
+		c.SerfLANConfig.MemberlistConfig.SecretKey = key
+	})
+	defer os.RemoveAll(dir2)
+	defer c2.Shutdown()
+
+	if c1.Encrypted() {
+		t.Fatalf("should not be encrypted")
+	}
+	if !c2.Encrypted() {
+		t.Fatalf("should be encrypted")
+	}
+}

consul/internal_endpoint.go

@@ -2,6 +2,7 @@ package consul
 
 import (
 	"github.com/hashicorp/consul/consul/structs"
+	"github.com/hashicorp/serf/serf"
 )
 
 // Internal endpoint is used to query the miscellaneous info that
@@ -62,3 +63,64 @@ func (m *Internal) EventFire(args *structs.EventFireRequest,
 	// Fire the event
 	return m.srv.UserEvent(args.Name, args.Payload)
 }
+
+// KeyringOperation will query the WAN and LAN gossip keyrings of all nodes.
+func (m *Internal) KeyringOperation(
+	args *structs.KeyringRequest,
+	reply *structs.KeyringResponses) error {
+
+	// Only perform WAN keyring querying and RPC forwarding once
+	if !args.Forwarded {
+		args.Forwarded = true
+		m.executeKeyringOp(args, reply, true)
+		return m.srv.globalRPC("Internal.KeyringOperation", args, reply)
+	}
+
+	// Query the LAN keyring of this node's DC
+	m.executeKeyringOp(args, reply, false)
+	return nil
+}
+
+// executeKeyringOp executes the appropriate keyring-related function based on
+// the type of keyring operation in the request. It takes the KeyManager as an
+// argument, so it can handle any operation for either LAN or WAN pools.
+func (m *Internal) executeKeyringOp(
+	args *structs.KeyringRequest,
+	reply *structs.KeyringResponses,
+	wan bool) {
+
+	var serfResp *serf.KeyResponse
+	var err error
+	var mgr *serf.KeyManager
+
+	if wan {
+		mgr = m.srv.KeyManagerWAN()
+	} else {
+		mgr = m.srv.KeyManagerLAN()
+	}
+
+	switch args.Operation {
+	case structs.KeyringList:
+		serfResp, err = mgr.ListKeys()
+	case structs.KeyringInstall:
+		serfResp, err = mgr.InstallKey(args.Key)
+	case structs.KeyringUse:
+		serfResp, err = mgr.UseKey(args.Key)
+	case structs.KeyringRemove:
+		serfResp, err = mgr.RemoveKey(args.Key)
+	}
+
+	errStr := ""
+	if err != nil {
+		errStr = err.Error()
+	}
+
+	reply.Responses = append(reply.Responses, &structs.KeyringResponse{
+		WAN:        wan,
+		Datacenter: m.srv.config.Datacenter,
+		Messages:   serfResp.Messages,
+		Keys:       serfResp.Keys,
+		NumNodes:   serfResp.NumNodes,
+		Error:      errStr,
+	})
+}

consul/internal_endpoint_test.go

@@ -1,6 +1,8 @@
 package consul
 
 import (
+	"encoding/base64"
+	"fmt"
 	"github.com/hashicorp/consul/consul/structs"
 	"github.com/hashicorp/consul/testutil"
 	"os"
@@ -150,3 +152,89 @@ func TestInternal_NodeDump(t *testing.T) {
 		t.Fatalf("missing foo or bar")
 	}
 }
+
+func TestInternal_KeyringOperation(t *testing.T) {
+	key1 := "H1dfkSZOVnP/JUnaBfTzXg=="
+	keyBytes1, err := base64.StdEncoding.DecodeString(key1)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	dir1, s1 := testServerWithConfig(t, func(c *Config) {
+		c.SerfLANConfig.MemberlistConfig.SecretKey = keyBytes1
+		c.SerfWANConfig.MemberlistConfig.SecretKey = keyBytes1
+	})
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+	client := rpcClient(t, s1)
+	defer client.Close()
+
+	testutil.WaitForLeader(t, client.Call, "dc1")
+
+	var out structs.KeyringResponses
+	req := structs.KeyringRequest{
+		Operation:  structs.KeyringList,
+		Datacenter: "dc1",
+	}
+	if err := client.Call("Internal.KeyringOperation", &req, &out); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Two responses (local lan/wan pools) from single-node cluster
+	if len(out.Responses) != 2 {
+		t.Fatalf("bad: %#v", out)
+	}
+	if _, ok := out.Responses[0].Keys[key1]; !ok {
+		t.Fatalf("bad: %#v", out)
+	}
+	wanResp, lanResp := 0, 0
+	for _, resp := range out.Responses {
+		if resp.WAN {
+			wanResp++
+		} else {
+			lanResp++
+		}
+	}
+	if lanResp != 1 || wanResp != 1 {
+		t.Fatalf("should have one lan and one wan response")
+	}
+
+	// Start a second agent to test cross-dc queries
+	dir2, s2 := testServerWithConfig(t, func(c *Config) {
+		c.SerfLANConfig.MemberlistConfig.SecretKey = keyBytes1
+		c.SerfWANConfig.MemberlistConfig.SecretKey = keyBytes1
+		c.Datacenter = "dc2"
+	})
+	defer os.RemoveAll(dir2)
+	defer s2.Shutdown()
+
+	// Try to join
+	addr := fmt.Sprintf("127.0.0.1:%d",
+		s1.config.SerfWANConfig.MemberlistConfig.BindPort)
+	if _, err := s2.JoinWAN([]string{addr}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	var out2 structs.KeyringResponses
+	req2 := structs.KeyringRequest{
+		Operation: structs.KeyringList,
+	}
+	if err := client.Call("Internal.KeyringOperation", &req2, &out2); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// 3 responses (one from each DC LAN, one from WAN) in two-node cluster
+	if len(out2.Responses) != 3 {
+		t.Fatalf("bad: %#v", out)
+	}
+	wanResp, lanResp = 0, 0
+	for _, resp := range out2.Responses {
+		if resp.WAN {
+			wanResp++
+		} else {
+			lanResp++
+		}
+	}
+	if lanResp != 2 || wanResp != 1 {
+		t.Fatalf("should have two lan and one wan response")
+	}
+}

consul/rpc.go

@@ -223,6 +223,40 @@ func (s *Server) forwardDC(method, dc string, args interface{}, reply interface{
 	return s.connPool.RPC(server.Addr, server.Version, method, args, reply)
 }
 
+// globalRPC is used to forward an RPC request to one server in each datacenter.
+// This will only error for RPC-related errors. Otherwise, application-level
+// errors can be sent in the response objects.
+func (s *Server) globalRPC(method string, args interface{},
+	reply structs.CompoundResponse) error {
+
+	errorCh := make(chan error)
+	respCh := make(chan interface{})
+
+	// Make a new request into each datacenter
+	for dc, _ := range s.remoteConsuls {
+		go func(dc string) {
+			rr := reply.New()
+			if err := s.forwardDC(method, dc, args, &rr); err != nil {
+				errorCh <- err
+				return
+			}
+			respCh <- rr
+		}(dc)
+	}
+
+	replies, total := 0, len(s.remoteConsuls)
+	for replies < total {
+		select {
+		case err := <-errorCh:
+			return err
+		case rr := <-respCh:
+			reply.Add(rr)
+			replies++
+		}
+	}
+	return nil
+}
+
 // raftApply is used to encode a message, run it through raft, and return
 // the FSM response along with any errors
 func (s *Server) raftApply(t structs.MessageType, msg interface{}) (interface{}, error) {

consul/server.go

@@ -551,6 +551,21 @@ func (s *Server) IsLeader() bool {
 	return s.raft.State() == raft.Leader
 }
 
+// KeyManagerLAN returns the LAN Serf keyring manager
+func (s *Server) KeyManagerLAN() *serf.KeyManager {
+	return s.serfLAN.KeyManager()
+}
+
+// KeyManagerWAN returns the WAN Serf keyring manager
+func (s *Server) KeyManagerWAN() *serf.KeyManager {
+	return s.serfWAN.KeyManager()
+}
+
+// Encrypted determines if gossip is encrypted
+func (s *Server) Encrypted() bool {
+	return s.serfLAN.EncryptionEnabled() && s.serfWAN.EncryptionEnabled()
+}
+
 // inmemCodec is used to do an RPC call without going over a network
 type inmemCodec struct {
 	method string

consul/server_test.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
+	"strings"
 	"testing"
 	"time"
 
@@ -471,5 +472,50 @@ func TestServer_BadExpect(t *testing.T) {
 	}, func(err error) {
 		t.Fatalf("should have 0 peers: %v", err)
 	})
-
+}
+
+type fakeGlobalResp struct{}
+
+func (r *fakeGlobalResp) Add(interface{}) {
+	return
+}
+
+func (r *fakeGlobalResp) New() interface{} {
+	return struct{}{}
+}
+
+func TestServer_globalRPCErrors(t *testing.T) {
+	dir1, s1 := testServerDC(t, "dc1")
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+
+	// Check that an error from a remote DC is returned
+	err := s1.globalRPC("Bad.Method", nil, &fakeGlobalResp{})
+	if err == nil {
+		t.Fatalf("should have errored")
+	}
+	if !strings.Contains(err.Error(), "Bad.Method") {
+		t.Fatalf("unexpcted error: %s", err)
+	}
+}
+
+func TestServer_Encrypted(t *testing.T) {
+	dir1, s1 := testServer(t)
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+
+	key := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}
+	dir2, s2 := testServerWithConfig(t, func(c *Config) {
+		c.SerfLANConfig.MemberlistConfig.SecretKey = key
+		c.SerfWANConfig.MemberlistConfig.SecretKey = key
+	})
+	defer os.RemoveAll(dir2)
+	defer s2.Shutdown()
+
+	if s1.Encrypted() {
+		t.Fatalf("should not be encrypted")
+	}
+	if !s2.Encrypted() {
+		t.Fatalf("should be encrypted")
+	}
 }

consul/structs/structs.go

@@ -531,3 +531,66 @@ func Encode(t MessageType, msg interface{}) ([]byte, error) {
 	err := codec.NewEncoder(&buf, msgpackHandle).Encode(msg)
 	return buf.Bytes(), err
 }
+
+// CompoundResponse is an interface for gathering multiple responses. It is
+// used in cross-datacenter RPC calls where more than 1 datacenter is
+// expected to reply.
+type CompoundResponse interface {
+	// Add adds a new response to the compound response
+	Add(interface{})
+
+	// New returns an empty response object which can be passed around by
+	// reference, and then passed to Add() later on.
+	New() interface{}
+}
+
+type KeyringOp string
+
+const (
+	KeyringList    KeyringOp = "list"
+	KeyringInstall           = "install"
+	KeyringUse               = "use"
+	KeyringRemove            = "remove"
+)
+
+// KeyringRequest encapsulates a request to modify an encryption keyring.
+// It can be used for install, remove, or use key type operations.
+type KeyringRequest struct {
+	Operation  KeyringOp
+	Key        string
+	Datacenter string
+	Forwarded  bool
+	QueryOptions
+}
+
+func (r *KeyringRequest) RequestDatacenter() string {
+	return r.Datacenter
+}
+
+// KeyringResponse is a unified key response and can be used for install,
+// remove, use, as well as listing key queries.
+type KeyringResponse struct {
+	WAN        bool
+	Datacenter string
+	Messages   map[string]string
+	Keys       map[string]int
+	NumNodes   int
+	Error      string
+}
+
+// KeyringResponses holds multiple responses to keyring queries. Each
+// datacenter replies independently, and KeyringResponses is used as a
+// container for the set of all responses.
+type KeyringResponses struct {
+	Responses []*KeyringResponse
+	QueryMeta
+}
+
+func (r *KeyringResponses) Add(v interface{}) {
+	val := v.(*KeyringResponses)
+	r.Responses = append(r.Responses, val.Responses...)
+}
+
+func (r *KeyringResponses) New() interface{} {
+	return new(KeyringResponses)
+}

consul/structs/structs_test.go

@@ -32,3 +32,23 @@ func TestEncodeDecode(t *testing.T) {
 		t.Fatalf("bad: %#v %#v", arg, out)
 	}
 }
+
+func TestStructs_Implements(t *testing.T) {
+	var (
+		_ RPCInfo          = &RegisterRequest{}
+		_ RPCInfo          = &DeregisterRequest{}
+		_ RPCInfo          = &DCSpecificRequest{}
+		_ RPCInfo          = &ServiceSpecificRequest{}
+		_ RPCInfo          = &NodeSpecificRequest{}
+		_ RPCInfo          = &ChecksInStateRequest{}
+		_ RPCInfo          = &KVSRequest{}
+		_ RPCInfo          = &KeyRequest{}
+		_ RPCInfo          = &KeyListRequest{}
+		_ RPCInfo          = &SessionRequest{}
+		_ RPCInfo          = &SessionSpecificRequest{}
+		_ RPCInfo          = &EventFireRequest{}
+		_ RPCInfo          = &ACLPolicyRequest{}
+		_ RPCInfo          = &KeyringRequest{}
+		_ CompoundResponse = &KeyringResponses{}
+	)
+}

website/source/docs/agent/options.html.markdown

@@ -89,6 +89,12 @@ The options below are all specified on the command-line.
   network traffic. This key must be 16-bytes that are base64 encoded. The
   easiest way to create an encryption key is to use `consul keygen`. All
   nodes within a cluster must share the same encryption key to communicate.
+  The provided key is automatically persisted to the data directory, and loaded
+  automatically whenever the agent is restarted. This means that to encrypt
+  Consul's gossip protocol, this option only needs to be provided once on each
+  agent's initial startup sequence. If it is provided after Consul has been
+  initialized with an encryption key, then the provided key is ignored and
+  a warning will be displayed.
 
 * `-join` - Address of another agent to join upon starting up. This can be
   specified multiple times to specify multiple agents to join. If Consul is

website/source/docs/commands/keyring.html.markdown

@@ -0,0 +1,101 @@
+---
+layout: "docs"
+page_title: "Commands: Keyring"
+sidebar_current: "docs-commands-keyring"
+---
+
+# Consul Keyring
+
+Command: `consul keyring`
+
+The `keyring` command is used to examine and modify the encryption keys used in
+Consul's [Gossip Pools](/docs/internals/gossip.html). It is capable of
+distributing new encryption keys to the cluster, retiring old encryption keys,
+and changing the keys used by the cluster to encrypt messages.
+
+Consul allows multiple encryption keys to be in use simultaneously. This is
+intended to provide a transition state while the cluster converges. It is the
+responsibility of the operator to ensure that only the required encryption keys
+are installed on the cluster. You can review the installed keys using the
+`-list` argument, and remove unneeded keys with `-remove`.
+
+All operations performed by this command can only be run against server nodes,
+and affect both the LAN and WAN keyrings in lock-step.
+
+All variations of the `keyring` command return 0 if all nodes reply and there
+are no errors. If any node fails to reply or reports failure, the exit code
+will be 1.
+
+## Usage
+
+Usage: `consul keyring [options]`
+
+Only one actionable argument may be specified per run, including `-list`,
+`-install`, `-remove`, and `-use`.
+
+The list of available flags are:
+
+* `-list` - List all keys currently in use within the cluster.
+
+* `-install` - Install a new encryption key. This will broadcast the new key to
+  all members in the cluster.
+
+* `-use` - Change the primary encryption key, which is used to encrypt messages.
+  The key must already be installed before this operation can succeed.
+
+* `-remove` - Remove the given key from the cluster. This operation may only be
+  performed on keys which are not currently the primary key.
+
+* `-rpc-addr` - RPC address of the Consul agent.
+
+## Output
+
+The output of the `consul keyring -list` command consolidates information from
+all nodes and all datacenters to provide a simple and easy to understand view of
+the cluster. The following is some example output from a cluster with two
+datacenters, each which consist of one server and one client:
+
+```
+==> Gathering installed encryption keys...
+==> Done!
+
+WAN:
+  a1i101sMY8rxB+0eAKD/gw== [2/2]
+
+dc2 (LAN):
+  a1i101sMY8rxB+0eAKD/gw== [2/2]
+
+dc1 (LAN):
+  a1i101sMY8rxB+0eAKD/gw== [2/2]
+```
+
+As you can see, the output above is divided first by gossip pool, and then by
+encryption key. The indicator to the right of each key displays the number of
+nodes the key is installed on over the total number of nodes in the pool.
+
+## Errors
+
+If any errors are encountered while performing a keyring operation, no key
+information is displayed, but instead only error information. The error
+information is arranged in a similar fashion, organized first by datacenter,
+followed by a simple list of nodes which had errors, and the actual text of the
+error. Below is sample output from the same cluster as above, if we try to do
+something that causes an error; in this case, trying to remove the primary key:
+
+```
+==> Removing gossip encryption key...
+
+dc1 (LAN) error: 2/2 nodes reported failure
+  server1: Removing the primary key is not allowed
+  client1: Removing the primary key is not allowed
+
+WAN error: 2/2 nodes reported failure
+  server1.dc1: Removing the primary key is not allowed
+  server2.dc2: Removing the primary key is not allowed
+
+dc2 (LAN) error: 2/2 nodes reported failure
+  server2: Removing the primary key is not allowed
+  client2: Removing the primary key is not allowed
+```
+
+As you can see, each node with a failure reported what went wrong.

website/source/layouts/docs.erb

@@ -79,6 +79,10 @@
 					<a href="/docs/commands/keygen.html">keygen</a>
                     </li>
 
+					<li<%= sidebar_current("docs-commands-keyring") %>>
+					<a href="/docs/commands/keyring.html">keyring</a>
+                    </li>
+
 					<li<%= sidebar_current("docs-commands-leave") %>>
 					<a href="/docs/commands/leave.html">leave</a>
 					</li>