luxolus
/
open-consul
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Fixed up config structure for EC2 discovery

This commit is contained in:
Kyle Havlovitz 2016-11-01 14:45:42 -04:00
parent b8e2963a4e
commit 0499784b94
No known key found for this signature in database
GPG Key ID: 8A5E6B173056AD6C
5 changed files with 90 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
command/agent/command.go
View File

@ -1,7 +1,6 @@
package agent
import (
"bytes"
"flag"
"fmt"
"io"
@ -104,11 +103,9 @@ func (c *Command) readConfig() *Config {
cmdFlags.BoolVar(&cmdConfig.AtlasJoin, "atlas-join", false, "auto-join with Atlas")
cmdFlags.StringVar(&cmdConfig.AtlasEndpoint, "atlas-endpoint", "", "endpoint for Atlas integration")
cmdFlags.StringVar(&cmdConfig.AwsAccessKey, "aws-access-key", "", "AWS key for EC2 discovery")
cmdFlags.StringVar(&cmdConfig.AwsSecretKey, "aws-secret-key", "", "AWS secret for EC2 discovery")
cmdFlags.StringVar(&cmdConfig.AwsRegion, "aws-region", "", "Region to search for instances in")
cmdFlags.StringVar(&cmdConfig.EC2TagKey, "ec2-tag-key", "", "EC2 tag key to filter for server discovery")
cmdFlags.StringVar(&cmdConfig.EC2TagValue, "ec2-tag-value", "", "EC2 tag value to filter for server discovery")
cmdFlags.StringVar(&cmdConfig.EC2Discovery.Region, "ec2-region", "", "Region to search for instances in")
cmdFlags.StringVar(&cmdConfig.EC2Discovery.TagKey, "ec2-tag-key", "", "EC2 tag key to filter for server discovery")
cmdFlags.StringVar(&cmdConfig.EC2Discovery.TagValue, "ec2-tag-value", "", "EC2 tag value to filter for server discovery")
cmdFlags.IntVar(&cmdConfig.Protocol, "protocol", -1, "protocol version")
@ -199,6 +196,15 @@ func (c *Command) readConfig() *Config {
config.SkipLeaveOnInt = Bool(config.Server)
}
// Load AWS creds for discovery from the environment (if present)
if os.Getenv("AWS_ACCESS_KEY_ID") != "" {
config.EC2Discovery.AccessKeyID = os.Getenv("AWS_ACCESS_KEY_ID")
}
if os.Getenv("AWS_SECRET_ACCESS_KEY") != "" {
config.EC2Discovery.SecretAccessKey = os.Getenv("AWS_SECRET_ACCESS_KEY")
}
// Ensure we have a data directory
if config.DataDir == "" && !dev {
c.Ui.Error("Must specify data directory using -data-dir")
@ -321,30 +327,24 @@ func (c *Command) readConfig() *Config {
c.Ui.Error("WARNING: Bootstrap mode enabled! Do not enable unless necessary")
}
if (config.AwsAccessKey != "" || config.AwsSecretKey != "") && (config.AwsAccessKey == "" && config.AwsSecretKey == "") {
c.Ui.Error("aws-acces-key and aws-secret-key are required together")
// Populate the join list using EC2 discovery if configured
if config.EC2Discovery.TagKey != "" || config.EC2Discovery.TagValue != "" {
if config.EC2Discovery.TagKey == "" || config.EC2Discovery.TagValue == "" {
c.Ui.Error("EC2 tag key and EC2 tag value are both required")
return nil
}
if config.EC2TagKey != "" || config.EC2TagValue != "" {
if config.EC2TagKey == "" && config.EC2TagValue == "" {
c.Ui.Error("ec2-tag-key and ec2-tag-value are required together")
if config.EC2Discovery.Region == "" {
c.Ui.Error("Amazon EC2 region is required")
return nil
}
if config.AwsRegion == "" {
c.Ui.Error("aws-region is required")
return nil
}
ec2servers, err := config.loadEc2Hosts()
ec2servers, err := config.discoverEc2Hosts()
if err != nil {
c.Ui.Error(fmt.Sprintf("Unable to query EC2 insances: %s", err))
return nil
}
config.StartJoin = append(config.StartJoin, ec2servers...)
fmt.Println(config.StartJoin)
os.Exit(1)
}
// Set the version info
@ -399,40 +399,36 @@ func (config *Config) verifyUniqueListeners() error {
return nil
}
func (config *Config) loadEc2Hosts() ([]string, error) {
// discoverEc2Hosts searches the given AWS region, returning a list of instance
// addresses where EC2TagKey = EC2TagValue
func (c *Config) discoverEc2Hosts() ([]string, error) {
config := c.EC2Discovery
awsConfig := &aws.Config{
Region: aws.String(config.AwsRegion),
}
if config.AwsAccessKey != "" {
awsConfig.Credentials = credentials.NewStaticCredentials(config.AwsAccessKey, config.AwsSecretKey, "")
Region: aws.String(config.Region),
Credentials: credentials.NewStaticCredentials(config.AccessKeyID, config.SecretAccessKey, ""),
}
svc := ec2.New(session.New(), awsConfig)
var search bytes.Buffer
search.WriteString("tag:")
search.WriteString(config.EC2TagKey)
resp, err := svc.DescribeInstances(&ec2.DescribeInstancesInput{
Filters: []*ec2.Filter{
{
Name: aws.String(search.String()),
Name: aws.String("tag:" + config.TagKey),
Values: []*string{
aws.String(config.EC2TagValue),
aws.String(config.TagValue),
},
},
},
})
var servers []string
if err != nil {
return servers, fmt.Errorf("Unable to fetch EC2 instances: %s", err)
return nil, err
}
servers := make([]string, 0)
for i := range resp.Reservations {
for _, inst := range resp.Reservations[i].Instances {
servers = append(servers, *inst.PrivateIpAddress)
for _, instance := range resp.Reservations[i].Instances {
servers = append(servers, *instance.PrivateIpAddress)
}
}
@ -1162,10 +1158,8 @@ Options:
-atlas-join Enables auto-joining the Atlas cluster
-atlas-token=token Provides the Atlas API token
-atlas-endpoint=1.2.3.4 The address of the endpoint for Atlas integration.
-aws-access-key AWS access key used to search for instances
-aws-secret-key AWS secret key for aws-acces-key
-aws-region AWS region to search for instances
-ec2-tag-key=tag The EC2 instance tag to filter on for EC2 discover
-ec2-region The AWS region to search for instances in
-ec2-tag-key=tag The EC2 instance tag to filter on
-ec2-tag-value=value The filter value for ec2-tag-key
-bootstrap Sets server to bootstrap mode
-bind=0.0.0.0 Sets the bind address for cluster communication

29
command/agent/command_test.go
View File

@ -266,6 +266,35 @@ func TestRetryJoinWanFail(t *testing.T) {
}
}
func TestDiscoverEC2Hosts(t *testing.T) {
if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
t.Skip("AWS_ACCESS_KEY_ID not set, skipping")
}
if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
t.Skip("AWS_SECRET_ACCESS_KEY not set, skipping")
}
c := &Config{
EC2Discovery: EC2Discovery{
Region: "us-east-1",
AccessKeyID: os.Getenv("AWS_ACCESS_KEY_ID"),
SecretAccessKey: os.Getenv("AWS_SECRET_ACCESS_KEY"),
TagKey: "ConsulRole",
TagValue: "Server",
},
}
servers, err := c.discoverEc2Hosts()
if err != nil {
t.Fatal(err)
}
t.Log(servers)
if len(servers) != 3 {
t.Fatalf("bad: %v", servers)
}
}
func TestSetupAgent_RPCUnixSocket_FileExists(t *testing.T) {
conf := nextConfig()
tmpDir, err := ioutil.TempDir("", "consul")

50
command/agent/config.go
View File

@ -118,6 +118,20 @@ type DNSConfig struct {
RecursorTimeoutRaw string `mapstructure:"recursor_timeout" json:"-"`
}
// EC2Discovery is used to configure discovery of instances via Amazon's EC2 api
type EC2Discovery struct {
// The AWS region to look for instances in
Region string `mapstructure:"region"`
// The tag key and value to use when filtering instances
TagKey string `mapstructure:"tag_key"`
TagValue string `mapstructure:"tag_value"`
// The AWS credentials to use for making requests to EC2
AccessKeyID string `mapstructure:"access_key_id"`
SecretAccessKey string `mapstructure:"secret_access_key"`
}
// Performance is used to tune the performance of Consul's subsystems.
type Performance struct {
// RaftMultiplier is an integer multiplier used to scale Raft timing
@ -530,20 +544,8 @@ type Config struct {
// empty, the defaults from the provider are used.
AtlasEndpoint string `mapstructure:"atlas_endpoint"`
// AwsAccessKey is an AWS IAM key used for discovering EC2 instances
AwsAccessKey string `mapstructure:"aws_access_key"`
// AwsSecretKey is the secret key for AwsAccessKey
AwsSecretKey string `mapstructure:aws_secret_key`
// AwsRegion is the region to attempt to discover instances in
AwsRegion string `mapstructure:aws_region`
// EC2TagKey is the tag applied to EC2 instances to filter on to discover servers
EC2TagKey string `mapstructure:"ec2_tag_key"`
// EC2TagValue is the value of ec2-tag-key to filter for
EC2TagValue string `mapstructure:"ec2_tag_value"`
// EC2Discovery configuration
EC2Discovery EC2Discovery `mapstructure:"ec2_discovery"`
// AEInterval controls the anti-entropy interval. This is how often
// the agent attempts to reconcile its local state with the server's
@ -1450,20 +1452,20 @@ func MergeConfig(a, b *Config) *Config {
if b.AtlasEndpoint != "" {
result.AtlasEndpoint = b.AtlasEndpoint
}
if b.AwsAccessKey != "" {
result.AwsAccessKey = b.AwsAccessKey
if b.EC2Discovery.AccessKeyID != "" {
result.EC2Discovery.AccessKeyID = b.EC2Discovery.AccessKeyID
}
if b.AwsSecretKey != "" {
result.AwsSecretKey = b.AwsSecretKey
if b.EC2Discovery.SecretAccessKey != "" {
result.EC2Discovery.SecretAccessKey = b.EC2Discovery.SecretAccessKey
}
if b.AwsRegion != "" {
result.AwsRegion = b.AwsRegion
if b.EC2Discovery.Region != "" {
result.EC2Discovery.Region = b.EC2Discovery.Region
}
if b.EC2TagKey != "" {
result.EC2TagKey = b.EC2TagKey
if b.EC2Discovery.TagKey != "" {
result.EC2Discovery.TagKey = b.EC2Discovery.TagKey
}
if b.EC2TagValue != "" {
result.EC2TagValue = b.EC2TagValue
if b.EC2Discovery.TagValue != "" {
result.EC2Discovery.TagValue = b.EC2Discovery.TagValue
}
if b.DisableCoordinates {
result.DisableCoordinates = true

1
terraform/aws/consul.tf
View File

@ -13,6 +13,7 @@ resource "aws_instance" "server" {
#Instance tags
tags {
Name = "${var.tagName}-${count.index}"
ConsulRole = "Server"
}
provisioner "file" {

2
terraform/shared/scripts/install.sh
View File

@ -12,7 +12,7 @@ fi
echo "Fetching Consul..."
CONSUL=0.6.4
CONSUL=0.7.0
cd /tmp
wget https://releases.hashicorp.com/consul/${CONSUL}/consul_${CONSUL}_linux_amd64.zip -O consul.zip