diff --git a/command/agent/command.go b/command/agent/command.go index bd1ced1de..61bc62c4a 100644 --- a/command/agent/command.go +++ b/command/agent/command.go @@ -1,7 +1,6 @@ package agent import ( - "bytes" "flag" "fmt" "io" @@ -104,11 +103,9 @@ func (c *Command) readConfig() *Config { cmdFlags.BoolVar(&cmdConfig.AtlasJoin, "atlas-join", false, "auto-join with Atlas") cmdFlags.StringVar(&cmdConfig.AtlasEndpoint, "atlas-endpoint", "", "endpoint for Atlas integration") - cmdFlags.StringVar(&cmdConfig.AwsAccessKey, "aws-access-key", "", "AWS key for EC2 discovery") - cmdFlags.StringVar(&cmdConfig.AwsSecretKey, "aws-secret-key", "", "AWS secret for EC2 discovery") - cmdFlags.StringVar(&cmdConfig.AwsRegion, "aws-region", "", "Region to search for instances in") - cmdFlags.StringVar(&cmdConfig.EC2TagKey, "ec2-tag-key", "", "EC2 tag key to filter for server discovery") - cmdFlags.StringVar(&cmdConfig.EC2TagValue, "ec2-tag-value", "", "EC2 tag value to filter for server discovery") + cmdFlags.StringVar(&cmdConfig.EC2Discovery.Region, "ec2-region", "", "Region to search for instances in") + cmdFlags.StringVar(&cmdConfig.EC2Discovery.TagKey, "ec2-tag-key", "", "EC2 tag key to filter for server discovery") + cmdFlags.StringVar(&cmdConfig.EC2Discovery.TagValue, "ec2-tag-value", "", "EC2 tag value to filter for server discovery") cmdFlags.IntVar(&cmdConfig.Protocol, "protocol", -1, "protocol version") @@ -199,6 +196,15 @@ func (c *Command) readConfig() *Config { config.SkipLeaveOnInt = Bool(config.Server) } + // Load AWS creds for discovery from the environment (if present) + if os.Getenv("AWS_ACCESS_KEY_ID") != "" { + config.EC2Discovery.AccessKeyID = os.Getenv("AWS_ACCESS_KEY_ID") + } + + if os.Getenv("AWS_SECRET_ACCESS_KEY") != "" { + config.EC2Discovery.SecretAccessKey = os.Getenv("AWS_SECRET_ACCESS_KEY") + } + // Ensure we have a data directory if config.DataDir == "" && !dev { c.Ui.Error("Must specify data directory using -data-dir") @@ -321,30 +327,24 @@ func (c *Command) readConfig() *Config { c.Ui.Error("WARNING: Bootstrap mode enabled! Do not enable unless necessary") } - if (config.AwsAccessKey != "" || config.AwsSecretKey != "") && (config.AwsAccessKey == "" && config.AwsSecretKey == "") { - c.Ui.Error("aws-acces-key and aws-secret-key are required together") - return nil - } - - if config.EC2TagKey != "" || config.EC2TagValue != "" { - if config.EC2TagKey == "" && config.EC2TagValue == "" { - c.Ui.Error("ec2-tag-key and ec2-tag-value are required together") + // Populate the join list using EC2 discovery if configured + if config.EC2Discovery.TagKey != "" || config.EC2Discovery.TagValue != "" { + if config.EC2Discovery.TagKey == "" || config.EC2Discovery.TagValue == "" { + c.Ui.Error("EC2 tag key and EC2 tag value are both required") return nil } - if config.AwsRegion == "" { - c.Ui.Error("aws-region is required") + if config.EC2Discovery.Region == "" { + c.Ui.Error("Amazon EC2 region is required") return nil } - ec2servers, err := config.loadEc2Hosts() + ec2servers, err := config.discoverEc2Hosts() if err != nil { c.Ui.Error(fmt.Sprintf("Unable to query EC2 insances: %s", err)) return nil } config.StartJoin = append(config.StartJoin, ec2servers...) - fmt.Println(config.StartJoin) - os.Exit(1) } // Set the version info @@ -399,40 +399,36 @@ func (config *Config) verifyUniqueListeners() error { return nil } -func (config *Config) loadEc2Hosts() ([]string, error) { +// discoverEc2Hosts searches the given AWS region, returning a list of instance +// addresses where EC2TagKey = EC2TagValue +func (c *Config) discoverEc2Hosts() ([]string, error) { + config := c.EC2Discovery awsConfig := &aws.Config{ - Region: aws.String(config.AwsRegion), - } - - if config.AwsAccessKey != "" { - awsConfig.Credentials = credentials.NewStaticCredentials(config.AwsAccessKey, config.AwsSecretKey, "") + Region: aws.String(config.Region), + Credentials: credentials.NewStaticCredentials(config.AccessKeyID, config.SecretAccessKey, ""), } svc := ec2.New(session.New(), awsConfig) - var search bytes.Buffer - search.WriteString("tag:") - search.WriteString(config.EC2TagKey) - resp, err := svc.DescribeInstances(&ec2.DescribeInstancesInput{ Filters: []*ec2.Filter{ { - Name: aws.String(search.String()), + Name: aws.String("tag:" + config.TagKey), Values: []*string{ - aws.String(config.EC2TagValue), + aws.String(config.TagValue), }, }, }, }) - var servers []string if err != nil { - return servers, fmt.Errorf("Unable to fetch EC2 instances: %s", err) + return nil, err } + servers := make([]string, 0) for i := range resp.Reservations { - for _, inst := range resp.Reservations[i].Instances { - servers = append(servers, *inst.PrivateIpAddress) + for _, instance := range resp.Reservations[i].Instances { + servers = append(servers, *instance.PrivateIpAddress) } } @@ -1162,10 +1158,8 @@ Options: -atlas-join Enables auto-joining the Atlas cluster -atlas-token=token Provides the Atlas API token -atlas-endpoint=1.2.3.4 The address of the endpoint for Atlas integration. - -aws-access-key AWS access key used to search for instances - -aws-secret-key AWS secret key for aws-acces-key - -aws-region AWS region to search for instances - -ec2-tag-key=tag The EC2 instance tag to filter on for EC2 discover + -ec2-region The AWS region to search for instances in + -ec2-tag-key=tag The EC2 instance tag to filter on -ec2-tag-value=value The filter value for ec2-tag-key -bootstrap Sets server to bootstrap mode -bind=0.0.0.0 Sets the bind address for cluster communication diff --git a/command/agent/command_test.go b/command/agent/command_test.go index 6eec51ab2..ab0f1990c 100644 --- a/command/agent/command_test.go +++ b/command/agent/command_test.go @@ -266,6 +266,35 @@ func TestRetryJoinWanFail(t *testing.T) { } } +func TestDiscoverEC2Hosts(t *testing.T) { + if os.Getenv("AWS_ACCESS_KEY_ID") == "" { + t.Skip("AWS_ACCESS_KEY_ID not set, skipping") + } + + if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" { + t.Skip("AWS_SECRET_ACCESS_KEY not set, skipping") + } + + c := &Config{ + EC2Discovery: EC2Discovery{ + Region: "us-east-1", + AccessKeyID: os.Getenv("AWS_ACCESS_KEY_ID"), + SecretAccessKey: os.Getenv("AWS_SECRET_ACCESS_KEY"), + TagKey: "ConsulRole", + TagValue: "Server", + }, + } + + servers, err := c.discoverEc2Hosts() + if err != nil { + t.Fatal(err) + } + t.Log(servers) + if len(servers) != 3 { + t.Fatalf("bad: %v", servers) + } +} + func TestSetupAgent_RPCUnixSocket_FileExists(t *testing.T) { conf := nextConfig() tmpDir, err := ioutil.TempDir("", "consul") diff --git a/command/agent/config.go b/command/agent/config.go index 3e5b44724..a80d42ae7 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -118,6 +118,20 @@ type DNSConfig struct { RecursorTimeoutRaw string `mapstructure:"recursor_timeout" json:"-"` } +// EC2Discovery is used to configure discovery of instances via Amazon's EC2 api +type EC2Discovery struct { + // The AWS region to look for instances in + Region string `mapstructure:"region"` + + // The tag key and value to use when filtering instances + TagKey string `mapstructure:"tag_key"` + TagValue string `mapstructure:"tag_value"` + + // The AWS credentials to use for making requests to EC2 + AccessKeyID string `mapstructure:"access_key_id"` + SecretAccessKey string `mapstructure:"secret_access_key"` +} + // Performance is used to tune the performance of Consul's subsystems. type Performance struct { // RaftMultiplier is an integer multiplier used to scale Raft timing @@ -530,20 +544,8 @@ type Config struct { // empty, the defaults from the provider are used. AtlasEndpoint string `mapstructure:"atlas_endpoint"` - // AwsAccessKey is an AWS IAM key used for discovering EC2 instances - AwsAccessKey string `mapstructure:"aws_access_key"` - - // AwsSecretKey is the secret key for AwsAccessKey - AwsSecretKey string `mapstructure:aws_secret_key` - - // AwsRegion is the region to attempt to discover instances in - AwsRegion string `mapstructure:aws_region` - - // EC2TagKey is the tag applied to EC2 instances to filter on to discover servers - EC2TagKey string `mapstructure:"ec2_tag_key"` - - // EC2TagValue is the value of ec2-tag-key to filter for - EC2TagValue string `mapstructure:"ec2_tag_value"` + // EC2Discovery configuration + EC2Discovery EC2Discovery `mapstructure:"ec2_discovery"` // AEInterval controls the anti-entropy interval. This is how often // the agent attempts to reconcile its local state with the server's @@ -1450,20 +1452,20 @@ func MergeConfig(a, b *Config) *Config { if b.AtlasEndpoint != "" { result.AtlasEndpoint = b.AtlasEndpoint } - if b.AwsAccessKey != "" { - result.AwsAccessKey = b.AwsAccessKey + if b.EC2Discovery.AccessKeyID != "" { + result.EC2Discovery.AccessKeyID = b.EC2Discovery.AccessKeyID } - if b.AwsSecretKey != "" { - result.AwsSecretKey = b.AwsSecretKey + if b.EC2Discovery.SecretAccessKey != "" { + result.EC2Discovery.SecretAccessKey = b.EC2Discovery.SecretAccessKey } - if b.AwsRegion != "" { - result.AwsRegion = b.AwsRegion + if b.EC2Discovery.Region != "" { + result.EC2Discovery.Region = b.EC2Discovery.Region } - if b.EC2TagKey != "" { - result.EC2TagKey = b.EC2TagKey + if b.EC2Discovery.TagKey != "" { + result.EC2Discovery.TagKey = b.EC2Discovery.TagKey } - if b.EC2TagValue != "" { - result.EC2TagValue = b.EC2TagValue + if b.EC2Discovery.TagValue != "" { + result.EC2Discovery.TagValue = b.EC2Discovery.TagValue } if b.DisableCoordinates { result.DisableCoordinates = true diff --git a/terraform/aws/consul.tf b/terraform/aws/consul.tf index 484b15831..19c5c318a 100644 --- a/terraform/aws/consul.tf +++ b/terraform/aws/consul.tf @@ -13,6 +13,7 @@ resource "aws_instance" "server" { #Instance tags tags { Name = "${var.tagName}-${count.index}" + ConsulRole = "Server" } provisioner "file" { diff --git a/terraform/shared/scripts/install.sh b/terraform/shared/scripts/install.sh index 08e2fdffb..9566cfed1 100644 --- a/terraform/shared/scripts/install.sh +++ b/terraform/shared/scripts/install.sh @@ -12,7 +12,7 @@ fi echo "Fetching Consul..." -CONSUL=0.6.4 +CONSUL=0.7.0 cd /tmp wget https://releases.hashicorp.com/consul/${CONSUL}/consul_${CONSUL}_linux_amd64.zip -O consul.zip