Merge pull request #7509 from hashicorp/b-ec2metadata-outside-aws

fingerprint: handle incomplete AWS imitation APIs

client/fingerprint/env_aws.go

@@ -80,15 +80,10 @@ func (f *EnvAWSFingerprint) Fingerprint(request *FingerprintRequest, response *F
 		return fmt.Errorf("failed to setup ec2Metadata client: %v", err)
 	}
 
-	if !ec2meta.Available() {
+	if !isAWS(ec2meta) {
 		return nil
 	}
 
-	// newNetwork is populated and added to the Nodes resources
-	newNetwork := &structs.NetworkResource{
-		Device: "eth0",
-	}
-
 	// Keys and whether they should be namespaced as unique. Any key whose value
 	// uniquely identifies a node, such as ip, should be marked as unique. When
 	// marked as unique, the key isn't included in the computed node class.
@@ -103,9 +98,14 @@ func (f *EnvAWSFingerprint) Fingerprint(request *FingerprintRequest, response *F
 		"public-ipv4":                 true,
 		"placement/availability-zone": false,
 	}
+
 	for k, unique := range keys {
 		resp, err := ec2meta.GetMetadata(k)
-		if awsErr, ok := err.(awserr.RequestFailure); ok {
+		v := strings.TrimSpace(resp)
+		if v == "" {
+			f.logger.Debug("read an empty value", "attribute", k)
+			continue
+		} else if awsErr, ok := err.(awserr.RequestFailure); ok {
 			f.logger.Debug("could not read attribute value", "attribute", k, "error", awsErr)
 			continue
 		} else if awsErr, ok := err.(awserr.Error); ok {
@@ -125,44 +125,27 @@ func (f *EnvAWSFingerprint) Fingerprint(request *FingerprintRequest, response *F
 			key = structs.UniqueNamespace(key)
 		}
 
-		response.AddAttribute(key, strings.Trim(resp, "\n"))
+		response.AddAttribute(key, v)
 	}
 
+	// newNetwork is populated and added to the Nodes resources
+	var newNetwork *structs.NetworkResource
+
 	// copy over network specific information
 	if val, ok := response.Attributes["unique.platform.aws.local-ipv4"]; ok && val != "" {
 		response.AddAttribute("unique.network.ip-address", val)
-		newNetwork.IP = val
+
-		newNetwork.CIDR = newNetwork.IP + "/32"
+		newNetwork = &structs.NetworkResource{
+			Device: "eth0",
+			IP:     val,
+			CIDR:   val + "/32",
+			MBits:  f.throughput(request, ec2meta, val),
 		}
 
-	// find LinkSpeed from lookup
-	throughput := cfg.NetworkSpeed
-	if throughput == 0 {
-		throughput = f.linkSpeed(ec2meta)
-	}
-	if throughput == 0 {
-		// Failed to determine speed. Check if the network fingerprint got it
-		found := false
-		if request.Node.Resources != nil && len(request.Node.Resources.Networks) > 0 {
-			for _, n := range request.Node.Resources.Networks {
-				if n.IP == newNetwork.IP {
-					throughput = n.MBits
-					found = true
-					break
-				}
-			}
-		}
-
-		// Nothing detected so default
-		if !found {
-			throughput = defaultNetworkSpeed
-		}
-	}
-
-	newNetwork.MBits = throughput
 		response.NodeResources = &structs.NodeResources{
 			Networks: []*structs.NetworkResource{newNetwork},
 		}
+	}
 
 	// populate Links
 	response.AddLink("aws.ec2", fmt.Sprintf("%s.%s",
@@ -173,6 +156,28 @@ func (f *EnvAWSFingerprint) Fingerprint(request *FingerprintRequest, response *F
 	return nil
 }
 
+func (f *EnvAWSFingerprint) throughput(request *FingerprintRequest, ec2meta *ec2metadata.EC2Metadata, ip string) int {
+	throughput := request.Config.NetworkSpeed
+	if throughput != 0 {
+		return throughput
+	}
+
+	throughput = f.linkSpeed(ec2meta)
+	if throughput != 0 {
+		return throughput
+	}
+
+	if request.Node.Resources != nil && len(request.Node.Resources.Networks) > 0 {
+		for _, n := range request.Node.Resources.Networks {
+			if n.IP == ip {
+				return n.MBits
+			}
+		}
+	}
+
+	return defaultNetworkSpeed
+}
+
 // EnvAWSFingerprint uses lookup table to approximate network speeds
 func (f *EnvAWSFingerprint) linkSpeed(ec2meta *ec2metadata.EC2Metadata) int {
 
@@ -211,3 +216,9 @@ func ec2MetaClient(endpoint string, timeout time.Duration) (*ec2metadata.EC2Meta
 	}
 	return ec2metadata.New(session, c), nil
 }
+
+func isAWS(ec2meta *ec2metadata.EC2Metadata) bool {
+	v, err := ec2meta.GetMetadata("ami-id")
+	v = strings.TrimSpace(v)
+	return err == nil && v != ""
+}

client/fingerprint/env_aws_test.go

@@ -1,7 +1,6 @@
 package fingerprint
 
 import (
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
@@ -29,7 +28,7 @@ func TestEnvAWSFingerprint_nonAws(t *testing.T) {
 }
 
 func TestEnvAWSFingerprint_aws(t *testing.T) {
-	endpoint, cleanup := startFakeEC2Metadata(t)
+	endpoint, cleanup := startFakeEC2Metadata(t, awsStubs)
 	defer cleanup()
 
 	f := NewEnvAWSFingerprint(testlog.HCLogger(t))
@@ -70,7 +69,7 @@ func TestEnvAWSFingerprint_aws(t *testing.T) {
 }
 
 func TestNetworkFingerprint_AWS(t *testing.T) {
-	endpoint, cleanup := startFakeEC2Metadata(t)
+	endpoint, cleanup := startFakeEC2Metadata(t, awsStubs)
 	defer cleanup()
 
 	f := NewEnvAWSFingerprint(testlog.HCLogger(t))
@@ -98,7 +97,7 @@ func TestNetworkFingerprint_AWS(t *testing.T) {
 }
 
 func TestNetworkFingerprint_AWS_network(t *testing.T) {
-	endpoint, cleanup := startFakeEC2Metadata(t)
+	endpoint, cleanup := startFakeEC2Metadata(t, awsStubs)
 	defer cleanup()
 
 	f := NewEnvAWSFingerprint(testlog.HCLogger(t))
@@ -158,16 +157,56 @@ func TestNetworkFingerprint_AWS_network(t *testing.T) {
 	}
 }
 
-/// Utility functions for tests
+func TestNetworkFingerprint_AWS_NoNetwork(t *testing.T) {
+	endpoint, cleanup := startFakeEC2Metadata(t, noNetworkAWSStubs)
+	defer cleanup()
 
-func startFakeEC2Metadata(t *testing.T) (endpoint string, cleanup func()) {
+	f := NewEnvAWSFingerprint(testlog.HCLogger(t))
-	routes := routes{}
+	f.(*EnvAWSFingerprint).endpoint = endpoint
-	if err := json.Unmarshal([]byte(aws_routes), &routes); err != nil {
+
-		t.Fatalf("Failed to unmarshal JSON in AWS ENV test: %s", err)
+	node := &structs.Node{
+		Attributes: make(map[string]string),
 	}
 
+	request := &FingerprintRequest{Config: &config.Config{}, Node: node}
+	var response FingerprintResponse
+	err := f.Fingerprint(request, &response)
+	require.NoError(t, err)
+
+	require.True(t, response.Detected, "expected response to be applicable")
+
+	require.Equal(t, "ami-1234", response.Attributes["platform.aws.ami-id"])
+
+	require.Nil(t, response.NodeResources)
+}
+
+func TestNetworkFingerprint_AWS_IncompleteImitation(t *testing.T) {
+	endpoint, cleanup := startFakeEC2Metadata(t, incompleteAWSImitationStubs)
+	defer cleanup()
+
+	f := NewEnvAWSFingerprint(testlog.HCLogger(t))
+	f.(*EnvAWSFingerprint).endpoint = endpoint
+
+	node := &structs.Node{
+		Attributes: make(map[string]string),
+	}
+
+	request := &FingerprintRequest{Config: &config.Config{}, Node: node}
+	var response FingerprintResponse
+	err := f.Fingerprint(request, &response)
+	require.NoError(t, err)
+
+	require.False(t, response.Detected, "expected response not to be applicable")
+
+	require.NotContains(t, response.Attributes, "platform.aws.ami-id")
+	require.Nil(t, response.NodeResources)
+}
+
+/// Utility functions for tests
+
+func startFakeEC2Metadata(t *testing.T, endpoints []endpoint) (endpoint string, cleanup func()) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		for _, e := range routes.Endpoints {
+		for _, e := range endpoints {
 			if r.RequestURI == e.Uri {
 				w.Header().Set("Content-Type", e.ContentType)
 				fmt.Fprintln(w, e.Body)
@@ -181,60 +220,133 @@ func startFakeEC2Metadata(t *testing.T) (endpoint string, cleanup func()) {
 type routes struct {
 	Endpoints []*endpoint `json:"endpoints"`
 }
+
 type endpoint struct {
 	Uri         string `json:"uri"`
 	ContentType string `json:"content-type"`
 	Body        string `json:"body"`
 }
 
-const aws_routes = `
+// awsStubs mimics normal EC2 instance metadata
+var awsStubs = []endpoint{
 	{
-  "endpoints": [
+		Uri:         "/latest/meta-data/ami-id",
-    {
+		ContentType: "text/plain",
-      "uri": "/latest/meta-data/ami-id",
+		Body:        "ami-1234",
-      "content-type": "text/plain",
-      "body": "ami-1234"
 	},
 	{
-      "uri": "/latest/meta-data/hostname",
+		Uri:         "/latest/meta-data/hostname",
-      "content-type": "text/plain",
+		ContentType: "text/plain",
-      "body": "ip-10-0-0-207.us-west-2.compute.internal"
+		Body:        "ip-10-0-0-207.us-west-2.compute.internal",
 	},
 	{
-      "uri": "/latest/meta-data/placement/availability-zone",
+		Uri:         "/latest/meta-data/placement/availability-zone",
-      "content-type": "text/plain",
+		ContentType: "text/plain",
-      "body": "us-west-2a"
+		Body:        "us-west-2a",
 	},
 	{
-      "uri": "/latest/meta-data/instance-id",
+		Uri:         "/latest/meta-data/instance-id",
-      "content-type": "text/plain",
+		ContentType: "text/plain",
-      "body": "i-b3ba3875"
+		Body:        "i-b3ba3875",
 	},
 	{
-      "uri": "/latest/meta-data/instance-type",
+		Uri:         "/latest/meta-data/instance-type",
-      "content-type": "text/plain",
+		ContentType: "text/plain",
-      "body": "m3.2xlarge"
+		Body:        "m3.2xlarge",
 	},
 	{
-      "uri": "/latest/meta-data/local-hostname",
+		Uri:         "/latest/meta-data/local-hostname",
-      "content-type": "text/plain",
+		ContentType: "text/plain",
-      "body": "ip-10-0-0-207.us-west-2.compute.internal"
+		Body:        "ip-10-0-0-207.us-west-2.compute.internal",
 	},
 	{
-      "uri": "/latest/meta-data/local-ipv4",
+		Uri:         "/latest/meta-data/local-ipv4",
-      "content-type": "text/plain",
+		ContentType: "text/plain",
-      "body": "10.0.0.207"
+		Body:        "10.0.0.207",
 	},
 	{
-      "uri": "/latest/meta-data/public-hostname",
+		Uri:         "/latest/meta-data/public-hostname",
-      "content-type": "text/plain",
+		ContentType: "text/plain",
-      "body": "ec2-54-191-117-175.us-west-2.compute.amazonaws.com"
+		Body:        "ec2-54-191-117-175.us-west-2.compute.amazonaws.com",
 	},
 	{
-      "uri": "/latest/meta-data/public-ipv4",
+		Uri:         "/latest/meta-data/public-ipv4",
-      "content-type": "text/plain",
+		ContentType: "text/plain",
-      "body": "54.191.117.175"
+		Body:        "54.191.117.175",
+	},
 }
-  ]
+
+// noNetworkAWSStubs mimics an EC2 instance but without local ip address
+// may happen in environments with odd EC2 Metadata emulation
+var noNetworkAWSStubs = []endpoint{
+	{
+		Uri:         "/latest/meta-data/ami-id",
+		ContentType: "text/plain",
+		Body:        "ami-1234",
+	},
+	{
+		Uri:         "/latest/meta-data/hostname",
+		ContentType: "text/plain",
+		Body:        "ip-10-0-0-207.us-west-2.compute.internal",
+	},
+	{
+		Uri:         "/latest/meta-data/placement/availability-zone",
+		ContentType: "text/plain",
+		Body:        "us-west-2a",
+	},
+	{
+		Uri:         "/latest/meta-data/instance-id",
+		ContentType: "text/plain",
+		Body:        "i-b3ba3875",
+	},
+	{
+		Uri:         "/latest/meta-data/instance-type",
+		ContentType: "text/plain",
+		Body:        "m3.2xlarge",
+	},
+	{
+		Uri:         "/latest/meta-data/local-hostname",
+		ContentType: "text/plain",
+		Body:        "ip-10-0-0-207.us-west-2.compute.internal",
+	},
+	{
+		Uri:         "/latest/meta-data/local-ipv4",
+		ContentType: "text/plain",
+		Body:        "",
+	},
+	{
+		Uri:         "/latest/meta-data/public-hostname",
+		ContentType: "text/plain",
+		Body:        "ec2-54-191-117-175.us-west-2.compute.amazonaws.com",
+	},
+	{
+		Uri:         "/latest/meta-data/public-ipv4",
+		ContentType: "text/plain",
+		Body:        "54.191.117.175",
+	},
+}
+
+// incompleteAWSImitationsStub mimics environments where some AWS endpoints
+// return empty, namely Hetzner
+var incompleteAWSImitationStubs = []endpoint{
+	{
+		Uri:         "/latest/meta-data/hostname",
+		ContentType: "text/plain",
+		Body:        "ip-10-0-0-207.us-west-2.compute.internal",
+	},
+	{
+		Uri:         "/latest/meta-data/instance-id",
+		ContentType: "text/plain",
+		Body:        "i-b3ba3875",
+	},
+	{
+		Uri:         "/latest/meta-data/local-ipv4",
+		ContentType: "text/plain",
+		Body:        "",
+	},
+	{
+		Uri:         "/latest/meta-data/public-ipv4",
+		ContentType: "text/plain",
+		Body:        "54.191.117.175",
+	},
 }
-`