luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Update client fingerprinters

This commit is contained in:
Alex Dadgar 2016-01-22 18:12:16 -08:00
parent 9a28f8ee18
commit d5c77cd4a4
12 changed files with 130 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
client/fingerprint/consul.go
View File

@ -73,12 +73,12 @@ func (f *ConsulFingerprint) Fingerprint(config *client.Config, node *structs.Nod
node.Attributes["consul.server"] = strconv.FormatBool(info["Config"]["Server"].(bool))
node.Attributes["consul.version"] = info["Config"]["Version"].(string)
node.Attributes["consul.revision"] = info["Config"]["Revision"].(string)
node.Attributes["consul.name"] = info["Config"]["NodeName"].(string)
node.Attributes["unique.consul.name"] = info["Config"]["NodeName"].(string)
node.Attributes["consul.datacenter"] = info["Config"]["Datacenter"].(string)
node.Links["consul"] = fmt.Sprintf("%s.%s",
node.Attributes["consul.datacenter"],
node.Attributes["consul.name"])
node.Attributes["unique.consul.name"])
// If the Consul Agent was previously unavailable print a message to
// indicate the Agent is available now
@ -95,7 +95,7 @@ func (f *ConsulFingerprint) clearConsulAttributes(n *structs.Node) {
delete(n.Attributes, "consul.server")
delete(n.Attributes, "consul.version")
delete(n.Attributes, "consul.revision")
delete(n.Attributes, "consul.name")
delete(n.Attributes, "unique.consul.name")
delete(n.Attributes, "consul.datacenter")
delete(n.Links, "consul")
}

2
client/fingerprint/consul_test.go
View File

@ -40,7 +40,7 @@ func TestConsulFingerprint(t *testing.T) {
assertNodeAttributeContains(t, node, "consul.server")
assertNodeAttributeContains(t, node, "consul.version")
assertNodeAttributeContains(t, node, "consul.revision")
assertNodeAttributeContains(t, node, "consul.name")
assertNodeAttributeContains(t, node, "unique.consul.name")
assertNodeAttributeContains(t, node, "consul.datacenter")
expectedLink := "vagrant.consul2"

30
client/fingerprint/env_aws.go
View File

@ -1,6 +1,7 @@
package fingerprint
import (
"fmt"
"io/ioutil"
"log"
"net/http"
@ -114,6 +115,17 @@ func (f *EnvAWSFingerprint) Fingerprint(cfg *config.Config, node *structs.Node)
"public-ipv4",
"placement/availability-zone",
}
// Keys that should be marked as unique
unique := map[string]struct{}{
"ami-id": struct{}{},
"hostname": struct{}{},
"instance-id": struct{}{},
"local-hostname": struct{}{},
"local-ipv4": struct{}{},
"public-hostname": struct{}{},
"public-ipv4": struct{}{},
}
for _, k := range keys {
res, err := client.Get(metadataURL + k)
if res.StatusCode != http.StatusOK {
@ -136,14 +148,18 @@ func (f *EnvAWSFingerprint) Fingerprint(cfg *config.Config, node *structs.Node)
}
// assume we want blank entries
key := strings.Replace(k, "/", ".", -1)
node.Attributes["platform.aws."+key] = strings.Trim(string(resp), "\n")
key := "platform.aws." + strings.Replace(k, "/", ".", -1)
if _, ok := unique[k]; ok {
key = structs.UniqueNamespace(key)
}
node.Attributes[key] = strings.Trim(string(resp), "\n")
}
// copy over network specific information
if node.Attributes["platform.aws.local-ipv4"] != "" {
node.Attributes["network.ip-address"] = node.Attributes["platform.aws.local-ipv4"]
newNetwork.IP = node.Attributes["platform.aws.local-ipv4"]
if val := node.Attributes["unique.platform.aws.local-ipv4"]; val != "" {
node.Attributes["unique.network.ip-address"] = val
newNetwork.IP = val
newNetwork.CIDR = newNetwork.IP + "/32"
}
@ -160,7 +176,9 @@ func (f *EnvAWSFingerprint) Fingerprint(cfg *config.Config, node *structs.Node)
// populate Node Network Resources
// populate Links
node.Links["aws.ec2"] = node.Attributes["platform.aws.placement.availability-zone"] + "." + node.Attributes["platform.aws.instance-id"]
node.Links["aws.ec2"] = fmt.Sprintf("%s.%s",
node.Attributes["platform.aws.placement.availability-zone"],
node.Attributes["unique.platform.aws.instance-id"])
return true, nil
}

18
client/fingerprint/env_aws_test.go
View File

@ -61,16 +61,16 @@ func TestEnvAWSFingerprint_aws(t *testing.T) {
}
keys := []string{
"platform.aws.ami-id",
"platform.aws.hostname",
"platform.aws.instance-id",
"unique.platform.aws.ami-id",
"unique.platform.aws.hostname",
"unique.platform.aws.instance-id",
"platform.aws.instance-type",
"platform.aws.local-hostname",
"platform.aws.local-ipv4",
"platform.aws.public-hostname",
"platform.aws.public-ipv4",
"unique.platform.aws.local-hostname",
"unique.platform.aws.local-ipv4",
"unique.platform.aws.public-hostname",
"unique.platform.aws.public-ipv4",
"platform.aws.placement.availability-zone",
"network.ip-address",
"unique.network.ip-address",
}
for _, k := range keys {
@ -180,7 +180,7 @@ func TestNetworkFingerprint_AWS(t *testing.T) {
t.Fatalf("should apply")
}
assertNodeAttributeContains(t, node, "network.ip-address")
assertNodeAttributeContains(t, node, "unique.network.ip-address")
if node.Resources == nil || len(node.Resources.Networks) == 0 {
t.Fatal("Expected to find Network Resources")

49
client/fingerprint/env_gce.go
View File

@ -2,6 +2,7 @@ package fingerprint
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
@ -140,6 +141,12 @@ func (f *EnvGCEFingerprint) Fingerprint(cfg *config.Config, node *structs.Node)
"scheduling/automatic-restart",
"scheduling/on-host-maintenance",
}
// Keys that should be marked as unique
unique := map[string]struct{}{
"id": struct{}{},
"hostname": struct{}{},
}
for _, k := range keys {
value, err := f.Get(k, false)
if err != nil {
@ -147,8 +154,11 @@ func (f *EnvGCEFingerprint) Fingerprint(cfg *config.Config, node *structs.Node)
}
// assume we want blank entries
key := strings.Replace(k, "/", ".", -1)
node.Attributes["platform.gce."+key] = strings.Trim(string(value), "\n")
key := "platform.gce." + strings.Replace(k, "/", ".", -1)
if _, ok := unique[k]; ok {
key = structs.UniqueNamespace(key)
}
node.Attributes[key] = strings.Trim(string(value), "\n")
}
// These keys need everything before the final slash removed to be usable.
@ -174,10 +184,11 @@ func (f *EnvGCEFingerprint) Fingerprint(cfg *config.Config, node *structs.Node)
for _, intf := range interfaces {
prefix := "platform.gce.network." + lastToken(intf.Network)
uniquePrefix := "unique." + prefix
node.Attributes[prefix] = "true"
node.Attributes[prefix+".ip"] = strings.Trim(intf.Ip, "\n")
node.Attributes[uniquePrefix+".ip"] = strings.Trim(intf.Ip, "\n")
for index, accessConfig := range intf.AccessConfigs {
node.Attributes[prefix+".external-ip."+strconv.Itoa(index)] = accessConfig.ExternalIp
node.Attributes[uniquePrefix+".external-ip."+strconv.Itoa(index)] = accessConfig.ExternalIp
}
}
@ -190,7 +201,19 @@ func (f *EnvGCEFingerprint) Fingerprint(cfg *config.Config, node *structs.Node)
f.logger.Printf("[WARN] fingerprint.env_gce: Error decoding instance tags: %s", err.Error())
}
for _, tag := range tagList {
node.Attributes["platform.gce.tag."+tag] = "true"
attr := "platform.gce.tag."
var key string
// If the tag is namespaced as unique, we strip it from the tag and
// prepend to the whole attribute.
if structs.IsUniqueNamespace(tag) {
tag = strings.TrimPrefix(tag, structs.NodeUniqueNamespace)
key = fmt.Sprintf("%s%s%s", structs.NodeUniqueNamespace, attr, tag)
} else {
key = fmt.Sprintf("%s%s", attr, tag)
}
node.Attributes[key] = "true"
}
var attrDict map[string]string
@ -202,11 +225,23 @@ func (f *EnvGCEFingerprint) Fingerprint(cfg *config.Config, node *structs.Node)
f.logger.Printf("[WARN] fingerprint.env_gce: Error decoding instance attributes: %s", err.Error())
}
for k, v := range attrDict {
node.Attributes["platform.gce.attr."+k] = strings.Trim(v, "\n")
attr := "platform.gce.attr."
var key string
// If the key is namespaced as unique, we strip it from the
// key and prepend to the whole attribute.
if structs.IsUniqueNamespace(k) {
k = strings.TrimPrefix(k, structs.NodeUniqueNamespace)
key = fmt.Sprintf("%s%s%s", structs.NodeUniqueNamespace, attr, k)
} else {
key = fmt.Sprintf("%s%s", attr, k)
}
node.Attributes[key] = strings.Trim(v, "\n")
}
// populate Links
node.Links["gce"] = node.Attributes["platform.gce.id"]
node.Links["gce"] = node.Attributes["unique.platform.gce.id"]
return true, nil
}

26
client/fingerprint/env_gce_test.go
View File

@ -86,15 +86,17 @@ func testFingerprint_GCE(t *testing.T, withExternalIp bool) {
}
keys := []string{
"platform.gce.id",
"platform.gce.hostname",
"unique.platform.gce.id",
"unique.platform.gce.hostname",
"platform.gce.zone",
"platform.gce.machine-type",
"platform.gce.zone",
"platform.gce.tag.abc",
"platform.gce.tag.def",
"unique.platform.gce.tag.foo",
"platform.gce.attr.ghi",
"platform.gce.attr.jkl",
"unique.platform.gce.attr.bar",
}
for _, k := range keys {
@ -110,17 +112,17 @@ func testFingerprint_GCE(t *testing.T, withExternalIp bool) {
assertNodeLinksContains(t, node, k)
}
assertNodeAttributeEquals(t, node, "platform.gce.id", "12345")
assertNodeAttributeEquals(t, node, "platform.gce.hostname", "instance-1.c.project.internal")
assertNodeAttributeEquals(t, node, "unique.platform.gce.id", "12345")
assertNodeAttributeEquals(t, node, "unique.platform.gce.hostname", "instance-1.c.project.internal")
assertNodeAttributeEquals(t, node, "platform.gce.zone", "us-central1-f")
assertNodeAttributeEquals(t, node, "platform.gce.machine-type", "n1-standard-1")
assertNodeAttributeEquals(t, node, "platform.gce.network.default", "true")
assertNodeAttributeEquals(t, node, "platform.gce.network.default.ip", "10.240.0.5")
assertNodeAttributeEquals(t, node, "unique.platform.gce.network.default.ip", "10.240.0.5")
if withExternalIp {
assertNodeAttributeEquals(t, node, "platform.gce.network.default.external-ip.0", "104.44.55.66")
assertNodeAttributeEquals(t, node, "platform.gce.network.default.external-ip.1", "104.44.55.67")
} else if _, ok := node.Attributes["platform.gce.network.default.external-ip.0"]; ok {
t.Fatal("platform.gce.network.default.external-ip is set without an external IP")
assertNodeAttributeEquals(t, node, "unique.platform.gce.network.default.external-ip.0", "104.44.55.66")
assertNodeAttributeEquals(t, node, "unique.platform.gce.network.default.external-ip.1", "104.44.55.67")
} else if _, ok := node.Attributes["unique.platform.gce.network.default.external-ip.0"]; ok {
t.Fatal("unique.platform.gce.network.default.external-ip is set without an external IP")
}
assertNodeAttributeEquals(t, node, "platform.gce.scheduling.automatic-restart", "TRUE")
@ -128,8 +130,10 @@ func testFingerprint_GCE(t *testing.T, withExternalIp bool) {
assertNodeAttributeEquals(t, node, "platform.gce.cpu-platform", "Intel Ivy Bridge")
assertNodeAttributeEquals(t, node, "platform.gce.tag.abc", "true")
assertNodeAttributeEquals(t, node, "platform.gce.tag.def", "true")
assertNodeAttributeEquals(t, node, "unique.platform.gce.tag.foo", "true")
assertNodeAttributeEquals(t, node, "platform.gce.attr.ghi", "111")
assertNodeAttributeEquals(t, node, "platform.gce.attr.jkl", "222")
assertNodeAttributeEquals(t, node, "unique.platform.gce.attr.bar", "333")
}
const GCE_routes = `
@ -158,12 +162,12 @@ const GCE_routes = `
{
"uri": "/computeMetadata/v1/instance/tags",
"content-type": "application/json",
"body": "[\"abc\", \"def\"]"
"body": "[\"abc\", \"def\", \"unique.foo\"]"
},
{
"uri": "/computeMetadata/v1/instance/attributes/?recursive=true",
"content-type": "application/json",
"body": "{\"ghi\":\"111\",\"jkl\":\"222\"}"
"body": "{\"ghi\":\"111\",\"jkl\":\"222\",\"unique.bar\":\"333\"}"
},
{
"uri": "/computeMetadata/v1/instance/scheduling/automatic-restart",

2
client/fingerprint/network.go
View File

@ -74,7 +74,7 @@ func (f *NetworkFingerprint) Fingerprint(cfg *config.Config, node *structs.Node)
}
newNetwork.Device = intf.Name
node.Attributes["network.ip-address"] = ip
node.Attributes["unique.network.ip-address"] = ip
newNetwork.IP = ip
newNetwork.CIDR = newNetwork.IP + "/32"

12
client/fingerprint/network_test.go
View File

@ -152,9 +152,9 @@ func TestNetworkFingerprint_basic(t *testing.T) {
t.Fatalf("should apply")
}
assertNodeAttributeContains(t, node, "network.ip-address")
assertNodeAttributeContains(t, node, "unique.network.ip-address")
ip := node.Attributes["network.ip-address"]
ip := node.Attributes["unique.network.ip-address"]
match := net.ParseIP(ip)
if match == nil {
t.Fatalf("Bad IP match: %s", ip)
@ -229,9 +229,9 @@ func TestNetworkFingerPrint_default_device(t *testing.T) {
t.Fatalf("should apply")
}
assertNodeAttributeContains(t, node, "network.ip-address")
assertNodeAttributeContains(t, node, "unique.network.ip-address")
ip := node.Attributes["network.ip-address"]
ip := node.Attributes["unique.network.ip-address"]
match := net.ParseIP(ip)
if match == nil {
t.Fatalf("Bad IP match: %s", ip)
@ -272,9 +272,9 @@ func TestNetworkFingerPrint_excludelo_down_interfaces(t *testing.T) {
t.Fatalf("should apply")
}
assertNodeAttributeContains(t, node, "network.ip-address")
assertNodeAttributeContains(t, node, "unique.network.ip-address")
ip := node.Attributes["network.ip-address"]
ip := node.Attributes["unique.network.ip-address"]
match := net.ParseIP(ip)
if match == nil {
t.Fatalf("Bad IP match: %s", ip)

12
client/fingerprint/storage.go
View File

@ -27,9 +27,9 @@ func NewStorageFingerprint(logger *log.Logger) Fingerprint {
func (f *StorageFingerprint) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
// Initialize these to empty defaults
node.Attributes["storage.volume"] = ""
node.Attributes["storage.bytestotal"] = ""
node.Attributes["storage.bytesfree"] = ""
node.Attributes["unique.storage.volume"] = ""
node.Attributes["unique.storage.bytestotal"] = ""
node.Attributes["unique.storage.bytesfree"] = ""
if node.Resources == nil {
node.Resources = &structs.Resources{}
}
@ -49,9 +49,9 @@ func (f *StorageFingerprint) Fingerprint(cfg *config.Config, node *structs.Node)
return false, fmt.Errorf("failed to determine disk space for %s: %v", storageDir, err)
}
node.Attributes["storage.volume"] = volume
node.Attributes["storage.bytestotal"] = strconv.FormatUint(total, 10)
node.Attributes["storage.bytesfree"] = strconv.FormatUint(free, 10)
node.Attributes["unique.storage.volume"] = volume
node.Attributes["unique.storage.bytestotal"] = strconv.FormatUint(total, 10)
node.Attributes["unique.storage.bytesfree"] = strconv.FormatUint(free, 10)
node.Resources.DiskMB = int(free / bytesPerMegabyte)

16
client/fingerprint/storage_test.go
View File

@ -15,21 +15,21 @@ func TestStorageFingerprint(t *testing.T) {
assertFingerprintOK(t, fp, node)
assertNodeAttributeContains(t, node, "storage.volume")
assertNodeAttributeContains(t, node, "storage.bytestotal")
assertNodeAttributeContains(t, node, "storage.bytesfree")
assertNodeAttributeContains(t, node, "unique.storage.volume")
assertNodeAttributeContains(t, node, "unique.storage.bytestotal")
assertNodeAttributeContains(t, node, "unique.storage.bytesfree")
total, err := strconv.ParseInt(node.Attributes["storage.bytestotal"], 10, 64)
total, err := strconv.ParseInt(node.Attributes["unique.storage.bytestotal"], 10, 64)
if err != nil {
t.Fatalf("Failed to parse storage.bytestotal: %s", err)
t.Fatalf("Failed to parse unique.storage.bytestotal: %s", err)
}
free, err := strconv.ParseInt(node.Attributes["storage.bytesfree"], 10, 64)
free, err := strconv.ParseInt(node.Attributes["unique.storage.bytesfree"], 10, 64)
if err != nil {
t.Fatalf("Failed to parse storage.bytesfree: %s", err)
t.Fatalf("Failed to parse unique.storage.bytesfree: %s", err)
}
if free > total {
t.Fatalf("storage.bytesfree %d is larger than storage.bytestotal %d", free, total)
t.Fatalf("unique.storage.bytesfree %d is larger than unique.storage.bytestotal %d", free, total)
}
if node.Resources == nil {

16
nomad/structs/node_class.go
View File

@ -13,6 +13,17 @@ const (
NodeUniqueNamespace = "unique."
)
// UniqueNamespace takes a key and returns the key marked under the unique
// namespace.
func UniqueNamespace(key string) string {
return fmt.Sprintf("%s%s", NodeUniqueNamespace, key)
}
// IsUniqueNamespace returns whether the key is under the unique namespace.
func IsUniqueNamespace(key string) bool {
return strings.HasPrefix(key, NodeUniqueNamespace)
}
// ComputeClass computes a derived class for the node based on its attributes.
// ComputedClass is a unique id that identifies nodes with a common set of
// attributes and capabilities. Thus, when calculating a node's computed class
@ -54,12 +65,9 @@ func (n Node) HashIncludeMap(field string, k, v interface{}) (bool, error) {
return false, fmt.Errorf("map key %v not a string")
}
// Check if the key is unique.
isUnique := strings.HasPrefix(key, NodeUniqueNamespace)
switch field {
case "Meta", "Attributes":
return !isUnique, nil
return !IsUniqueNamespace(key), nil
default:
return false, fmt.Errorf("unexpected map field: %v", field)
}

10
nomad/structs/node_class_test.go
View File

@ -1,9 +1,6 @@
package structs
import (
"fmt"
"testing"
)
import "testing"
func testNode() *Node {
return &Node{
@ -110,7 +107,7 @@ func TestNode_ComputedClass_Attr(t *testing.T) {
old := n.ComputedClass
// Add a unique addr and compute the class again
n.Attributes[fmt.Sprintf("%s%s", NodeUniqueNamespace, "foo")] = "bar"
n.Attributes["unique.foo"] = "bar"
if err := n.ComputeClass(); err != nil {
t.Fatalf("ComputeClass() failed: %v", err)
}
@ -157,8 +154,7 @@ func TestNode_ComputedClass_Meta(t *testing.T) {
old = n.ComputedClass
// Add a unique meta key and compute the class again.
key := fmt.Sprintf("%s%s", NodeUniqueNamespace, "foo")
n.Meta[key] = "ignore"
n.Meta["unique.foo"] = "ignore"
if err := n.ComputeClass(); err != nil {
t.Fatalf("ComputeClass() failed: %v", err)
}