2016-11-01 19:45:37 +00:00
|
|
|
// Copyright 2016 Circonus, Inc. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
2016-07-22 16:33:10 +00:00
|
|
|
package checkmgr
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"math/rand"
|
|
|
|
"net"
|
|
|
|
"net/url"
|
|
|
|
"reflect"
|
2016-11-09 00:06:55 +00:00
|
|
|
"strconv"
|
2016-07-22 16:33:10 +00:00
|
|
|
"strings"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/circonus-labs/circonus-gometrics/api"
|
|
|
|
)
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
rand.Seed(time.Now().UnixNano())
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get Broker to use when creating a check
|
|
|
|
func (cm *CheckManager) getBroker() (*api.Broker, error) {
|
|
|
|
if cm.brokerID != 0 {
|
2017-01-31 19:42:18 +00:00
|
|
|
cid := fmt.Sprintf("/broker/%d", cm.brokerID)
|
|
|
|
broker, err := cm.apih.FetchBroker(api.CIDType(&cid))
|
2016-07-22 16:33:10 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if !cm.isValidBroker(broker) {
|
|
|
|
return nil, fmt.Errorf(
|
|
|
|
"[ERROR] designated broker %d [%s] is invalid (not active, does not support required check type, or connectivity issue)",
|
|
|
|
cm.brokerID,
|
|
|
|
broker.Name)
|
|
|
|
}
|
|
|
|
return broker, nil
|
|
|
|
}
|
|
|
|
broker, err := cm.selectBroker()
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("[ERROR] Unable to fetch suitable broker %s", err)
|
|
|
|
}
|
|
|
|
return broker, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get CN of Broker associated with submission_url to satisfy no IP SANS in certs
|
|
|
|
func (cm *CheckManager) getBrokerCN(broker *api.Broker, submissionURL api.URLType) (string, error) {
|
|
|
|
u, err := url.Parse(string(submissionURL))
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
hostParts := strings.Split(u.Host, ":")
|
|
|
|
host := hostParts[0]
|
|
|
|
|
|
|
|
if net.ParseIP(host) == nil { // it's a non-ip string
|
|
|
|
return u.Host, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
cn := ""
|
|
|
|
|
|
|
|
for _, detail := range broker.Details {
|
2017-01-31 19:42:18 +00:00
|
|
|
if *detail.IP == host {
|
2016-07-22 16:33:10 +00:00
|
|
|
cn = detail.CN
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if cn == "" {
|
|
|
|
return "", fmt.Errorf("[ERROR] Unable to match URL host (%s) to Broker", u.Host)
|
|
|
|
}
|
|
|
|
|
|
|
|
return cn, nil
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
// Select a broker for use when creating a check, if a specific broker
|
|
|
|
// was not specified.
|
|
|
|
func (cm *CheckManager) selectBroker() (*api.Broker, error) {
|
2017-01-31 19:42:18 +00:00
|
|
|
var brokerList *[]api.Broker
|
2016-07-22 16:33:10 +00:00
|
|
|
var err error
|
2018-09-10 22:03:17 +00:00
|
|
|
enterpriseType := "enterprise"
|
2016-07-22 16:33:10 +00:00
|
|
|
|
2016-11-01 19:45:37 +00:00
|
|
|
if len(cm.brokerSelectTag) > 0 {
|
2017-01-31 19:42:18 +00:00
|
|
|
filter := api.SearchFilterType{
|
|
|
|
"f__tags_has": cm.brokerSelectTag,
|
|
|
|
}
|
|
|
|
brokerList, err = cm.apih.SearchBrokers(nil, &filter)
|
2016-07-22 16:33:10 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
} else {
|
2017-01-31 19:42:18 +00:00
|
|
|
brokerList, err = cm.apih.FetchBrokers()
|
2016-07-22 16:33:10 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-01-31 19:42:18 +00:00
|
|
|
if len(*brokerList) == 0 {
|
2016-07-22 16:33:10 +00:00
|
|
|
return nil, fmt.Errorf("zero brokers found")
|
|
|
|
}
|
|
|
|
|
|
|
|
validBrokers := make(map[string]api.Broker)
|
|
|
|
haveEnterprise := false
|
|
|
|
|
2017-01-31 19:42:18 +00:00
|
|
|
for _, broker := range *brokerList {
|
2018-09-10 22:03:17 +00:00
|
|
|
broker := broker
|
2016-07-22 16:33:10 +00:00
|
|
|
if cm.isValidBroker(&broker) {
|
2017-01-31 19:42:18 +00:00
|
|
|
validBrokers[broker.CID] = broker
|
2018-09-10 22:03:17 +00:00
|
|
|
if broker.Type == enterpriseType {
|
2016-07-22 16:33:10 +00:00
|
|
|
haveEnterprise = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if haveEnterprise { // eliminate non-enterprise brokers from valid brokers
|
|
|
|
for k, v := range validBrokers {
|
2018-09-10 22:03:17 +00:00
|
|
|
if v.Type != enterpriseType {
|
2016-07-22 16:33:10 +00:00
|
|
|
delete(validBrokers, k)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(validBrokers) == 0 {
|
2017-01-31 19:42:18 +00:00
|
|
|
return nil, fmt.Errorf("found %d broker(s), zero are valid", len(*brokerList))
|
2016-07-22 16:33:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
validBrokerKeys := reflect.ValueOf(validBrokers).MapKeys()
|
|
|
|
selectedBroker := validBrokers[validBrokerKeys[rand.Intn(len(validBrokerKeys))].String()]
|
|
|
|
|
|
|
|
if cm.Debug {
|
|
|
|
cm.Log.Printf("[DEBUG] Selected broker '%s'\n", selectedBroker.Name)
|
|
|
|
}
|
|
|
|
|
|
|
|
return &selectedBroker, nil
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
// Verify broker supports the check type to be used
|
|
|
|
func (cm *CheckManager) brokerSupportsCheckType(checkType CheckTypeType, details *api.BrokerDetail) bool {
|
|
|
|
|
2018-09-10 22:03:17 +00:00
|
|
|
baseType := string(checkType)
|
|
|
|
|
|
|
|
for _, module := range details.Modules {
|
|
|
|
if module == baseType {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if idx := strings.Index(baseType, ":"); idx > 0 {
|
|
|
|
baseType = baseType[0:idx]
|
|
|
|
}
|
|
|
|
|
2016-07-22 16:33:10 +00:00
|
|
|
for _, module := range details.Modules {
|
2018-09-10 22:03:17 +00:00
|
|
|
if module == baseType {
|
2016-07-22 16:33:10 +00:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return false
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
// Is the broker valid (active, supports check type, and reachable)
|
|
|
|
func (cm *CheckManager) isValidBroker(broker *api.Broker) bool {
|
2017-01-31 19:42:18 +00:00
|
|
|
var brokerHost string
|
|
|
|
var brokerPort string
|
2018-09-10 22:03:17 +00:00
|
|
|
|
|
|
|
if broker.Type != "circonus" && broker.Type != "enterprise" {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2016-07-22 16:33:10 +00:00
|
|
|
valid := false
|
2018-09-10 22:03:17 +00:00
|
|
|
|
2016-07-22 16:33:10 +00:00
|
|
|
for _, detail := range broker.Details {
|
2018-09-10 22:03:17 +00:00
|
|
|
detail := detail
|
2016-07-22 16:33:10 +00:00
|
|
|
|
|
|
|
// broker must be active
|
|
|
|
if detail.Status != statusActive {
|
|
|
|
if cm.Debug {
|
|
|
|
cm.Log.Printf("[DEBUG] Broker '%s' is not active.\n", broker.Name)
|
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
// broker must have module loaded for the check type to be used
|
|
|
|
if !cm.brokerSupportsCheckType(cm.checkType, &detail) {
|
|
|
|
if cm.Debug {
|
|
|
|
cm.Log.Printf("[DEBUG] Broker '%s' does not support '%s' checks.\n", broker.Name, cm.checkType)
|
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2016-11-09 00:06:55 +00:00
|
|
|
if detail.ExternalPort != 0 {
|
2017-01-31 19:42:18 +00:00
|
|
|
brokerPort = strconv.Itoa(int(detail.ExternalPort))
|
2016-11-09 00:06:55 +00:00
|
|
|
} else {
|
2018-09-10 22:03:17 +00:00
|
|
|
if detail.Port != nil && *detail.Port != 0 {
|
2017-01-31 19:42:18 +00:00
|
|
|
brokerPort = strconv.Itoa(int(*detail.Port))
|
2016-11-09 00:06:55 +00:00
|
|
|
} else {
|
|
|
|
brokerPort = "43191"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-01-31 19:42:18 +00:00
|
|
|
if detail.ExternalHost != nil && *detail.ExternalHost != "" {
|
|
|
|
brokerHost = *detail.ExternalHost
|
2018-09-10 22:03:17 +00:00
|
|
|
} else if detail.IP != nil && *detail.IP != "" {
|
2017-01-31 19:42:18 +00:00
|
|
|
brokerHost = *detail.IP
|
2016-11-09 00:06:55 +00:00
|
|
|
}
|
|
|
|
|
2018-09-10 22:03:17 +00:00
|
|
|
if brokerHost == "" {
|
|
|
|
cm.Log.Printf("[WARN] Broker '%s' instance %s has no IP or external host set", broker.Name, detail.CN)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2017-01-31 19:42:18 +00:00
|
|
|
if brokerHost == "trap.noit.circonus.net" && brokerPort != "443" {
|
2016-11-09 00:06:55 +00:00
|
|
|
brokerPort = "443"
|
2017-01-31 19:42:18 +00:00
|
|
|
}
|
2016-07-22 16:33:10 +00:00
|
|
|
|
2017-01-31 19:42:18 +00:00
|
|
|
retries := 5
|
|
|
|
for attempt := 1; attempt <= retries; attempt++ {
|
|
|
|
// broker must be reachable and respond within designated time
|
|
|
|
conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", brokerHost, brokerPort), cm.brokerMaxResponseTime)
|
|
|
|
if err == nil {
|
|
|
|
conn.Close()
|
|
|
|
valid = true
|
|
|
|
break
|
|
|
|
}
|
2017-01-31 23:04:38 +00:00
|
|
|
|
2017-01-31 19:42:18 +00:00
|
|
|
cm.Log.Printf("[WARN] Broker '%s' unable to connect, %v. Retrying in 2 seconds, attempt %d of %d.", broker.Name, err, attempt, retries)
|
|
|
|
time.Sleep(2 * time.Second)
|
|
|
|
}
|
2017-01-31 23:04:38 +00:00
|
|
|
|
2017-01-31 19:42:18 +00:00
|
|
|
if valid {
|
|
|
|
if cm.Debug {
|
|
|
|
cm.Log.Printf("[DEBUG] Broker '%s' is valid\n", broker.Name)
|
|
|
|
}
|
|
|
|
break
|
|
|
|
}
|
2016-07-22 16:33:10 +00:00
|
|
|
}
|
|
|
|
return valid
|
|
|
|
}
|