Merge pull request #2322 from hashicorp/pr-2321-slackpad

Adds a configurable timeout for DNS recursor client.

command/agent/config.go

@@ -109,6 +109,13 @@ type DNSConfig struct {
 	// compressed. In Consul 0.7 this was turned on by default and this
 	// config was added as an opt-out.
 	DisableCompression bool `mapstructure:"disable_compression"`
+
+	// RecursorTimeout specifies the timeout in seconds
+	// for Consul's internal dns client used for recursion.
+	// This value is used for the connection, read and write timeout.
+	// Default: 2s
+	RecursorTimeout			time.Duration `mapstructure:"-"`
+	RecursorTimeoutRaw string	`mapstructure:"recursor_timeout" json:"-"`
 }
 
 // Performance is used to tune the performance of Consul's subsystems.
@@ -645,9 +652,10 @@ func DefaultConfig() *Config {
 			Server:  8300,
 		},
 		DNSConfig: DNSConfig{
-			AllowStale:     Bool(true),
-			UDPAnswerLimit: 3,
-			MaxStale:       5 * time.Second,
+			AllowStale:      Bool(true),
+			UDPAnswerLimit:  3,
+			MaxStale:        5 * time.Second,
+			RecursorTimeout: 2 * time.Second,
 		},
 		Telemetry: Telemetry{
 			StatsitePrefix: "consul",
@@ -840,6 +848,14 @@ func DecodeConfig(r io.Reader) (*Config, error) {
 		result.DNSConfig.MaxStale = dur
 	}
 
+	if raw := result.DNSConfig.RecursorTimeoutRaw; raw != "" {
+		dur, err := time.ParseDuration(raw)
+		if err != nil {
+			return nil, fmt.Errorf("RecursorTimeout invalid: %v", err)
+		}
+		result.DNSConfig.RecursorTimeout = dur
+	}
+
 	if len(result.DNSConfig.ServiceTTLRaw) != 0 {
 		if result.DNSConfig.ServiceTTL == nil {
 			result.DNSConfig.ServiceTTL = make(map[string]time.Duration)
@@ -1355,6 +1371,9 @@ func MergeConfig(a, b *Config) *Config {
 	if b.DNSConfig.DisableCompression {
 		result.DNSConfig.DisableCompression = true
 	}
+	if b.DNSConfig.RecursorTimeout != 0 {
+		result.DNSConfig.RecursorTimeout = b.DNSConfig.RecursorTimeout
+	}
 	if b.CheckUpdateIntervalRaw != "" || b.CheckUpdateInterval != 0 {
 		result.CheckUpdateInterval = b.CheckUpdateInterval
 	}

command/agent/config_test.go

@@ -544,7 +544,7 @@ func TestDecodeConfig(t *testing.T) {
 	}
 
 	// DNS node ttl, max stale
-	input = `{"dns_config": {"allow_stale": false, "enable_truncate": false, "max_stale": "15s", "node_ttl": "5s", "only_passing": true, "udp_answer_limit": 6}}`
+	input = `{"dns_config": {"allow_stale": false, "enable_truncate": false, "max_stale": "15s", "node_ttl": "5s", "only_passing": true, "udp_answer_limit": 6, "recursor_timeout": "7s"}}`
 	config, err = DecodeConfig(bytes.NewReader([]byte(input)))
 	if err != nil {
 		t.Fatalf("err: %s", err)
@@ -568,6 +568,9 @@ func TestDecodeConfig(t *testing.T) {
 	if config.DNSConfig.UDPAnswerLimit != 6 {
 		t.Fatalf("bad: %#v", config)
 	}
+	if config.DNSConfig.RecursorTimeout != 7*time.Second {
+		t.Fatalf("bad: %#v", config)
+	}
 
 	// DNS service ttl
 	input = `{"dns_config": {"service_ttl": {"*": "1s", "api": "10s", "web": "30s"}}}`
@@ -1416,7 +1419,8 @@ func TestMergeConfig(t *testing.T) {
 			ServiceTTL: map[string]time.Duration{
 				"api": 10 * time.Second,
 			},
-			UDPAnswerLimit: 4,
+			UDPAnswerLimit:  4,
+			RecursorTimeout: 30 * time.Second,
 		},
 		Domain:           "other",
 		LogLevel:         "info",

command/agent/dns.go

@@ -842,7 +842,7 @@ func (d *DNSServer) handleRecurse(resp dns.ResponseWriter, req *dns.Msg) {
 	}
 
 	// Recursively resolve
-	c := &dns.Client{Net: network}
+	c := &dns.Client{Net: network, Timeout: d.config.RecursorTimeout}
 	var r *dns.Msg
 	var rtt time.Duration
 	var err error
@@ -887,7 +887,7 @@ func (d *DNSServer) resolveCNAME(name string) []dns.RR {
 	m.SetQuestion(name, dns.TypeA)
 
 	// Make a DNS lookup request
-	c := &dns.Client{Net: "udp"}
+	c := &dns.Client{Net: "udp", Timeout: d.config.RecursorTimeout}
 	var r *dns.Msg
 	var rtt time.Duration
 	var err error

command/agent/dns_test.go

@@ -1400,6 +1400,47 @@ func TestDNS_Recurse(t *testing.T) {
 	}
 }
 
+func TestDNS_RecursorTimeout(t *testing.T) {
+	serverClientTimeout := 3 * time.Second
+	testClientTimeout := serverClientTimeout + 5*time.Second
+
+	dir, srv := makeDNSServerConfig(t, func(c *Config) {
+		c.DNSRecursor = "10.255.255.1" // host must cause a connection|read|write timeout
+	}, func(c *DNSConfig) {
+		c.RecursorTimeout = serverClientTimeout
+	})
+	defer os.RemoveAll(dir)
+	defer srv.agent.Shutdown()
+
+	m := new(dns.Msg)
+	m.SetQuestion("apple.com.", dns.TypeANY)
+
+	// This client calling the server under test must have a longer timeout than the one we set internally
+	c := &dns.Client{Timeout: testClientTimeout}
+	addr, _ := srv.agent.config.ClientListener("", srv.agent.config.Ports.DNS)
+
+	start := time.Now()
+	in, _, err := c.Exchange(m, addr.String())
+
+	duration := time.Now().Sub(start)
+
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if len(in.Answer) != 0 {
+		t.Fatalf("Bad: %#v", in)
+	}
+	if in.Rcode != dns.RcodeServerFailure {
+		t.Fatalf("Bad: %#v", in)
+	}
+
+	if duration < serverClientTimeout {
+		t.Fatalf("Expected the call to return after at least %f seconds but lasted only %f", serverClientTimeout.Seconds(), duration.Seconds())
+	}
+
+}
+
 func TestDNS_ServiceLookup_FilterCritical(t *testing.T) {
 	dir, srv := makeDNSServer(t)
 	defer os.RemoveAll(dir)

website/source/docs/agent/options.html.markdown

@@ -517,6 +517,10 @@ Consul will not enable TLS for the HTTP API unless the `https` port has been ass
   are considered. For example, if a node has a health check that is critical then all services on
   that node will be excluded because they are also considered critical.
 
+  * <a name="recursor_timeout"></a><a href="#recursor_timeout">`recursor_timeout`</a> Timeout used
+  by Consul when recursively querying an upstream DNS server. See <a href="#recursors">`recursors`</a>
+  for more details. Default is 2s.
+
   * <a name="disable_compression"></a><a href="#disable_compression">`disable_compression`</a> If
   set to true, DNS responses will not be compressed. Compression was added and enabled by default
   in Consul 0.7.
@@ -636,7 +640,7 @@ Consul will not enable TLS for the HTTP API unless the `https` port has been ass
 
 * <a name="recursors"></a><a href="#recursors">`recursors`</a> This flag provides addresses of
   upstream DNS servers that are used to recursively resolve queries if they are not inside the service
-  domain for consul. For example, a node can use Consul directly as a DNS server, and if the record is
+  domain for Consul. For example, a node can use Consul directly as a DNS server, and if the record is
   outside of the "consul." domain, the query will be resolved upstream.
 
 * <a name="rejoin_after_leave"></a><a href="#rejoin_after_leave">`rejoin_after_leave`</a> Equivalent