From b907c4611d2f35e130fef09e47521298ad3d4165 Mon Sep 17 00:00:00 2001 From: Frank Schroeder Date: Thu, 26 Oct 2017 11:56:08 +0200 Subject: [PATCH 1/6] docker: ignore "connection reset by peer" The Docker agent closes the connection during read after we have read the body. This causes a "connection reset by peer" even though the command was successful. We ignore that error here since we got the correct status code and a response body. --- agent/checks/docker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/checks/docker.go b/agent/checks/docker.go index 28c42a757..46ff4adf0 100644 --- a/agent/checks/docker.go +++ b/agent/checks/docker.go @@ -155,7 +155,7 @@ func (c *DockerClient) StartExec(containerID, execID string) (*circbuf.Buffer, e uri := fmt.Sprintf("/exec/%s/start", execID) b, code, err := c.call("POST", uri, data) switch { - case err != nil: + case err != nil && !strings.Contains(err.Error(), "connection reset by peer"): return nil, fmt.Errorf("start exec failed for container %s: %s", containerID, err) case code == 200: return b, nil From b1a5a6b64d678e8b9d387a432e3abee893f90631 Mon Sep 17 00:00:00 2001 From: Frank Schroeder Date: Thu, 26 Oct 2017 11:56:36 +0200 Subject: [PATCH 2/6] docker: make sure to log the error when we fall through --- agent/checks/docker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/checks/docker.go b/agent/checks/docker.go index 46ff4adf0..809a07252 100644 --- a/agent/checks/docker.go +++ b/agent/checks/docker.go @@ -164,7 +164,7 @@ func (c *DockerClient) StartExec(containerID, execID string) (*circbuf.Buffer, e case code == 409: return nil, fmt.Errorf("start exec failed since container %s is paused or stopped", containerID) default: - return nil, fmt.Errorf("start exec failed for container %s with status %d: %s", containerID, code, b) + return nil, fmt.Errorf("start exec failed for container %s with status %d: body: %s err: %s", containerID, code, b, err) } } From 0a9d2a367ee21c1968b1833ce2f674352cefe717 Mon Sep 17 00:00:00 2001 From: Frank Schroeder Date: Thu, 26 Oct 2017 11:56:54 +0200 Subject: [PATCH 3/6] docker: do not alloc a tty since this is not interactive --- agent/checks/docker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/checks/docker.go b/agent/checks/docker.go index 809a07252..c3cddd646 100644 --- a/agent/checks/docker.go +++ b/agent/checks/docker.go @@ -151,7 +151,7 @@ func (c *DockerClient) CreateExec(containerID string, cmd []string) (string, err } func (c *DockerClient) StartExec(containerID, execID string) (*circbuf.Buffer, error) { - data := struct{ Detach, Tty bool }{Detach: false, Tty: true} + data := struct{ Detach, Tty bool }{Detach: false, Tty: false} uri := fmt.Sprintf("/exec/%s/start", execID) b, code, err := c.call("POST", uri, data) switch { From bf98779d84ba508ebc3c0dbaad3296778884f311 Mon Sep 17 00:00:00 2001 From: Frank Schroeder Date: Thu, 26 Oct 2017 11:57:18 +0200 Subject: [PATCH 4/6] docker: close idle connections on stop --- agent/checks/check.go | 1 + agent/checks/docker.go | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/agent/checks/check.go b/agent/checks/check.go index 446521021..87e2c904b 100644 --- a/agent/checks/check.go +++ b/agent/checks/check.go @@ -573,6 +573,7 @@ func (c *CheckDocker) Stop() { } func (c *CheckDocker) run() { + defer c.Client.Close() firstWait := lib.RandomStagger(c.Interval) next := time.After(firstWait) for { diff --git a/agent/checks/docker.go b/agent/checks/docker.go index c3cddd646..c703cb0cb 100644 --- a/agent/checks/docker.go +++ b/agent/checks/docker.go @@ -54,6 +54,13 @@ func NewDockerClient(host string, maxbuf int64) (*DockerClient, error) { }, nil } +func (c *DockerClient) Close() error { + if t, ok := c.client.Transport.(*http.Transport); ok { + t.CloseIdleConnections() + } + return nil +} + func (c *DockerClient) Host() string { return c.host } From 7d05e557344415fdaaed4df50effcd0b543247dd Mon Sep 17 00:00:00 2001 From: Frank Schroeder Date: Thu, 26 Oct 2017 12:03:07 +0200 Subject: [PATCH 5/6] docker: stop previous check on replace --- agent/agent.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/agent/agent.go b/agent/agent.go index 54628bca1..0dadd7025 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1776,6 +1776,9 @@ func (a *Agent) AddCheck(check *structs.HealthCheck, chkType *structs.CheckType, Logger: a.logger, Client: a.dockerClient, } + if prev := a.checkDockers[check.CheckID]; prev != nil { + prev.Stop() + } dockerCheck.Start() a.checkDockers[check.CheckID] = dockerCheck From 712447026f8d1524a9359e7daa2427fb39ecb09e Mon Sep 17 00:00:00 2001 From: Frank Schroeder Date: Thu, 26 Oct 2017 12:14:19 +0200 Subject: [PATCH 6/6] docker: add comment about "connection reset by peer" error --- agent/checks/docker.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/agent/checks/docker.go b/agent/checks/docker.go index c703cb0cb..3644a6852 100644 --- a/agent/checks/docker.go +++ b/agent/checks/docker.go @@ -162,6 +162,12 @@ func (c *DockerClient) StartExec(containerID, execID string) (*circbuf.Buffer, e uri := fmt.Sprintf("/exec/%s/start", execID) b, code, err := c.call("POST", uri, data) switch { + // todo(fs): https://github.com/hashicorp/consul/pull/3621 + // todo(fs): for some reason the docker agent closes the connection during the + // todo(fs): io.Copy call in c.call which causes a "connection reset by peer" error + // todo(fs): even though both body and status code have been received. My current is + // todo(fs): that the docker agent closes this prematurely but I don't understand why. + // todo(fs): the code below ignores this error. case err != nil && !strings.Contains(err.Error(), "connection reset by peer"): return nil, fmt.Errorf("start exec failed for container %s: %s", containerID, err) case code == 200: