From a9a04141a3c46e669f3232db49167a956823533b Mon Sep 17 00:00:00 2001 From: James Rasell Date: Thu, 5 Aug 2021 11:27:35 +0200 Subject: [PATCH] consul/connect: avoid warn messages on connect proxy errors When creating a TCP proxy bridge for Connect tasks, we are at the mercy of either end for managing the connection state. For long lived gRPC connections the proxy could reasonably expect to stay open until the context was cancelled. For the HTTP connections used by connect native tasks, we experience connection disconnects. The proxy gets recreated as needed on follow up requests, however we also emit a WARN log when the connection is broken. This PR lowers the WARN to a TRACE, because these disconnects are to be expected. Ideally we would be able to proxy at the HTTP layer, however Consul or the connect native task could be configured to expect mTLS, preventing Nomad from MiTM the requests. We also can't mange the proxy lifecycle more intelligently, because we have no control over the HTTP client or server and how they wish to manage connection state. What we have now works, it's just noisy. Fixes #10933 --- .changelog/10951.txt | 3 +++ client/allocrunner/consul_grpc_sock_hook.go | 15 +++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) create mode 100644 .changelog/10951.txt diff --git a/.changelog/10951.txt b/.changelog/10951.txt new file mode 100644 index 000000000..0ca2b4401 --- /dev/null +++ b/.changelog/10951.txt @@ -0,0 +1,3 @@ +```release-note:improvement +consul/connect: Reduced the noise of log messages emitted for connect native tasks +``` diff --git a/client/allocrunner/consul_grpc_sock_hook.go b/client/allocrunner/consul_grpc_sock_hook.go index 91014fedc..07fe6ef47 100644 --- a/client/allocrunner/consul_grpc_sock_hook.go +++ b/client/allocrunner/consul_grpc_sock_hook.go @@ -313,10 +313,10 @@ func proxyConn(ctx context.Context, logger hclog.Logger, destAddr string, conn n defer cancel() n, err := io.Copy(dest, conn) if ctx.Err() == nil && err != nil { - logger.Warn("error proxying to Consul", "error", err, "dest", destAddr, - "src_local", conn.LocalAddr(), "src_remote", conn.RemoteAddr(), - "bytes", n, - ) + // expect disconnects when proxying http + logger.Trace("error message received proxying to Consul", + "msg", err, "dest", destAddr, "src_local", conn.LocalAddr(), + "src_remote", conn.RemoteAddr(), "bytes", n) return } logger.Trace("proxy to Consul complete", @@ -332,10 +332,9 @@ func proxyConn(ctx context.Context, logger hclog.Logger, destAddr string, conn n defer cancel() n, err := io.Copy(conn, dest) if ctx.Err() == nil && err != nil { - logger.Warn("error proxying from Consul", "error", err, "dest", destAddr, - "src_local", conn.LocalAddr(), "src_remote", conn.RemoteAddr(), - "bytes", n, - ) + logger.Trace("error message received proxying from Consul", + "msg", err, "dest", destAddr, "src_local", conn.LocalAddr(), + "src_remote", conn.RemoteAddr(), "bytes", n) return } logger.Trace("proxy from Consul complete",