From 24aa32c5037e94951d11b2f39a4a7511e8a6b0c8 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 14 May 2020 15:44:03 -0400 Subject: [PATCH] csi: use a blocking initial connection with timeout The plugin supervisor lazily connects to plugins, but this means we only get "Unavailable" back from the gRPC call in cases where the plugin can never be reached (for example, if the Nomad client has the wrong permissions for the socket). This changeset improves the operator experience by switching to a blocking `DialWithContext`. It eagerly connects so that we can validate the connection is real and get a "failed to open" error in case where Nomad can't establish the initial connection. --- client/allocrunner/taskrunner/plugin_supervisor_hook.go | 2 +- plugins/csi/client.go | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/client/allocrunner/taskrunner/plugin_supervisor_hook.go b/client/allocrunner/taskrunner/plugin_supervisor_hook.go index 7d730ead7..b95daa857 100644 --- a/client/allocrunner/taskrunner/plugin_supervisor_hook.go +++ b/client/allocrunner/taskrunner/plugin_supervisor_hook.go @@ -335,10 +335,10 @@ func (h *csiPluginSupervisorHook) supervisorLoopOnce(ctx context.Context, socket } client, err := csi.NewClient(socketPath, h.logger.Named("csi_client").With("plugin.name", h.task.CSIPluginConfig.ID, "plugin.type", h.task.CSIPluginConfig.Type)) - defer client.Close() if err != nil { return false, fmt.Errorf("failed to create csi client: %v", err) } + defer client.Close() healthy, err := client.PluginProbe(ctx) if err != nil { diff --git a/plugins/csi/client.go b/plugins/csi/client.go index 99c0cad38..0305b04d9 100644 --- a/plugins/csi/client.go +++ b/plugins/csi/client.go @@ -114,8 +114,12 @@ func NewClient(addr string, logger hclog.Logger) (CSIPlugin, error) { } func newGrpcConn(addr string, logger hclog.Logger) (*grpc.ClientConn, error) { - conn, err := grpc.Dial( + ctx, cancel := context.WithTimeout(context.Background(), time.Second*1) + defer cancel() + conn, err := grpc.DialContext( + ctx, addr, + grpc.WithBlock(), grpc.WithInsecure(), grpc.WithUnaryInterceptor(logging.UnaryClientInterceptor(logger)), grpc.WithStreamInterceptor(logging.StreamClientInterceptor(logger)),