Add retry policy and fix documentation for Cassandra storage backend (#10467)

* add simple_retry policy and initial_connection_timeout options,
fix docs for connection_timeout

* Cassandra: policy fix - added changelog.

Co-authored-by: Mehdi Ahmadi <aphorise@gmail.com>

changelog/10467.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+storage/cassandra: tuning parameters for clustered environments `connection_timeout`, `initial_connection_timeout`, `simple_retry_policy_retries`.
+```

physical/cassandra/cassandra.go

@@ -102,6 +102,14 @@ func NewCassandraBackend(conf map[string]string, logger log.Logger) (physical.Ba
 	cluster.Port = port
 	cluster.Keyspace = keyspace
 
+	if retryCountStr, ok := conf["simple_retry_policy_retries"]; ok {
+		retryCount, err := strconv.Atoi(retryCountStr)
+		if err != nil || retryCount <= 0 {
+			return nil, fmt.Errorf("'simple_retry_policy_retries' must be a positive integer")
+		}
+		cluster.RetryPolicy = &gocql.SimpleRetryPolicy{NumRetries: retryCount}
+	}
+
 	cluster.ProtoVersion = 2
 	if protoVersionStr, ok := conf["protocol_version"]; ok {
 		protoVersion, err := strconv.Atoi(protoVersionStr)
@@ -122,10 +130,18 @@ func NewCassandraBackend(conf map[string]string, logger log.Logger) (physical.Ba
 		cluster.Authenticator = authenticator
 	}
 
+	if initialConnectionTimeoutStr, ok := conf["initial_connection_timeout"]; ok {
+		initialConnectionTimeout, err := strconv.Atoi(initialConnectionTimeoutStr)
+		if err != nil || initialConnectionTimeout <= 0 {
+			return nil, fmt.Errorf("'initial_connection_timeout' must be a positive integer")
+		}
+		cluster.ConnectTimeout = time.Duration(initialConnectionTimeout) * time.Second
+	}
+
 	if connTimeoutStr, ok := conf["connection_timeout"]; ok {
 		connectionTimeout, err := strconv.Atoi(connTimeoutStr)
-		if err != nil {
+		if err != nil || connectionTimeout <= 0 {
-			return nil, fmt.Errorf("'connection_timeout' must be an integer")
+			return nil, fmt.Errorf("'connection_timeout' must be a positive integer")
 		}
 		cluster.Timeout = time.Duration(connectionTimeout) * time.Second
 	}

physical/cassandra/cassandra_test.go

@@ -25,8 +25,11 @@ func TestCassandraBackend(t *testing.T) {
 	// Run vault tests
 	logger := logging.NewVaultLogger(log.Debug)
 	b, err := NewCassandraBackend(map[string]string{
-		"hosts":            host.ConnectionURL(),
+		"hosts":                       host.ConnectionURL(),
-		"protocol_version": "3",
+		"protocol_version":            "3",
+		"connection_timeout":          "5",
+		"initial_connection_timeout":  "5",
+		"simple_retry_policy_retries": "3",
 	}, logger)
 	if err != nil {
 		t.Fatalf("Failed to create new backend: %v", err)

website/content/docs/configuration/storage/cassandra.mdx

@@ -69,8 +69,15 @@ CREATE TABLE "vault"."entries" (
 - `password` `(string: "")` – Password to use when authenticating with the
   Cassandra hosts.
 
-- `connection_timeout` `(int: 0)` - A timeout in seconds to wait until a
+- `initial_connection_timeout` `(int: 0)` - A timeout in seconds to wait until an initial connection is established
-  connection is established with the Cassandra hosts.
+  with the Cassandra hosts. If not set, default value from Cassandra driver(gocql) will be used - 600ms
+
+- `connection_timeout` `(int: 0)` - A timeout in seconds for each query.
+  If not set, default value from Cassandra driver(gocql) will be used - 600ms
+
+- `simple_retry_policy_retries` `(int: 0)` - Useful for Cassandra cluster with several nodes.
+  If current master node is down request will be retried on the next node `simple_retry_policy_retries`
+  times, and the client won't get an error.
 
 - `tls` `(int: 0)` – If `1`, indicates the connection with the Cassandra hosts
   should use TLS.