From 99213c2616e42e4f92ff950efdda0207229cfb6a Mon Sep 17 00:00:00 2001
From: Ivan Buymov <ivan@buymov.ru>
Date: Tue, 30 Aug 2022 21:00:48 +0300
Subject: [PATCH] Add retry policy and fix documentation for Cassandra storage
 backend (#10467)

* add simple_retry policy and initial_connection_timeout options,
fix docs for connection_timeout

* Cassandra: policy fix - added changelog.

Co-authored-by: Mehdi Ahmadi <aphorise@gmail.com>
---
 changelog/10467.txt                           |  3 +++
 physical/cassandra/cassandra.go               | 20 +++++++++++++++++--
 physical/cassandra/cassandra_test.go          |  7 +++++--
 .../docs/configuration/storage/cassandra.mdx  | 11 ++++++++--
 4 files changed, 35 insertions(+), 6 deletions(-)
 create mode 100644 changelog/10467.txt

diff --git a/changelog/10467.txt b/changelog/10467.txt
new file mode 100644
index 000000000..411bbf78e
--- /dev/null
+++ b/changelog/10467.txt
@@ -0,0 +1,3 @@
+```release-note:improvement
+storage/cassandra: tuning parameters for clustered environments `connection_timeout`, `initial_connection_timeout`, `simple_retry_policy_retries`.
+```
diff --git a/physical/cassandra/cassandra.go b/physical/cassandra/cassandra.go
index f20b99205..84c2ab149 100644
--- a/physical/cassandra/cassandra.go
+++ b/physical/cassandra/cassandra.go
@@ -102,6 +102,14 @@ func NewCassandraBackend(conf map[string]string, logger log.Logger) (physical.Ba
 	cluster.Port = port
 	cluster.Keyspace = keyspace
 
+	if retryCountStr, ok := conf["simple_retry_policy_retries"]; ok {
+		retryCount, err := strconv.Atoi(retryCountStr)
+		if err != nil || retryCount <= 0 {
+			return nil, fmt.Errorf("'simple_retry_policy_retries' must be a positive integer")
+		}
+		cluster.RetryPolicy = &gocql.SimpleRetryPolicy{NumRetries: retryCount}
+	}
+
 	cluster.ProtoVersion = 2
 	if protoVersionStr, ok := conf["protocol_version"]; ok {
 		protoVersion, err := strconv.Atoi(protoVersionStr)
@@ -122,10 +130,18 @@ func NewCassandraBackend(conf map[string]string, logger log.Logger) (physical.Ba
 		cluster.Authenticator = authenticator
 	}
 
+	if initialConnectionTimeoutStr, ok := conf["initial_connection_timeout"]; ok {
+		initialConnectionTimeout, err := strconv.Atoi(initialConnectionTimeoutStr)
+		if err != nil || initialConnectionTimeout <= 0 {
+			return nil, fmt.Errorf("'initial_connection_timeout' must be a positive integer")
+		}
+		cluster.ConnectTimeout = time.Duration(initialConnectionTimeout) * time.Second
+	}
+
 	if connTimeoutStr, ok := conf["connection_timeout"]; ok {
 		connectionTimeout, err := strconv.Atoi(connTimeoutStr)
-		if err != nil {
-			return nil, fmt.Errorf("'connection_timeout' must be an integer")
+		if err != nil || connectionTimeout <= 0 {
+			return nil, fmt.Errorf("'connection_timeout' must be a positive integer")
 		}
 		cluster.Timeout = time.Duration(connectionTimeout) * time.Second
 	}
diff --git a/physical/cassandra/cassandra_test.go b/physical/cassandra/cassandra_test.go
index ea1e4e129..e9fe7bc05 100644
--- a/physical/cassandra/cassandra_test.go
+++ b/physical/cassandra/cassandra_test.go
@@ -25,8 +25,11 @@ func TestCassandraBackend(t *testing.T) {
 	// Run vault tests
 	logger := logging.NewVaultLogger(log.Debug)
 	b, err := NewCassandraBackend(map[string]string{
-		"hosts":            host.ConnectionURL(),
-		"protocol_version": "3",
+		"hosts":                       host.ConnectionURL(),
+		"protocol_version":            "3",
+		"connection_timeout":          "5",
+		"initial_connection_timeout":  "5",
+		"simple_retry_policy_retries": "3",
 	}, logger)
 	if err != nil {
 		t.Fatalf("Failed to create new backend: %v", err)
diff --git a/website/content/docs/configuration/storage/cassandra.mdx b/website/content/docs/configuration/storage/cassandra.mdx
index 888d3b208..421d17af2 100644
--- a/website/content/docs/configuration/storage/cassandra.mdx
+++ b/website/content/docs/configuration/storage/cassandra.mdx
@@ -69,8 +69,15 @@ CREATE TABLE "vault"."entries" (
 - `password` `(string: "")` – Password to use when authenticating with the
   Cassandra hosts.
 
-- `connection_timeout` `(int: 0)` - A timeout in seconds to wait until a
-  connection is established with the Cassandra hosts.
+- `initial_connection_timeout` `(int: 0)` - A timeout in seconds to wait until an initial connection is established
+  with the Cassandra hosts. If not set, default value from Cassandra driver(gocql) will be used - 600ms
+
+- `connection_timeout` `(int: 0)` - A timeout in seconds for each query.
+  If not set, default value from Cassandra driver(gocql) will be used - 600ms
+
+- `simple_retry_policy_retries` `(int: 0)` - Useful for Cassandra cluster with several nodes.
+  If current master node is down request will be retried on the next node `simple_retry_policy_retries`
+  times, and the client won't get an error.
 
 - `tls` `(int: 0)` – If `1`, indicates the connection with the Cassandra hosts
   should use TLS.