diff --git a/GNUmakefile b/GNUmakefile
index 4742cd878..c24df9308 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -330,15 +330,15 @@ ifeq ("$(CIRCLECI)","true")
 # Run in CI
 	gotestsum --format=short-verbose --junitfile "$(TEST_RESULTS_DIR)/gotestsum-report.xml" -- -cover -coverprofile=coverage.txt ./agent/connect/ca
 # Run leader tests that require Vault
-	gotestsum --format=short-verbose --junitfile "$(TEST_RESULTS_DIR)/gotestsum-report-leader.xml" -- -cover -coverprofile=coverage-leader.txt -run '.*_Vault_' ./agent/consul
+	gotestsum --format=short-verbose --junitfile "$(TEST_RESULTS_DIR)/gotestsum-report-leader.xml" -- -cover -coverprofile=coverage-leader.txt -run Vault ./agent/consul
 # Run agent tests that require Vault
-	gotestsum --format=short-verbose --junitfile "$(TEST_RESULTS_DIR)/gotestsum-report-agent.xml" -- -cover -coverprofile=coverage-agent.txt -run '.*_Vault_' ./agent
+	gotestsum --format=short-verbose --junitfile "$(TEST_RESULTS_DIR)/gotestsum-report-agent.xml" -- -cover -coverprofile=coverage-agent.txt -run Vault ./agent
 else
 # Run locally
 	@echo "Running /agent/connect/ca tests in verbose mode"
 	@go test -v ./agent/connect/ca
-	@go test -v ./agent/consul -run '.*_Vault_'
-	@go test -v ./agent -run '.*_Vault_'
+	@go test -v ./agent/consul -run Vault
+	@go test -v ./agent -run Vault
 endif
 
 proto: $(PROTOGOFILES) $(PROTOGOBINFILES)
diff --git a/agent/connect/ca/testing.go b/agent/connect/ca/testing.go
index 92673dcb9..97c28871d 100644
--- a/agent/connect/ca/testing.go
+++ b/agent/connect/ca/testing.go
@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"strings"
 	"sync"
 
 	"github.com/hashicorp/go-hclog"
@@ -86,6 +87,13 @@ func TestConsulProvider(t testing.T, d ConsulProviderStateDelegate) *ConsulProvi
 // These tests may be skipped in CI. They are run as part of a separate
 // integration test suite.
 func SkipIfVaultNotPresent(t testing.T) {
+	// Try to safeguard against tests that will never run in CI.
+	// This substring should match the pattern used by the
+	// test-connect-ca-providers CI job.
+	if !strings.Contains(t.Name(), "Vault") {
+		t.Fatalf("test name must contain Vault, otherwise CI will never run it")
+	}
+
 	vaultBinaryName := os.Getenv("VAULT_BINARY_NAME")
 	if vaultBinaryName == "" {
 		vaultBinaryName = "vault"