open-nomad/nomad/vault_test.go

868 lines
22 KiB
Go

package nomad
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"reflect"
"strings"
"testing"
"time"
"golang.org/x/time/rate"
"github.com/hashicorp/nomad/nomad/mock"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/nomad/structs/config"
"github.com/hashicorp/nomad/testutil"
vapi "github.com/hashicorp/vault/api"
)
const (
// authPolicy is a policy that allows token creation operations
authPolicy = `path "auth/token/create/test" {
capabilities = ["create", "update"]
}
path "auth/token/lookup/*" {
capabilities = ["read"]
}
path "auth/token/roles/test" {
capabilities = ["read"]
}
path "/auth/token/revoke-accessor/*" {
capabilities = ["update"]
}
`
)
func TestVaultClient_BadConfig(t *testing.T) {
conf := &config.VaultConfig{}
logger := log.New(os.Stderr, "", log.LstdFlags)
// Should be no error since Vault is not enabled
_, err := NewVaultClient(nil, logger, nil)
if err == nil || !strings.Contains(err.Error(), "valid") {
t.Fatalf("expected config error: %v", err)
}
tr := true
conf.Enabled = &tr
_, err = NewVaultClient(conf, logger, nil)
if err == nil || !strings.Contains(err.Error(), "token must be set") {
t.Fatalf("Expected token unset error: %v", err)
}
conf.Token = "123"
_, err = NewVaultClient(conf, logger, nil)
if err == nil || !strings.Contains(err.Error(), "address must be set") {
t.Fatalf("Expected address unset error: %v", err)
}
}
// Test that the Vault Client can establish a connection even if it is started
// before Vault is available.
func TestVaultClient_EstablishConnection(t *testing.T) {
v := testutil.NewTestVault(t)
defer v.Stop()
logger := log.New(os.Stderr, "", log.LstdFlags)
v.Config.ConnectionRetryIntv = 100 * time.Millisecond
client, err := NewVaultClient(v.Config, logger, nil)
if err != nil {
t.Fatalf("failed to build vault client: %v", err)
}
defer client.Stop()
// Sleep a little while and check that no connection has been established.
time.Sleep(100 * time.Duration(testutil.TestMultiplier()) * time.Millisecond)
if established, _ := client.ConnectionEstablished(); established {
t.Fatalf("ConnectionEstablished() returned true before Vault server started")
}
// Start Vault
v.Start()
waitForConnection(client, t)
}
func TestVaultClient_ValidateRole(t *testing.T) {
v := testutil.NewTestVault(t).Start()
defer v.Stop()
// Set the configs token in a new test role
data := map[string]interface{}{
"allowed_policies": "default,root",
"orphan": true,
"renewable": true,
"explicit_max_ttl": 10,
}
v.Config.Token = testVaultRoleAndToken(v, t, data)
logger := log.New(os.Stderr, "", log.LstdFlags)
v.Config.ConnectionRetryIntv = 100 * time.Millisecond
client, err := NewVaultClient(v.Config, logger, nil)
if err != nil {
t.Fatalf("failed to build vault client: %v", err)
}
defer client.Stop()
// Wait for an error
var conn bool
var connErr error
testutil.WaitForResult(func() (bool, error) {
conn, connErr = client.ConnectionEstablished()
if conn {
return false, fmt.Errorf("Should not connect")
}
if connErr == nil {
return false, fmt.Errorf("expect an error")
}
return true, nil
}, func(err error) {
t.Fatalf("bad: %v", err)
})
errStr := connErr.Error()
if !strings.Contains(errStr, "not allow orphans") {
t.Fatalf("Expect orphan error")
}
if !strings.Contains(errStr, "explicit max ttl") {
t.Fatalf("Expect explicit max ttl error")
}
}
func TestVaultClient_SetActive(t *testing.T) {
v := testutil.NewTestVault(t).Start()
defer v.Stop()
logger := log.New(os.Stderr, "", log.LstdFlags)
client, err := NewVaultClient(v.Config, logger, nil)
if err != nil {
t.Fatalf("failed to build vault client: %v", err)
}
defer client.Stop()
waitForConnection(client, t)
// Do a lookup and expect an error about not being active
_, err = client.LookupToken(context.Background(), "123")
if err == nil || !strings.Contains(err.Error(), "not active") {
t.Fatalf("Expected not-active error: %v", err)
}
client.SetActive(true)
// Do a lookup of ourselves
_, err = client.LookupToken(context.Background(), v.RootToken)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
}
// Test that we can update the config and things keep working
func TestVaultClient_SetConfig(t *testing.T) {
v := testutil.NewTestVault(t).Start()
defer v.Stop()
v2 := testutil.NewTestVault(t).Start()
defer v2.Stop()
// Set the configs token in a new test role
v2.Config.Token = defaultTestVaultRoleAndToken(v2, t, 20)
logger := log.New(os.Stderr, "", log.LstdFlags)
client, err := NewVaultClient(v.Config, logger, nil)
if err != nil {
t.Fatalf("failed to build vault client: %v", err)
}
defer client.Stop()
waitForConnection(client, t)
if client.tokenData == nil || len(client.tokenData.Policies) != 1 {
t.Fatalf("unexpected token: %v", client.tokenData)
}
// Update the config
if err := client.SetConfig(v2.Config); err != nil {
t.Fatalf("SetConfig failed: %v", err)
}
waitForConnection(client, t)
if client.tokenData == nil || len(client.tokenData.Policies) != 2 {
t.Fatalf("unexpected token: %v", client.tokenData)
}
}
// defaultTestVaultRoleAndToken creates a test Vault role and returns a token
// created in that role
func defaultTestVaultRoleAndToken(v *testutil.TestVault, t *testing.T, rolePeriod int) string {
d := make(map[string]interface{}, 2)
d["allowed_policies"] = "auth"
d["period"] = rolePeriod
return testVaultRoleAndToken(v, t, d)
}
// testVaultRoleAndToken creates a test Vault role with the specified data and
// returns a token created in that role
func testVaultRoleAndToken(v *testutil.TestVault, t *testing.T, data map[string]interface{}) string {
// Build the auth policy
sys := v.Client.Sys()
if err := sys.PutPolicy("auth", authPolicy); err != nil {
t.Fatalf("failed to create auth policy: %v", err)
}
// Build a role
l := v.Client.Logical()
l.Write("auth/token/roles/test", data)
// Create a new token with the role
a := v.Client.Auth().Token()
req := vapi.TokenCreateRequest{}
s, err := a.CreateWithRole(&req, "test")
if err != nil {
t.Fatalf("failed to create child token: %v", err)
}
// Get the client token
if s == nil || s.Auth == nil {
t.Fatalf("bad secret response: %+v", s)
}
return s.Auth.ClientToken
}
func TestVaultClient_RenewalLoop(t *testing.T) {
v := testutil.NewTestVault(t).Start()
defer v.Stop()
// Set the configs token in a new test role
v.Config.Token = defaultTestVaultRoleAndToken(v, t, 5)
// Start the client
logger := log.New(os.Stderr, "", log.LstdFlags)
client, err := NewVaultClient(v.Config, logger, nil)
if err != nil {
t.Fatalf("failed to build vault client: %v", err)
}
defer client.Stop()
// Sleep 8 seconds and ensure we have a non-zero TTL
time.Sleep(8 * time.Second)
// Get the current TTL
a := v.Client.Auth().Token()
s2, err := a.Lookup(v.Config.Token)
if err != nil {
t.Fatalf("failed to lookup token: %v", err)
}
ttl := parseTTLFromLookup(s2, t)
if ttl == 0 {
t.Fatalf("token renewal failed; ttl %v", ttl)
}
}
func parseTTLFromLookup(s *vapi.Secret, t *testing.T) int64 {
if s == nil {
t.Fatalf("nil secret")
} else if s.Data == nil {
t.Fatalf("nil data block in secret")
}
ttlRaw, ok := s.Data["ttl"]
if !ok {
t.Fatalf("no ttl")
}
ttlNumber, ok := ttlRaw.(json.Number)
if !ok {
t.Fatalf("failed to convert ttl %q to json Number", ttlRaw)
}
ttl, err := ttlNumber.Int64()
if err != nil {
t.Fatalf("Failed to get ttl from json.Number: %v", err)
}
return ttl
}
func TestVaultClient_LookupToken_Invalid(t *testing.T) {
tr := true
conf := &config.VaultConfig{
Enabled: &tr,
Addr: "http://foobar:12345",
Token: structs.GenerateUUID(),
}
// Enable vault but use a bad address so it never establishes a conn
logger := log.New(os.Stderr, "", log.LstdFlags)
client, err := NewVaultClient(conf, logger, nil)
if err != nil {
t.Fatalf("failed to build vault client: %v", err)
}
client.SetActive(true)
defer client.Stop()
_, err = client.LookupToken(context.Background(), "foo")
if err == nil || !strings.Contains(err.Error(), "established") {
t.Fatalf("Expected error because connection to Vault hasn't been made: %v", err)
}
}
func TestVaultClient_LookupToken_Root(t *testing.T) {
v := testutil.NewTestVault(t).Start()
defer v.Stop()
logger := log.New(os.Stderr, "", log.LstdFlags)
client, err := NewVaultClient(v.Config, logger, nil)
if err != nil {
t.Fatalf("failed to build vault client: %v", err)
}
client.SetActive(true)
defer client.Stop()
waitForConnection(client, t)
// Lookup ourselves
s, err := client.LookupToken(context.Background(), v.Config.Token)
if err != nil {
t.Fatalf("self lookup failed: %v", err)
}
policies, err := PoliciesFrom(s)
if err != nil {
t.Fatalf("failed to parse policies: %v", err)
}
expected := []string{"root"}
if !reflect.DeepEqual(policies, expected) {
t.Fatalf("Unexpected policies; got %v; want %v", policies, expected)
}
// Create a token with a different set of policies
expected = []string{"default"}
req := vapi.TokenCreateRequest{
Policies: expected,
}
s, err = v.Client.Auth().Token().Create(&req)
if err != nil {
t.Fatalf("failed to create child token: %v", err)
}
// Get the client token
if s == nil || s.Auth == nil {
t.Fatalf("bad secret response: %+v", s)
}
// Lookup new child
s, err = client.LookupToken(context.Background(), s.Auth.ClientToken)
if err != nil {
t.Fatalf("self lookup failed: %v", err)
}
policies, err = PoliciesFrom(s)
if err != nil {
t.Fatalf("failed to parse policies: %v", err)
}
if !reflect.DeepEqual(policies, expected) {
t.Fatalf("Unexpected policies; got %v; want %v", policies, expected)
}
}
func TestVaultClient_LookupToken_Role(t *testing.T) {
v := testutil.NewTestVault(t).Start()
defer v.Stop()
// Set the configs token in a new test role
v.Config.Token = defaultTestVaultRoleAndToken(v, t, 5)
logger := log.New(os.Stderr, "", log.LstdFlags)
client, err := NewVaultClient(v.Config, logger, nil)
if err != nil {
t.Fatalf("failed to build vault client: %v", err)
}
client.SetActive(true)
defer client.Stop()
waitForConnection(client, t)
// Lookup ourselves
s, err := client.LookupToken(context.Background(), v.Config.Token)
if err != nil {
t.Fatalf("self lookup failed: %v", err)
}
policies, err := PoliciesFrom(s)
if err != nil {
t.Fatalf("failed to parse policies: %v", err)
}
expected := []string{"auth", "default"}
if !reflect.DeepEqual(policies, expected) {
t.Fatalf("Unexpected policies; got %v; want %v", policies, expected)
}
// Create a token with a different set of policies
expected = []string{"default"}
req := vapi.TokenCreateRequest{
Policies: expected,
}
s, err = v.Client.Auth().Token().Create(&req)
if err != nil {
t.Fatalf("failed to create child token: %v", err)
}
// Get the client token
if s == nil || s.Auth == nil {
t.Fatalf("bad secret response: %+v", s)
}
// Lookup new child
s, err = client.LookupToken(context.Background(), s.Auth.ClientToken)
if err != nil {
t.Fatalf("self lookup failed: %v", err)
}
policies, err = PoliciesFrom(s)
if err != nil {
t.Fatalf("failed to parse policies: %v", err)
}
if !reflect.DeepEqual(policies, expected) {
t.Fatalf("Unexpected policies; got %v; want %v", policies, expected)
}
}
func TestVaultClient_LookupToken_RateLimit(t *testing.T) {
v := testutil.NewTestVault(t).Start()
defer v.Stop()
logger := log.New(os.Stderr, "", log.LstdFlags)
client, err := NewVaultClient(v.Config, logger, nil)
if err != nil {
t.Fatalf("failed to build vault client: %v", err)
}
client.SetActive(true)
defer client.Stop()
client.setLimit(rate.Limit(1.0))
waitForConnection(client, t)
// Spin up many requests. These should block
ctx, cancel := context.WithCancel(context.Background())
cancels := 0
numRequests := 10
unblock := make(chan struct{})
for i := 0; i < numRequests; i++ {
go func() {
// Ensure all the goroutines are made
time.Sleep(10 * time.Millisecond)
// Lookup ourselves
_, err := client.LookupToken(ctx, v.Config.Token)
if err != nil {
if err == context.Canceled {
cancels += 1
return
}
t.Fatalf("self lookup failed: %v", err)
return
}
// Cancel the context
cancel()
time.AfterFunc(1*time.Second, func() { close(unblock) })
}()
}
select {
case <-time.After(5 * time.Second):
t.Fatalf("timeout")
case <-unblock:
}
desired := numRequests - 1
if cancels != desired {
t.Fatalf("Incorrect number of cancels; got %d; want %d", cancels, desired)
}
}
func TestVaultClient_CreateToken_Root(t *testing.T) {
v := testutil.NewTestVault(t).Start()
defer v.Stop()
logger := log.New(os.Stderr, "", log.LstdFlags)
client, err := NewVaultClient(v.Config, logger, nil)
if err != nil {
t.Fatalf("failed to build vault client: %v", err)
}
client.SetActive(true)
defer client.Stop()
waitForConnection(client, t)
// Create an allocation that requires a Vault policy
a := mock.Alloc()
task := a.Job.TaskGroups[0].Tasks[0]
task.Vault = &structs.Vault{Policies: []string{"default"}}
s, err := client.CreateToken(context.Background(), a, task.Name)
if err != nil {
t.Fatalf("CreateToken failed: %v", err)
}
// Ensure that created secret is a wrapped token
if s == nil || s.WrapInfo == nil {
t.Fatalf("Bad secret: %#v", s)
}
d, err := time.ParseDuration(vaultTokenCreateTTL)
if err != nil {
t.Fatalf("bad: %v", err)
}
if s.WrapInfo.WrappedAccessor == "" {
t.Fatalf("Bad accessor: %v", s.WrapInfo.WrappedAccessor)
} else if s.WrapInfo.Token == "" {
t.Fatalf("Bad token: %v", s.WrapInfo.WrappedAccessor)
} else if s.WrapInfo.TTL != int(d.Seconds()) {
t.Fatalf("Bad ttl: %v", s.WrapInfo.WrappedAccessor)
}
}
func TestVaultClient_CreateToken_Role(t *testing.T) {
v := testutil.NewTestVault(t).Start()
defer v.Stop()
// Set the configs token in a new test role
v.Config.Token = defaultTestVaultRoleAndToken(v, t, 5)
// Start the client
logger := log.New(os.Stderr, "", log.LstdFlags)
client, err := NewVaultClient(v.Config, logger, nil)
if err != nil {
t.Fatalf("failed to build vault client: %v", err)
}
client.SetActive(true)
defer client.Stop()
waitForConnection(client, t)
// Create an allocation that requires a Vault policy
a := mock.Alloc()
task := a.Job.TaskGroups[0].Tasks[0]
task.Vault = &structs.Vault{Policies: []string{"default"}}
s, err := client.CreateToken(context.Background(), a, task.Name)
if err != nil {
t.Fatalf("CreateToken failed: %v", err)
}
// Ensure that created secret is a wrapped token
if s == nil || s.WrapInfo == nil {
t.Fatalf("Bad secret: %#v", s)
}
d, err := time.ParseDuration(vaultTokenCreateTTL)
if err != nil {
t.Fatalf("bad: %v", err)
}
if s.WrapInfo.WrappedAccessor == "" {
t.Fatalf("Bad accessor: %v", s.WrapInfo.WrappedAccessor)
} else if s.WrapInfo.Token == "" {
t.Fatalf("Bad token: %v", s.WrapInfo.WrappedAccessor)
} else if s.WrapInfo.TTL != int(d.Seconds()) {
t.Fatalf("Bad ttl: %v", s.WrapInfo.WrappedAccessor)
}
}
func TestVaultClient_CreateToken_Role_InvalidToken(t *testing.T) {
v := testutil.NewTestVault(t).Start()
defer v.Stop()
// Set the configs token in a new test role
defaultTestVaultRoleAndToken(v, t, 5)
v.Config.Token = "foo-bar"
// Start the client
logger := log.New(os.Stderr, "", log.LstdFlags)
client, err := NewVaultClient(v.Config, logger, nil)
if err != nil {
t.Fatalf("failed to build vault client: %v", err)
}
client.SetActive(true)
defer client.Stop()
testutil.WaitForResult(func() (bool, error) {
established, err := client.ConnectionEstablished()
if established {
return false, fmt.Errorf("Shouldn't establish")
}
return err != nil, nil
}, func(err error) {
t.Fatalf("Connection not established")
})
// Create an allocation that requires a Vault policy
a := mock.Alloc()
task := a.Job.TaskGroups[0].Tasks[0]
task.Vault = &structs.Vault{Policies: []string{"default"}}
_, err = client.CreateToken(context.Background(), a, task.Name)
if err == nil || !strings.Contains(err.Error(), "Connection to Vault failed") {
t.Fatalf("CreateToken should have failed: %v", err)
}
}
func TestVaultClient_CreateToken_Role_Unrecoverable(t *testing.T) {
v := testutil.NewTestVault(t).Start()
defer v.Stop()
// Set the configs token in a new test role
v.Config.Token = defaultTestVaultRoleAndToken(v, t, 5)
// Start the client
logger := log.New(os.Stderr, "", log.LstdFlags)
client, err := NewVaultClient(v.Config, logger, nil)
if err != nil {
t.Fatalf("failed to build vault client: %v", err)
}
client.SetActive(true)
defer client.Stop()
waitForConnection(client, t)
// Create an allocation that requires a Vault policy
a := mock.Alloc()
task := a.Job.TaskGroups[0].Tasks[0]
task.Vault = &structs.Vault{Policies: []string{"unknown_policy"}}
_, err = client.CreateToken(context.Background(), a, task.Name)
if err == nil {
t.Fatalf("CreateToken should have failed: %v", err)
}
_, ok := err.(*structs.RecoverableError)
if ok {
t.Fatalf("CreateToken should not be a recoverable error type: %v", err)
}
}
func TestVaultClient_CreateToken_Prestart(t *testing.T) {
v := testutil.NewTestVault(t)
defer v.Stop()
logger := log.New(os.Stderr, "", log.LstdFlags)
client, err := NewVaultClient(v.Config, logger, nil)
if err != nil {
t.Fatalf("failed to build vault client: %v", err)
}
client.SetActive(true)
defer client.Stop()
// Create an allocation that requires a Vault policy
a := mock.Alloc()
task := a.Job.TaskGroups[0].Tasks[0]
task.Vault = &structs.Vault{Policies: []string{"default"}}
_, err = client.CreateToken(context.Background(), a, task.Name)
if err == nil {
t.Fatalf("CreateToken should have failed: %v", err)
}
if rerr, ok := err.(*structs.RecoverableError); !ok {
t.Fatalf("Err should have been type recoverable error")
} else if ok && !rerr.Recoverable {
t.Fatalf("Err should have been recoverable")
}
}
func TestVaultClient_RevokeTokens_PreEstablishs(t *testing.T) {
v := testutil.NewTestVault(t)
logger := log.New(os.Stderr, "", log.LstdFlags)
client, err := NewVaultClient(v.Config, logger, nil)
if err != nil {
t.Fatalf("failed to build vault client: %v", err)
}
client.SetActive(true)
defer client.Stop()
// Create some VaultAccessors
vas := []*structs.VaultAccessor{
mock.VaultAccessor(),
mock.VaultAccessor(),
}
if err := client.RevokeTokens(context.Background(), vas, false); err != nil {
t.Fatalf("RevokeTokens failed: %v", err)
}
// Wasn't committed
if len(client.revoking) != 0 {
t.Fatalf("didn't add to revoke loop")
}
if err := client.RevokeTokens(context.Background(), vas, true); err != nil {
t.Fatalf("RevokeTokens failed: %v", err)
}
// Was committed
if len(client.revoking) != 2 {
t.Fatalf("didn't add to revoke loop")
}
}
func TestVaultClient_RevokeTokens_Root(t *testing.T) {
v := testutil.NewTestVault(t).Start()
defer v.Stop()
purged := 0
purge := func(accessors []*structs.VaultAccessor) error {
purged += len(accessors)
return nil
}
logger := log.New(os.Stderr, "", log.LstdFlags)
client, err := NewVaultClient(v.Config, logger, purge)
if err != nil {
t.Fatalf("failed to build vault client: %v", err)
}
client.SetActive(true)
defer client.Stop()
waitForConnection(client, t)
// Create some vault tokens
auth := v.Client.Auth().Token()
req := vapi.TokenCreateRequest{
Policies: []string{"default"},
}
t1, err := auth.Create(&req)
if err != nil {
t.Fatalf("Failed to create vault token: %v", err)
}
if t1 == nil || t1.Auth == nil {
t.Fatalf("bad secret response: %+v", t1)
}
t2, err := auth.Create(&req)
if err != nil {
t.Fatalf("Failed to create vault token: %v", err)
}
if t2 == nil || t2.Auth == nil {
t.Fatalf("bad secret response: %+v", t2)
}
// Create two VaultAccessors
vas := []*structs.VaultAccessor{
&structs.VaultAccessor{Accessor: t1.Auth.Accessor},
&structs.VaultAccessor{Accessor: t2.Auth.Accessor},
}
// Issue a token revocation
if err := client.RevokeTokens(context.Background(), vas, true); err != nil {
t.Fatalf("RevokeTokens failed: %v", err)
}
// Lookup the token and make sure we get an error
if s, err := auth.Lookup(t1.Auth.ClientToken); err == nil {
t.Fatalf("Revoked token lookup didn't fail: %+v", s)
}
if s, err := auth.Lookup(t2.Auth.ClientToken); err == nil {
t.Fatalf("Revoked token lookup didn't fail: %+v", s)
}
if purged != 2 {
t.Fatalf("Expected purged 2; got %d", purged)
}
}
func TestVaultClient_RevokeTokens_Role(t *testing.T) {
v := testutil.NewTestVault(t).Start()
defer v.Stop()
// Set the configs token in a new test role
v.Config.Token = defaultTestVaultRoleAndToken(v, t, 5)
purged := 0
purge := func(accessors []*structs.VaultAccessor) error {
purged += len(accessors)
return nil
}
logger := log.New(os.Stderr, "", log.LstdFlags)
client, err := NewVaultClient(v.Config, logger, purge)
if err != nil {
t.Fatalf("failed to build vault client: %v", err)
}
client.SetActive(true)
defer client.Stop()
waitForConnection(client, t)
// Create some vault tokens
auth := v.Client.Auth().Token()
req := vapi.TokenCreateRequest{
Policies: []string{"default"},
}
t1, err := auth.Create(&req)
if err != nil {
t.Fatalf("Failed to create vault token: %v", err)
}
if t1 == nil || t1.Auth == nil {
t.Fatalf("bad secret response: %+v", t1)
}
t2, err := auth.Create(&req)
if err != nil {
t.Fatalf("Failed to create vault token: %v", err)
}
if t2 == nil || t2.Auth == nil {
t.Fatalf("bad secret response: %+v", t2)
}
// Create two VaultAccessors
vas := []*structs.VaultAccessor{
&structs.VaultAccessor{Accessor: t1.Auth.Accessor},
&structs.VaultAccessor{Accessor: t2.Auth.Accessor},
}
// Issue a token revocation
if err := client.RevokeTokens(context.Background(), vas, true); err != nil {
t.Fatalf("RevokeTokens failed: %v", err)
}
// Lookup the token and make sure we get an error
if s, err := auth.Lookup(t1.Auth.ClientToken); err == nil {
t.Fatalf("Revoked token lookup didn't fail: %+v", s)
}
if s, err := auth.Lookup(t2.Auth.ClientToken); err == nil {
t.Fatalf("Revoked token lookup didn't fail: %+v", s)
}
if purged != 2 {
t.Fatalf("Expected purged 2; got %d", purged)
}
}
func waitForConnection(v *vaultClient, t *testing.T) {
testutil.WaitForResult(func() (bool, error) {
return v.ConnectionEstablished()
}, func(err error) {
t.Fatalf("Connection not established")
})
}