open-vault/helper/testhelpers/testhelpers.go
ncabatoff cc41e608dc
Add core.coreNumber field, used to differentiate multiple cores/clusters when running tests (#6855)
This is not used or exposed in prod.

Remove some test-specific code from the cluster-building helpers. The corresponding additions go on the ent side.
2019-06-10 14:07:16 -04:00

625 lines
18 KiB
Go

package testhelpers
import (
"context"
"encoding/base64"
"errors"
"fmt"
"math/rand"
"net/http"
"reflect"
"sync"
"time"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/vault/sdk/logical"
"github.com/hashicorp/vault/vault/cluster"
log "github.com/hashicorp/go-hclog"
"github.com/hashicorp/vault/api"
credAppRole "github.com/hashicorp/vault/builtin/credential/approle"
credUserpass "github.com/hashicorp/vault/builtin/credential/userpass"
"github.com/hashicorp/vault/helper/xor"
"github.com/hashicorp/vault/sdk/helper/consts"
"github.com/hashicorp/vault/sdk/physical"
"github.com/hashicorp/vault/sdk/physical/inmem"
"github.com/hashicorp/vault/vault"
testing "github.com/mitchellh/go-testing-interface"
)
type ReplicatedTestClusters struct {
PerfPrimaryCluster *vault.TestCluster
PerfSecondaryCluster *vault.TestCluster
PerfPrimaryDRCluster *vault.TestCluster
PerfSecondaryDRCluster *vault.TestCluster
}
func (r *ReplicatedTestClusters) Cleanup() {
r.PerfPrimaryCluster.Cleanup()
r.PerfSecondaryCluster.Cleanup()
if r.PerfPrimaryDRCluster != nil {
r.PerfPrimaryDRCluster.Cleanup()
}
if r.PerfSecondaryDRCluster != nil {
r.PerfSecondaryDRCluster.Cleanup()
}
}
func (r *ReplicatedTestClusters) Primary() (*vault.TestCluster, *vault.TestClusterCore, *api.Client) {
return r.PerfPrimaryCluster, r.PerfPrimaryCluster.Cores[0], r.PerfPrimaryCluster.Cores[0].Client
}
func (r *ReplicatedTestClusters) Secondary() (*vault.TestCluster, *vault.TestClusterCore, *api.Client) {
return r.PerfSecondaryCluster, r.PerfSecondaryCluster.Cores[0], r.PerfSecondaryCluster.Cores[0].Client
}
func (r *ReplicatedTestClusters) PrimaryDR() (*vault.TestCluster, *vault.TestClusterCore, *api.Client) {
return r.PerfPrimaryDRCluster, r.PerfPrimaryDRCluster.Cores[0], r.PerfPrimaryDRCluster.Cores[0].Client
}
func (r *ReplicatedTestClusters) SecondaryDR() (*vault.TestCluster, *vault.TestClusterCore, *api.Client) {
return r.PerfSecondaryDRCluster, r.PerfSecondaryDRCluster.Cores[0], r.PerfSecondaryDRCluster.Cores[0].Client
}
// Generates a root token on the target cluster.
func GenerateRoot(t testing.T, cluster *vault.TestCluster, drToken bool) string {
token, err := GenerateRootWithError(t, cluster, drToken)
if err != nil {
t.Fatal(err)
}
return token
}
func GenerateRootWithError(t testing.T, cluster *vault.TestCluster, drToken bool) (string, error) {
// If recovery keys supported, use those to perform root token generation instead
var keys [][]byte
if cluster.Cores[0].SealAccess().RecoveryKeySupported() {
keys = cluster.RecoveryKeys
} else {
keys = cluster.BarrierKeys
}
client := cluster.Cores[0].Client
f := client.Sys().GenerateRootInit
if drToken {
f = client.Sys().GenerateDROperationTokenInit
}
status, err := f("", "")
if err != nil {
return "", err
}
if status.Required > len(keys) {
return "", fmt.Errorf("need more keys than have, need %d have %d", status.Required, len(keys))
}
otp := status.OTP
for i, key := range keys {
if i >= status.Required {
break
}
f := client.Sys().GenerateRootUpdate
if drToken {
f = client.Sys().GenerateDROperationTokenUpdate
}
status, err = f(base64.StdEncoding.EncodeToString(key), status.Nonce)
if err != nil {
return "", err
}
}
if !status.Complete {
return "", errors.New("generate root operation did not end successfully")
}
tokenBytes, err := base64.RawStdEncoding.DecodeString(status.EncodedToken)
if err != nil {
return "", err
}
tokenBytes, err = xor.XORBytes(tokenBytes, []byte(otp))
if err != nil {
return "", err
}
return string(tokenBytes), nil
}
// RandomWithPrefix is used to generate a unique name with a prefix, for
// randomizing names in acceptance tests
func RandomWithPrefix(name string) string {
return fmt.Sprintf("%s-%d", name, rand.New(rand.NewSource(time.Now().UnixNano())).Int())
}
func EnsureCoresSealed(t testing.T, c *vault.TestCluster) {
t.Helper()
for _, core := range c.Cores {
EnsureCoreSealed(t, core)
}
}
func EnsureCoreSealed(t testing.T, core *vault.TestClusterCore) error {
core.Seal(t)
timeout := time.Now().Add(60 * time.Second)
for {
if time.Now().After(timeout) {
return fmt.Errorf("timeout waiting for core to seal")
}
if core.Core.Sealed() {
break
}
time.Sleep(250 * time.Millisecond)
}
return nil
}
func EnsureCoresUnsealed(t testing.T, c *vault.TestCluster) {
t.Helper()
for _, core := range c.Cores {
EnsureCoreUnsealed(t, c, core)
}
}
func EnsureCoreUnsealed(t testing.T, c *vault.TestCluster, core *vault.TestClusterCore) {
if !core.Sealed() {
return
}
client := core.Client
client.Sys().ResetUnsealProcess()
for j := 0; j < len(c.BarrierKeys); j++ {
statusResp, err := client.Sys().Unseal(base64.StdEncoding.EncodeToString(c.BarrierKeys[j]))
if err != nil {
// Sometimes when we get here it's already unsealed on its own
// and then this fails for DR secondaries so check again
if core.Sealed() {
t.Fatal(err)
}
break
}
if statusResp == nil {
t.Fatal("nil status response during unseal")
}
if !statusResp.Sealed {
break
}
}
if core.Sealed() {
t.Fatal("core is still sealed")
}
}
func EnsureCoreIsPerfStandby(t testing.T, client *api.Client) {
t.Helper()
start := time.Now()
for {
health, err := client.Sys().Health()
if err != nil {
t.Fatal(err)
}
if health.PerformanceStandby {
break
}
t.Log("waiting for performance standby", fmt.Sprintf("%+v", health))
time.Sleep(time.Millisecond * 500)
if time.Now().After(start.Add(time.Second * 60)) {
t.Fatal("did not become a perf standby")
}
}
}
func WaitForReplicationState(t testing.T, c *vault.Core, state consts.ReplicationState) {
timeout := time.Now().Add(10 * time.Second)
for {
if time.Now().After(timeout) {
t.Fatalf("timeout waiting for core to have state %d", uint32(state))
}
state := c.ReplicationState()
if state.HasState(state) {
break
}
time.Sleep(1 * time.Second)
}
}
type PassthroughWithLocalPaths struct {
logical.Backend
}
func (p *PassthroughWithLocalPaths) SpecialPaths() *logical.Paths {
return &logical.Paths{
LocalStorage: []string{
"*",
},
}
}
func PassthroughWithLocalPathsFactory(ctx context.Context, c *logical.BackendConfig) (logical.Backend, error) {
b, err := vault.PassthroughBackendFactory(ctx, c)
if err != nil {
return nil, err
}
return &PassthroughWithLocalPaths{b}, nil
}
func ConfClusterAndCore(t testing.T, conf *vault.CoreConfig, opts *vault.TestClusterOptions) (*vault.TestCluster, *vault.TestClusterCore) {
if conf.Physical != nil || conf.HAPhysical != nil {
t.Fatalf("conf.Physical and conf.HAPhysical cannot be specified")
}
if opts.Logger == nil {
t.Fatalf("opts.Logger must be specified")
}
inm, err := inmem.NewTransactionalInmem(nil, opts.Logger)
if err != nil {
t.Fatal(err)
}
inmha, err := inmem.NewInmemHA(nil, opts.Logger)
if err != nil {
t.Fatal(err)
}
coreConfig := *conf
coreConfig.Physical = inm
coreConfig.HAPhysical = inmha.(physical.HABackend)
coreConfig.CredentialBackends = map[string]logical.Factory{
"approle": credAppRole.Factory,
"userpass": credUserpass.Factory,
}
vault.AddNoopAudit(&coreConfig)
cluster := vault.NewTestCluster(t, &coreConfig, opts)
cluster.Start()
cores := cluster.Cores
core := cores[0]
vault.TestWaitActive(t, core.Core)
return cluster, core
}
func GetPerfReplicatedClusters(t testing.T, conf *vault.CoreConfig, opts *vault.TestClusterOptions) *ReplicatedTestClusters {
ret := &ReplicatedTestClusters{}
var logger hclog.Logger
if opts != nil {
logger = opts.Logger
}
if logger == nil {
logger = log.New(&log.LoggerOptions{
Mutex: &sync.Mutex{},
Level: log.Trace,
})
}
// Set this lower so that state populates quickly to standby nodes
cluster.HeartbeatInterval = 2 * time.Second
numCores := opts.NumCores
if numCores == 0 {
numCores = vault.DefaultNumCores
}
localopts := *opts
localopts.Logger = logger.Named("perf-pri")
ret.PerfPrimaryCluster, _ = ConfClusterAndCore(t, conf, &localopts)
localopts.Logger = logger.Named("perf-sec")
localopts.FirstCoreNumber += numCores
ret.PerfSecondaryCluster, _ = ConfClusterAndCore(t, conf, &localopts)
SetupTwoClusterPerfReplication(t, ret.PerfPrimaryCluster, ret.PerfSecondaryCluster)
return ret
}
func GetFourReplicatedClusters(t testing.T, handlerFunc func(*vault.HandlerProperties) http.Handler) *ReplicatedTestClusters {
return GetFourReplicatedClustersWithConf(t, &vault.CoreConfig{}, &vault.TestClusterOptions{
HandlerFunc: handlerFunc,
})
}
func GetFourReplicatedClustersWithConf(t testing.T, conf *vault.CoreConfig, opts *vault.TestClusterOptions) *ReplicatedTestClusters {
ret := &ReplicatedTestClusters{}
logger := log.New(&log.LoggerOptions{
Mutex: &sync.Mutex{},
Level: log.Trace,
})
// Set this lower so that state populates quickly to standby nodes
cluster.HeartbeatInterval = 2 * time.Second
numCores := opts.NumCores
if numCores == 0 {
numCores = vault.DefaultNumCores
}
localopts := *opts
localopts.Logger = logger.Named("perf-pri")
ret.PerfPrimaryCluster, _ = ConfClusterAndCore(t, conf, &localopts)
localopts.Logger = logger.Named("perf-sec")
localopts.FirstCoreNumber += numCores
ret.PerfSecondaryCluster, _ = ConfClusterAndCore(t, conf, &localopts)
localopts.Logger = logger.Named("perf-pri-dr")
localopts.FirstCoreNumber += numCores
ret.PerfPrimaryDRCluster, _ = ConfClusterAndCore(t, conf, &localopts)
localopts.Logger = logger.Named("perf-sec-dr")
localopts.FirstCoreNumber += numCores
ret.PerfSecondaryDRCluster, _ = ConfClusterAndCore(t, conf, &localopts)
builder := &ReplicatedTestClustersBuilder{clusters: ret}
builder.setupFourClusterReplication(t)
// Wait until poison pills have been read
time.Sleep(45 * time.Second)
EnsureCoresUnsealed(t, ret.PerfPrimaryCluster)
EnsureCoresUnsealed(t, ret.PerfSecondaryCluster)
EnsureCoresUnsealed(t, ret.PerfPrimaryDRCluster)
EnsureCoresUnsealed(t, ret.PerfSecondaryDRCluster)
return ret
}
type ReplicatedTestClustersBuilder struct {
clusters *ReplicatedTestClusters
perfToken string
drToken string
perfSecondaryRootToken string
perfSecondaryDRToken string
}
func SetupTwoClusterPerfReplication(t testing.T, pri, sec *vault.TestCluster) {
clusters := &ReplicatedTestClusters{
PerfPrimaryCluster: pri,
PerfSecondaryCluster: sec,
}
builder := &ReplicatedTestClustersBuilder{clusters: clusters}
builder.setupTwoClusterReplication(t)
}
func (r *ReplicatedTestClustersBuilder) setupTwoClusterReplication(t testing.T) {
t.Log("enabling perf primary")
r.enablePerfPrimary(t)
WaitForActiveNode(t, r.clusters.PerfPrimaryCluster)
r.getPerformanceToken(t)
t.Log("enabling perf secondary")
r.enablePerformanceSecondary(t)
}
func SetupFourClusterReplication(t testing.T, pri, sec, pridr, secdr *vault.TestCluster) {
clusters := &ReplicatedTestClusters{
PerfPrimaryCluster: pri,
PerfSecondaryCluster: sec,
PerfPrimaryDRCluster: pridr,
PerfSecondaryDRCluster: secdr,
}
builder := &ReplicatedTestClustersBuilder{clusters: clusters}
builder.setupFourClusterReplication(t)
}
func (r *ReplicatedTestClustersBuilder) setupFourClusterReplication(t testing.T) {
t.Log("enabling perf primary")
r.enablePerfPrimary(t)
r.getPerformanceToken(t)
t.Log("enabling dr primary")
enableDrPrimary(t, r.clusters.PerfPrimaryCluster)
r.drToken = getDrToken(t, r.clusters.PerfPrimaryCluster, "primary-dr-secondary")
WaitForActiveNode(t, r.clusters.PerfPrimaryCluster)
time.Sleep(1 * time.Second)
t.Log("enabling perf secondary")
r.enablePerformanceSecondary(t)
enableDrPrimary(t, r.clusters.PerfSecondaryCluster)
r.perfSecondaryDRToken = getDrToken(t, r.clusters.PerfSecondaryCluster, "secondary-dr-secondary")
t.Log("enabling dr secondary on primary dr cluster")
r.enableDrSecondary(t, r.clusters.PerfPrimaryDRCluster, r.drToken, r.clusters.PerfPrimaryCluster.CACertPEMFile)
r.clusters.PerfPrimaryDRCluster.Cores[0].Client.SetToken(r.clusters.PerfPrimaryCluster.Cores[0].Client.Token())
WaitForActiveNode(t, r.clusters.PerfPrimaryDRCluster)
time.Sleep(1 * time.Second)
t.Log("enabling dr secondary on secondary dr cluster")
r.enableDrSecondary(t, r.clusters.PerfSecondaryDRCluster, r.perfSecondaryDRToken, r.clusters.PerfSecondaryCluster.CACertPEMFile)
r.clusters.PerfSecondaryDRCluster.Cores[0].Client.SetToken(r.perfSecondaryRootToken)
WaitForActiveNode(t, r.clusters.PerfSecondaryDRCluster)
}
func (r *ReplicatedTestClustersBuilder) enablePerfPrimary(t testing.T) {
c := r.clusters.PerfPrimaryCluster.Cores[0]
_, err := c.Client.Logical().Write("sys/replication/performance/primary/enable", nil)
if err != nil {
t.Fatal(err)
}
WaitForReplicationState(t, c.Core, consts.ReplicationPerformancePrimary)
}
func (r *ReplicatedTestClustersBuilder) getPerformanceToken(t testing.T) {
client := r.clusters.PerfPrimaryCluster.Cores[0].Client
req := map[string]interface{}{
"id": "perf-secondary",
}
secret, err := client.Logical().Write("sys/replication/performance/primary/secondary-token", req)
if err != nil {
t.Fatal(err)
}
r.perfToken = secret.WrapInfo.Token
}
func enableDrPrimary(t testing.T, tc *vault.TestCluster) {
c := tc.Cores[0]
_, err := c.Client.Logical().Write("sys/replication/dr/primary/enable", nil)
if err != nil {
t.Fatal(err)
}
WaitForReplicationState(t, c.Core, consts.ReplicationDRPrimary)
}
func getDrToken(t testing.T, tc *vault.TestCluster, id string) string {
req := map[string]interface{}{
"id": id,
}
secret, err := tc.Cores[0].Client.Logical().Write("sys/replication/dr/primary/secondary-token", req)
if err != nil {
t.Fatal(err)
}
return secret.WrapInfo.Token
}
func (r *ReplicatedTestClustersBuilder) enablePerformanceSecondary(t testing.T) {
c := r.clusters.PerfSecondaryCluster.Cores[0]
_, err := c.Client.Logical().Write("sys/replication/performance/secondary/enable", map[string]interface{}{
"token": r.perfToken,
"ca_file": r.clusters.PerfPrimaryCluster.CACertPEMFile,
})
if err != nil {
t.Fatal(err)
}
WaitForReplicationState(t, c.Core, consts.ReplicationPerformanceSecondary)
r.clusters.PerfSecondaryCluster.BarrierKeys = r.clusters.PerfPrimaryCluster.BarrierKeys
// We want to make sure we unseal all the nodes so we first need to wait
// until two of the nodes seal due to the poison pill being written
WaitForNCoresSealed(t, r.clusters.PerfSecondaryCluster, 2)
EnsureCoresUnsealed(t, r.clusters.PerfSecondaryCluster)
WaitForActiveNode(t, r.clusters.PerfSecondaryCluster)
r.perfSecondaryRootToken = GenerateRoot(t, r.clusters.PerfSecondaryCluster, false)
for _, core := range r.clusters.PerfSecondaryCluster.Cores {
core.Client.SetToken(r.perfSecondaryRootToken)
}
}
func (r *ReplicatedTestClustersBuilder) enableDrSecondary(t testing.T, tc *vault.TestCluster, token, ca_file string) {
_, err := tc.Cores[0].Client.Logical().Write("sys/replication/dr/secondary/enable", map[string]interface{}{
"token": token,
"ca_file": ca_file,
})
if err != nil {
t.Fatal(err)
}
WaitForReplicationState(t, tc.Cores[0].Core, consts.ReplicationDRSecondary)
tc.BarrierKeys = r.clusters.PerfPrimaryCluster.BarrierKeys
// We want to make sure we unseal all the nodes so we first need to wait
// until two of the nodes seal due to the poison pill being written
WaitForNCoresSealed(t, tc, 2)
EnsureCoresUnsealed(t, tc)
}
func DeriveActiveCore(t testing.T, cluster *vault.TestCluster) *vault.TestClusterCore {
for i := 0; i < 10; i++ {
for _, core := range cluster.Cores {
leaderResp, err := core.Client.Sys().Leader()
if err != nil {
t.Fatal(err)
}
if leaderResp.IsSelf {
return core
}
}
time.Sleep(1 * time.Second)
}
t.Fatal("could not derive the active core")
return nil
}
func DeriveStandbyCores(t testing.T, cluster *vault.TestCluster) []*vault.TestClusterCore {
cores := make([]*vault.TestClusterCore, 0, 2)
for _, core := range cluster.Cores {
leaderResp, err := core.Client.Sys().Leader()
if err != nil {
t.Fatal(err)
}
if !leaderResp.IsSelf {
cores = append(cores, core)
}
}
return cores
}
func WaitForNCoresSealed(t testing.T, cluster *vault.TestCluster, n int) {
t.Helper()
for i := 0; i < 30; i++ {
sealed := 0
for _, core := range cluster.Cores {
if core.Core.Sealed() {
sealed++
}
}
if sealed >= n {
return
}
time.Sleep(time.Second)
}
t.Fatalf("%d cores were not sealed", n)
}
func WaitForActiveNode(t testing.T, cluster *vault.TestCluster) *vault.TestClusterCore {
t.Helper()
for i := 0; i < 30; i++ {
for _, core := range cluster.Cores {
if standby, _ := core.Core.Standby(); !standby {
return core
}
}
time.Sleep(time.Second)
}
t.Fatalf("node did not become active")
return nil
}
func WaitForMatchingMerkleRoots(t testing.T, endpoint string, primary, secondary *api.Client) {
getRoot := func(mode string, cli *api.Client) string {
status, err := cli.Logical().Read(endpoint + "status")
if err != nil {
t.Fatal(err)
}
if status == nil || status.Data == nil || status.Data["mode"] == nil {
t.Fatal("got nil secret or data")
}
if status.Data["mode"].(string) != mode {
t.Fatalf("expected mode=%s, got %s", mode, status.Data["mode"].(string))
}
return status.Data["merkle_root"].(string)
}
t.Helper()
for i := 0; i < 30; i++ {
secRoot := getRoot("secondary", secondary)
priRoot := getRoot("primary", primary)
if reflect.DeepEqual(priRoot, secRoot) {
return
}
time.Sleep(time.Second)
}
t.Fatalf("roots did not become equal")
}
func WaitForWAL(t testing.T, c *vault.TestClusterCore, wal uint64) {
timeout := time.Now().Add(3 * time.Second)
for {
if time.Now().After(timeout) {
t.Fatal("timeout waiting for WAL")
}
if vault.LastRemoteWAL(c.Core) >= wal {
break
}
time.Sleep(1 * time.Second)
}
}