luxolus
/
open-consul
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

test: cleanup and unflake parts of the upgrade compat tests (#13126)

This commit is contained in:
R.B. Boyer 2022-05-18 14:52:26 -05:00 committed by GitHub
parent 5fdfec1275
commit e8bbcaca86
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 250 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
GNUmakefile
View File

@ -335,6 +335,20 @@ ui-docker: ui-build-image
test-envoy-integ: $(ENVOY_INTEG_DEPS)
@go test -v -timeout=30m -tags integration ./test/integration/connect/envoy
.PHONY: test-compat-integ
test-compat-integ: dev-docker
ifeq ("$(GOTAGS)","")
@docker tag consul-dev:latest consul:local
@docker run --rm -t consul:local consul version
@cd ./test/integration/consul-container && \
go test -v -timeout=30m ./upgrade --target-version local --latest-version latest
else
@docker tag consul-dev:latest hashicorp/consul-enterprise:local
@docker run --rm -t hashicorp/consul-enterprise:local consul version
@cd ./test/integration/consul-container && \
go test -v -timeout=30m ./upgrade --tags $(GOTAGS) --target-version local --latest-version latest
endif
test-connect-ca-providers:
ifeq ("$(CIRCLECI)","true")
# Run in CI

125
test/integration/consul-container/libs/node/consul-container.go
View File

@ -26,6 +26,24 @@ type consulContainerNode struct {
container testcontainers.Container
ip string
port int
config Config
req testcontainers.ContainerRequest
dataDir string
}
func (c *consulContainerNode) GetConfig() Config {
return c.config
}
func newConsulContainerWithReq(ctx context.Context, req testcontainers.ContainerRequest) (testcontainers.Container, error) {
container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
ContainerRequest: req,
Started: true,
})
if err != nil {
return nil, err
}
return container, nil
}
// NewConsulContainer starts a Consul node in a container with the given config.
@ -36,39 +54,36 @@ func NewConsulContainer(ctx context.Context, config Config) (Node, error) {
return nil, err
}
name := utils.RandName("consul-")
tmpDir, err := ioutils.TempDir("", name)
tmpDirData, err := ioutils.TempDir("", name)
if err != nil {
return nil, err
}
err = os.Chmod(tmpDir, 0777)
err = os.Chmod(tmpDirData, 0777)
if err != nil {
return nil, err
}
err = os.Mkdir(tmpDir+"/config", 0777)
if err != nil {
return nil, err
}
configFile := tmpDir + "/config/config.hcl"
err = os.WriteFile(configFile, []byte(config.HCL), 0644)
configFile, err := createConfigFile(config.HCL)
if err != nil {
return nil, err
}
skipReaper := isRYUKDisabled()
req := testcontainers.ContainerRequest{
Image: "consul:" + config.Version,
Image: consulImage + ":" + config.Version,
ExposedPorts: []string{"8500/tcp"},
WaitingFor: wait.ForLog(bootLogLine).WithStartupTimeout(10 * time.Second),
AutoRemove: false,
Name: name,
Mounts: testcontainers.ContainerMounts{testcontainers.ContainerMount{Source: testcontainers.DockerBindMountSource{HostPath: configFile}, Target: "/consul/config/config.hcl"}},
Cmd: config.Cmd,
SkipReaper: skipReaper,
Env: map[string]string{"CONSUL_LICENSE": license},
Mounts: testcontainers.ContainerMounts{
testcontainers.ContainerMount{Source: testcontainers.DockerBindMountSource{HostPath: configFile}, Target: "/consul/config/config.hcl"},
testcontainers.ContainerMount{Source: testcontainers.DockerBindMountSource{HostPath: tmpDirData}, Target: "/consul/data"},
},
Cmd: config.Cmd,
SkipReaper: skipReaper,
Env: map[string]string{"CONSUL_LICENSE": license},
}
container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
ContainerRequest: req,
Started: true,
})
container, err := newConsulContainerWithReq(ctx, req)
if err != nil {
return nil, err
}
@ -90,6 +105,7 @@ func NewConsulContainer(ctx context.Context, config Config) (Node, error) {
uri := fmt.Sprintf("http://%s:%s", localIP, mappedPort.Port())
c := new(consulContainerNode)
c.config = config
c.container = container
c.ip = ip
c.port = mappedPort.Int()
@ -97,7 +113,8 @@ func NewConsulContainer(ctx context.Context, config Config) (Node, error) {
apiConfig.Address = uri
c.client, err = api.NewClient(apiConfig)
c.ctx = ctx
c.req = req
c.dataDir = tmpDirData
if err != nil {
return nil, err
}
@ -114,6 +131,57 @@ func (c *consulContainerNode) GetAddr() (string, int) {
return c.ip, c.port
}
func (c *consulContainerNode) Upgrade(ctx context.Context, config Config) error {
file, err := createConfigFile(config.HCL)
if err != nil {
return err
}
c.req.Cmd = config.Cmd
c.req.Mounts = testcontainers.ContainerMounts{
testcontainers.ContainerMount{Source: testcontainers.DockerBindMountSource{HostPath: file}, Target: "/consul/config/config.hcl"},
testcontainers.ContainerMount{Source: testcontainers.DockerBindMountSource{HostPath: c.dataDir}, Target: "/consul/data"},
}
c.req.Image = consulImage + ":" + config.Version
err = c.container.Terminate(ctx)
if err != nil {
return err
}
container, err := newConsulContainerWithReq(ctx, c.req)
if err != nil {
return err
}
c.container = container
localIP, err := container.Host(ctx)
if err != nil {
return err
}
mappedPort, err := container.MappedPort(ctx, "8500")
if err != nil {
return err
}
ip, err := container.ContainerIP(ctx)
if err != nil {
return err
}
uri := fmt.Sprintf("http://%s:%s", localIP, mappedPort.Port())
c.ip = ip
c.port = mappedPort.Int()
apiConfig := api.DefaultConfig()
apiConfig.Address = uri
c.client, err = api.NewClient(apiConfig)
c.ctx = ctx
c.config = config
if err != nil {
return err
}
return nil
}
// Terminate attempts to terminate the container. On failure, an error will be
// returned and the reaper process (RYUK) will handle cleanup.
func (c *consulContainerNode) Terminate() error {
@ -147,3 +215,24 @@ func readLicense() (string, error) {
}
return license, nil
}
func createConfigFile(HCL string) (string, error) {
tmpDir, err := ioutils.TempDir("", "consul-container-test-config")
if err != nil {
return "", err
}
err = os.Chmod(tmpDir, 0777)
if err != nil {
return "", err
}
err = os.Mkdir(tmpDir+"/config", 0777)
if err != nil {
return "", err
}
configFile := tmpDir + "/config/config.hcl"
err = os.WriteFile(configFile, []byte(HCL), 0644)
if err != nil {
return "", err
}
return configFile, nil
}

6
test/integration/consul-container/libs/node/consul-container_oss.go Normal file
View File

@ -0,0 +1,6 @@
//go:build !consulent
// +build !consulent
package node
const consulImage = "consul"

21
test/integration/consul-container/libs/node/node.go
View File

@ -1,13 +1,20 @@
package node
import "github.com/hashicorp/consul/api"
import (
"context"
"github.com/hashicorp/consul/api"
)
// Node represent a Consul node abstraction
type Node interface {
Terminate() error
GetClient() *api.Client
GetAddr() (string, int)
}
type (
// Node represent a Consul node abstraction
Node interface {
Terminate() error
GetClient() *api.Client
GetAddr() (string, int)
GetConfig() Config
Upgrade(ctx context.Context, config Config) error
}
)
// Config is a set of configurations required to create a Node
type Config struct {

141
test/integration/consul-container/upgrade/healthcheck_test.go
View File

@ -6,48 +6,42 @@ import (
"testing"
"time"
"github.com/hashicorp/consul/api"
"github.com/stretchr/testify/require"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/integration/consul-container/libs/cluster"
"github.com/hashicorp/consul/integration/consul-container/libs/node"
"github.com/hashicorp/consul/integration/consul-container/libs/utils"
"github.com/hashicorp/consul/sdk/testutil/retry"
"github.com/stretchr/testify/require"
)
const retryTimeout = 20 * time.Second
const retryFrequency = 500 * time.Millisecond
// Test health check GRPC call using Target Servers and Latest GA Clients
func TestTargetServersWithLatestGAClients(t *testing.T) {
numServers := 3
cluster, err := serversCluster(t, numServers, *targetImage)
require.NoError(t, err)
defer Terminate(t, cluster)
numClients := 1
const (
numServers = 3
numClients = 1
)
cluster := serversCluster(t, numServers, *targetImage)
defer Terminate(t, cluster)
clients := clientsCreate(t, numClients, *latestImage)
require.NoError(t, cluster.AddNodes(clients))
clients, err := clientsCreate(numClients, *latestImage)
require.NoError(t, err)
client := cluster.Nodes[0].GetClient()
err = cluster.AddNodes(clients)
retry.RunWith(&retry.Timer{Timeout: retryTimeout, Wait: retryFrequency}, t, func(r *retry.R) {
leader, err := cluster.Leader()
require.NoError(r, err)
require.NotEmpty(r, leader)
members, err := client.Agent().Members(false)
require.Len(r, members, 4)
})
waitForLeader(t, cluster)
waitForMembers(t, client, 4)
serviceName := "api"
err, index := serviceCreate(t, client, serviceName)
index := serviceCreate(t, client, serviceName)
ch := make(chan []*api.ServiceEntry)
errCh := make(chan error)
go func() {
service, q, err := client.Health().Service(serviceName, "", false, &api.QueryOptions{WaitIndex: index})
if q.QueryBackend != api.QueryBackendStreaming {
if err == nil && q.QueryBackend != api.QueryBackendStreaming {
err = fmt.Errorf("invalid backend for this test %s", q.QueryBackend)
}
if err != nil {
@ -56,7 +50,11 @@ func TestTargetServersWithLatestGAClients(t *testing.T) {
ch <- service
}
}()
err = client.Agent().ServiceRegister(&api.AgentServiceRegistration{Name: serviceName, Port: 9998})
require.NoError(t, client.Agent().ServiceRegister(
&api.AgentServiceRegistration{Name: serviceName, Port: 9998},
))
timer := time.NewTimer(1 * time.Second)
select {
case err := <-errCh:
@ -99,26 +97,27 @@ func TestMixedServersMajorityLatestGAClient(t *testing.T) {
require.NoError(t, err)
defer Terminate(t, cluster)
numClients := 1
clients, err := clientsCreate(numClients, *latestImage)
const (
numClients = 1
)
clients := clientsCreate(t, numClients, *latestImage)
require.NoError(t, cluster.AddNodes(clients))
client := clients[0].GetClient()
err = cluster.AddNodes(clients)
retry.RunWith(&retry.Timer{Timeout: retryTimeout, Wait: retryFrequency}, t, func(r *retry.R) {
leader, err := cluster.Leader()
require.NoError(r, err)
require.NotEmpty(r, leader)
members, err := client.Agent().Members(false)
require.Len(r, members, 4)
})
waitForLeader(t, cluster)
waitForMembers(t, client, 4)
serviceName := "api"
err, index := serviceCreate(t, client, serviceName)
index := serviceCreate(t, client, serviceName)
ch := make(chan []*api.ServiceEntry)
errCh := make(chan error)
go func() {
service, q, err := client.Health().Service(serviceName, "", false, &api.QueryOptions{WaitIndex: index})
if q.QueryBackend != api.QueryBackendStreaming {
if err == nil && q.QueryBackend != api.QueryBackendStreaming {
err = fmt.Errorf("invalid backend for this test %s", q.QueryBackend)
}
if err != nil {
@ -127,7 +126,11 @@ func TestMixedServersMajorityLatestGAClient(t *testing.T) {
ch <- service
}
}()
err = client.Agent().ServiceRegister(&api.AgentServiceRegistration{Name: serviceName, Port: 9998})
require.NoError(t, client.Agent().ServiceRegister(
&api.AgentServiceRegistration{Name: serviceName, Port: 9998},
))
timer := time.NewTimer(1 * time.Second)
select {
case err := <-errCh:
@ -169,26 +172,27 @@ func TestMixedServersMajorityTargetGAClient(t *testing.T) {
require.NoError(t, err)
defer Terminate(t, cluster)
numClients := 1
clients, err := clientsCreate(numClients, *latestImage)
const (
numClients = 1
)
clients := clientsCreate(t, numClients, *latestImage)
require.NoError(t, cluster.AddNodes(clients))
client := clients[0].GetClient()
err = cluster.AddNodes(clients)
retry.RunWith(&retry.Timer{Timeout: retryTimeout, Wait: retryFrequency}, t, func(r *retry.R) {
leader, err := cluster.Leader()
require.NoError(r, err)
require.NotEmpty(r, leader)
members, err := client.Agent().Members(false)
require.Len(r, members, 4)
})
waitForLeader(t, cluster)
waitForMembers(t, client, 4)
serviceName := "api"
err, index := serviceCreate(t, client, serviceName)
index := serviceCreate(t, client, serviceName)
ch := make(chan []*api.ServiceEntry)
errCh := make(chan error)
go func() {
service, q, err := client.Health().Service(serviceName, "", false, &api.QueryOptions{WaitIndex: index})
if q.QueryBackend != api.QueryBackendStreaming {
if err == nil && q.QueryBackend != api.QueryBackendStreaming {
err = fmt.Errorf("invalid backend for this test %s", q.QueryBackend)
}
if err != nil {
@ -197,7 +201,11 @@ func TestMixedServersMajorityTargetGAClient(t *testing.T) {
ch <- service
}
}()
err = client.Agent().ServiceRegister(&api.AgentServiceRegistration{Name: serviceName, Port: 9998})
require.NoError(t, client.Agent().ServiceRegister(
&api.AgentServiceRegistration{Name: serviceName, Port: 9998},
))
timer := time.NewTimer(1 * time.Second)
select {
case err := <-errCh:
@ -211,10 +219,10 @@ func TestMixedServersMajorityTargetGAClient(t *testing.T) {
}
}
func clientsCreate(numClients int, version string) ([]node.Node, error) {
func clientsCreate(t *testing.T, numClients int, version string) []node.Node {
clients := make([]node.Node, numClients)
var err error
for i := 0; i < numClients; i++ {
var err error
clients[i], err = node.NewConsulContainer(context.Background(),
node.Config{
HCL: `node_name="` + utils.RandName("consul-client") + `"
@ -222,23 +230,25 @@ func clientsCreate(numClients int, version string) ([]node.Node, error) {
Cmd: []string{"agent", "-client=0.0.0.0"},
Version: version,
})
require.NoError(t, err)
}
return clients, err
return clients
}
func serviceCreate(t *testing.T, client *api.Client, serviceName string) (error, uint64) {
func serviceCreate(t *testing.T, client *api.Client, serviceName string) uint64 {
err := client.Agent().ServiceRegister(&api.AgentServiceRegistration{Name: serviceName, Port: 9999})
require.NoError(t, err)
service, meta, err := client.Catalog().Service(serviceName, "", &api.QueryOptions{})
require.NoError(t, err)
require.Len(t, service, 1)
require.Equal(t, serviceName, service[0].ServiceName)
require.Equal(t, 9999, service[0].ServicePort)
return err, meta.LastIndex
return meta.LastIndex
}
func serversCluster(t *testing.T, numServers int, version string) (*cluster.Cluster, error) {
var err error
func serversCluster(t *testing.T, numServers int, version string) *cluster.Cluster {
var configs []node.Config
for i := 0; i < numServers; i++ {
configs = append(configs, node.Config{
@ -252,14 +262,11 @@ func serversCluster(t *testing.T, numServers int, version string) (*cluster.Clus
}
cluster, err := cluster.New(configs)
require.NoError(t, err)
retry.RunWith(&retry.Timer{Timeout: retryTimeout, Wait: retryFrequency}, t, func(r *retry.R) {
leader, err := cluster.Leader()
require.NoError(r, err)
require.NotEmpty(r, leader)
members, err := cluster.Nodes[0].GetClient().Agent().Members(false)
require.Len(r, members, numServers)
})
return cluster, err
waitForLeader(t, cluster)
waitForMembers(t, cluster.Nodes[0].GetClient(), numServers)
return cluster
}
func Terminate(t *testing.T, cluster *cluster.Cluster) {

35
test/integration/consul-container/upgrade/util_test.go Normal file
View File

@ -0,0 +1,35 @@
package upgrade
import (
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/integration/consul-container/libs/cluster"
"github.com/hashicorp/consul/sdk/testutil/retry"
)
const retryTimeout = 20 * time.Second
const retryFrequency = 500 * time.Millisecond
func LongFailer() *retry.Timer {
return &retry.Timer{Timeout: retryTimeout, Wait: retryFrequency}
}
func waitForLeader(t *testing.T, Cluster *cluster.Cluster) {
retry.RunWith(LongFailer(), t, func(r *retry.R) {
leader, err := Cluster.Leader()
require.NoError(r, err)
require.NotEmpty(r, leader)
})
}
func waitForMembers(t *testing.T, client *api.Client, expectN int) {
retry.RunWith(LongFailer(), t, func(r *retry.R) {
members, err := client.Agent().Members(false)
require.NoError(r, err)
require.Len(r, members, expectN)
})
}