luxolus
/
open-nomad
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

e2e/framework: add minimal godocs

This commit is contained in:
Nick Ethier 2018-06-28 15:55:53 -04:00
parent 0f15bab3ba
commit 026e86ecb9
No known key found for this signature in database
GPG Key ID: 07C1A3ECED90D24A
4 changed files with 74 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
e2e/framework/case.go
View File

@ -9,26 +9,27 @@ import (
"github.com/stretchr/testify/require"
)
// TestSuite defines a set of test cases and under what conditions to run them
type TestSuite struct {
Component string
Component string // Name of the component/system/feature tested
CanRunLocal bool
Cases []TestCase
Constraints Constraints
Parallel bool
Slow bool
CanRunLocal bool // Flags if the cases are safe to run on a local nomad cluster
Cases []TestCase // Cases to run
Constraints Constraints // Environment constraints to follow
Parallel bool // If true, will run test cases in parallel
Slow bool // Slow test suites don't run by default
}
type Constraints struct {
CloudProvider string
OS string
Arch string
Environment string
Tags []string
Provider string // Cloud provider ex. 'aws', 'azure', 'gcp'
OS string // Operating system ex. 'windows', 'linux'
Arch string // CPU architecture ex. 'amd64', 'arm64'
Environment string // Environment name ex. 'simple'
Tags []string // Generic tags that must all exist in the environment
}
func (c Constraints) matches(env Environment) error {
if len(c.CloudProvider) != 0 && c.CloudProvider != env.Provider {
if len(c.Provider) != 0 && c.Provider != env.Provider {
return fmt.Errorf("provider constraint does not match environment")
}
@ -52,6 +53,7 @@ func (c Constraints) matches(env Environment) error {
return nil
}
// TC is the base test case which should be embedded in TestCase implementations
type TC struct {
*assert.Assertions
require *require.Assertions
@ -62,22 +64,28 @@ type TC struct {
name string
}
// Nomad returns a configured nomad api client
func (tc *TC) Nomad() *api.Client {
return tc.cluster.NomadClient
}
// Prefix will return a test case unique prefix which can be used to scope resources
// during parallel tests.
func (tc *TC) Prefix() string {
return fmt.Sprintf("%s-", tc.cluster.ID)
}
// Name is the Name of the test cluster.
func (tc *TC) Name() string {
return tc.cluster.Name
}
// T retrieves the current *testing.T context
func (tc *TC) T() *testing.T {
return tc.t
}
// SetT sets the current *testing.T context
func (tc *TC) SetT(t *testing.T) {
tc.t = t
tc.Assertions = assert.New(t)

8
e2e/framework/doc.go Normal file
View File

@ -0,0 +1,8 @@
/*
Package framework implements a model for developing end-to-end test suites. The
model includes a top level Framework which TestSuites can be added to. TestSuites
include conditions under which the suite will run and a list of TestCase
implementations to run. TestCases can be implemented with methods that run
before/after each and all tests.
*/
package framework

49
e2e/framework/framework.go
View File

@ -8,11 +8,11 @@ import (
"testing"
)
var fProvider = flag.String("nomad.env.provider", "", "cloud provider for which environment is executing against")
var fEnv = flag.String("nomad.env", "", "name of the environment executing against")
var fOS = flag.String("nomad.os", "", "operating system for which the environment is executing against")
var fArch = flag.String("nomad.arch", "", "cpu architecture for which the environment is executing against")
var fTags = flag.String("nomad.tags", "", "comma delimited list of tags associated with the environment")
var fProvider = flag.String("nomad.env.provider", "", "cloud provider for which environment is executing against")
var fOS = flag.String("nomad.env.os", "", "operating system for which the environment is executing against")
var fArch = flag.String("nomad.env.arch", "", "cpu architecture for which the environment is executing against")
var fTags = flag.String("nomad.env.tags", "", "comma delimited list of tags associated with the environment")
var fLocal = flag.Bool("nomad.local", false, "denotes execution is against a local environment")
var fSlow = flag.Bool("nomad.slow", false, "toggles execution of slow test suites")
var fForceAll = flag.Bool("nomad.force", false, "if set, skips all environment checks when filtering test suites")
@ -29,6 +29,17 @@ type Framework struct {
force bool
}
// Environment includes information about the target environment the test
// framework is targeting. During 'go run', these fields are populated by
// the following flags:
//
// -nomad.env <string> "name of the environment executing against"
// -nomad.env.provider <string> "cloud provider for which the environment is executing against"
// -nomad.env.os <string> "operating system of environment"
// -nomad.env.arch <string> "cpu architecture of environment"
// -nomad.env.tags <string> "comma delimited list of environment tags"
//
// These flags are not needed when executing locally
type Environment struct {
Name string
Provider string
@ -37,6 +48,7 @@ type Environment struct {
Tags map[string]struct{}
}
// New creates a Framework
func New() *Framework {
env := Environment{
Name: *fEnv,
@ -57,17 +69,19 @@ func New() *Framework {
}
}
// AddSuites adds a set of test suites to a Framework
func (f *Framework) AddSuites(s ...*TestSuite) *Framework {
f.suites = append(f.suites, s...)
return f
}
// AddSuites adds a set of test suites to the package scoped Framework
func AddSuites(s ...*TestSuite) *Framework {
pkgFramework.AddSuites(s...)
return pkgFramework
}
// Run starts the test framework and runs each TestSuite
// Run starts the test framework, running each TestSuite
func (f *Framework) Run(t *testing.T) {
for _, s := range f.suites {
t.Run(s.Component, func(t *testing.T) {
@ -84,13 +98,18 @@ func (f *Framework) Run(t *testing.T) {
}
// Run starts the package scoped Framework, running each TestSuite
func Run(t *testing.T) {
pkgFramework.Run(t)
}
// runSuite is called from Framework.Run inside of a sub test for each TestSuite
// runSuite is called from Framework.Run inside of a sub test for each TestSuite.
// If skip is returned as true, the test suite is skipped with the error text added
// to the Skip reason
// If skip is false and an error is returned, the test suite is failed.
func (f *Framework) runSuite(t *testing.T, s *TestSuite) (skip bool, err error) {
// If -nomad.force is set, skip all constraint checks
if !f.force {
// If this is a local run, check that the suite supports running locally
if !s.CanRunLocal && f.isLocalRun {
@ -109,15 +128,17 @@ func (f *Framework) runSuite(t *testing.T, s *TestSuite) (skip bool, err error)
}
for _, c := range s.Cases {
// The test name is set to the name of the implementing type, including package
name := fmt.Sprintf("%T", c)
// Each TestCase is provisioned a nomad cluster
info, err := f.provisioner.ProvisionCluster(ProvisionerOptions{Name: name})
if err != nil {
return false, fmt.Errorf("could not provision cluster for case: %v", err)
}
defer f.provisioner.DestroyCluster(info.ID)
c.setClusterInfo(info)
// Each TestCase runs as a subtest of the TestSuite
t.Run(c.Name(), func(t *testing.T) {
c.SetT(t)
@ -139,14 +160,15 @@ func (f *Framework) runSuite(t *testing.T, s *TestSuite) (skip bool, err error)
}()
// Here we need to iterate through the methods of the case to find
// ones that at test functions
// ones that are test functions
reflectC := reflect.TypeOf(c)
for i := 0; i < reflectC.NumMethod(); i++ {
method := reflectC.Method(i)
if ok, _ := isTestMethod(method.Name); !ok {
if ok := isTestMethod(method.Name); !ok {
continue
}
// Each step is run as its own sub test of the case
// Test cases are never parallel
t.Run(method.Name, func(t *testing.T) {
// Since the test function interacts with testing.T through
@ -163,6 +185,7 @@ func (f *Framework) runSuite(t *testing.T, s *TestSuite) (skip bool, err error)
}
c.SetT(parentT)
}()
//Call the method
method.Func.Call([]reflect.Value{reflect.ValueOf(c)})
})
@ -174,10 +197,10 @@ func (f *Framework) runSuite(t *testing.T, s *TestSuite) (skip bool, err error)
return false, nil
}
func isTestMethod(m string) (bool, error) {
func isTestMethod(m string) bool {
if !strings.HasPrefix(m, "Test") {
return false, nil
return false
}
return true, nil
// THINKING: adding flag to target a specific step or step regex?
return true
}

17
e2e/framework/interfaces.go
View File

@ -4,16 +4,13 @@ import (
"testing"
)
// Named exists simply to make sure the Name() method was implemented since it
// is the only required method implementation of a test case
type Named interface {
Name() string
}
// TestCase is the interface which an E2E test case implements.
// It is not meant to be implemented directly, instead the struct should embed
// the 'framework.TC' struct
type TestCase interface {
Named
internalTestCase
Name() string
T() *testing.T
SetT(*testing.T)
}
@ -22,18 +19,24 @@ type internalTestCase interface {
setClusterInfo(*ClusterInfo)
}
// BeforeAllSteps is used to define a method to be called before the execution
// of all test steps.
type BeforeAllSteps interface {
BeforeAllSteps()
}
// AfterAllSteps is used to define a method to be called after the execution of
// all test steps.
type AfterAllSteps interface {
AfterAllSteps()
}
// BeforeEachStep is used to define a method to be called before each test step.
type BeforeEachStep interface {
BeforeEachStep()
}
// AfterEachStep is used to degine a method to be called after each test step.
type AfterEachStep interface {
AfterEachStep()
}