From 33bb3d6a840a6246180a497d5f1f0f5eda02b7df Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Thu, 12 Jul 2018 15:06:14 -0400 Subject: [PATCH 01/10] e2e/cli: initial cli implementation for nomad-e2e --- e2e/cli/command/provision.go | 275 ++++++++++++++++++ e2e/cli/command/run.go | 70 +++++ e2e/cli/command/scanner.go | 229 +++++++++++++++ e2e/cli/environments/mock/mock/main.tf | 5 + .../environments/mock/mock/terraform.tfvars | 1 + e2e/cli/main.go | 26 ++ e2e/example/example.go | 20 ++ 7 files changed, 626 insertions(+) create mode 100644 e2e/cli/command/provision.go create mode 100644 e2e/cli/command/run.go create mode 100644 e2e/cli/command/scanner.go create mode 100644 e2e/cli/environments/mock/mock/main.tf create mode 100644 e2e/cli/environments/mock/mock/terraform.tfvars create mode 100644 e2e/cli/main.go diff --git a/e2e/cli/command/provision.go b/e2e/cli/command/provision.go new file mode 100644 index 000000000..b9f5dcb06 --- /dev/null +++ b/e2e/cli/command/provision.go @@ -0,0 +1,275 @@ +package command + +import ( + "bufio" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "path" + "strings" + + getter "github.com/hashicorp/go-getter" + "github.com/mitchellh/cli" +) + +func init() { + getter.Getters["file"].(*getter.FileGetter).Copy = true +} + +func ProvisionCommandFactory() (cli.Command, error) { + return &Provision{}, nil +} + +type Provision struct { +} + +func (c *Provision) Help() string { + helpText := ` +Usage: nomad-e2e provision + + Uses terraform to provision a target test environment to use + for end-to-end testing. + + The output is a list of environment variables used to configure + various api clients such as Nomad, Consul and Vault. + +Provision Options: + + -env-path + Sets the path for where to search for test environment configuration. + This defaults to './environments/'. + + -nomad-binary + Sets the target nomad-binary to use when provisioning a nomad cluster. + The binary is retrieved by go-getter and can therefore be a local file + path, remote http url, or other support go-getter uri. + + -nomad-checksum + If set, will ensure the binary from -nomad-binary matches the given + checksum. + + -destroy + If set, will destroy the target environment. + + -tf-path + Sets the path for which terraform state files are stored. Defaults to + the current working directory. +` + return strings.TrimSpace(helpText) +} + +func (c *Provision) Synopsis() string { + return "Provisions the target testing environment" +} + +func (c *Provision) Run(args []string) int { + var envPath string + var nomadBinary string + var destroy bool + var tfPath string + cmdFlags := flag.NewFlagSet("provision", flag.ContinueOnError) + cmdFlags.Usage = func() { log.Println(c.Help()) } + cmdFlags.StringVar(&envPath, "env-path", "./environments/", "Path to e2e environment terraform configs") + cmdFlags.StringVar(&nomadBinary, "nomad-binary", "", "") + cmdFlags.BoolVar(&destroy, "destroy", false, "") + cmdFlags.StringVar(&tfPath, "tf-path", "", "") + + if err := cmdFlags.Parse(args); err != nil { + log.Fatalf("failed to parse flags: %v", err) + } + + args = cmdFlags.Args() + if len(args) != 2 { + log.Fatalf("expected 2 args, but got: %v", args) + log.Fatal(c.Help()) + } + + env, err := newEnv(envPath, args[0], args[1], tfPath) + if err != nil { + log.Fatal(err) + } + + if destroy { + if err := env.destroy(); err != nil { + log.Fatal(err) + return 1 + } + fmt.Println("Environment successfully destroyed") + return 0 + } + + // Use go-getter to fetch the nomad binary + nomadPath, err := c.fetchBinary(nomadBinary) + defer os.RemoveAll(nomadPath) + + results, err := env.provision(nomadPath) + if err != nil { + log.Fatal(err) + } + + fmt.Printf(strings.TrimSpace(` +NOMAD_ADDR=%s + `), results.nomadAddr) + + return 0 +} + +// Fetches the nomad binary and returns the temporary directory where it exists +func (c *Provision) fetchBinary(bin string) (string, error) { + nomadBinaryDir, err := ioutil.TempDir("", "") + if err != nil { + return "", fmt.Errorf("failed to create temp dir: %v", err) + } + + if err = getter.GetFile(path.Join(nomadBinaryDir, "nomad"), bin); err != nil { + return "", fmt.Errorf("failed to get nomad binary: %v", err) + } + + return nomadBinaryDir, nil +} + +type environment struct { + path string + provider string + name string + + tf string + tfPath string + tfState string +} + +type envResults struct { + nomadAddr string +} + +func newEnv(envPath, provider, name, tfStatePath string) (*environment, error) { + // Make sure terraform is on the PATH + tf, err := exec.LookPath("terraform") + if err != nil { + return nil, fmt.Errorf("failed to lookup terraform binary: %v", err) + } + + // set the path to the terraform module + tfPath := path.Join(envPath, provider, name) + log.Printf("[DEBUG] provision: using tf path %s", tfPath) + if _, err := os.Stat(tfPath); os.IsNotExist(err) { + return nil, fmt.Errorf("failed to lookup terraform configuration dir %s: %v", tfPath, err) + } + + // set the path to state file + tfState := path.Join(tfStatePath, fmt.Sprintf("e2e.%s.%s.tfstate", provider, name)) + + env := &environment{ + path: envPath, + provider: provider, + name: name, + tf: tf, + tfPath: tfPath, + tfState: tfState, + } + return env, nil +} + +// provision calls terraform to setup the environment with the given nomad binary +func (env *environment) provision(nomadPath string) (*envResults, error) { + tfArgs := []string{"apply", "-auto-approve", "-input=false", "-no-color", + "-state", env.tfState, + "-var", fmt.Sprintf("nomad_binary=%s", path.Join(nomadPath, "nomad")), + env.tfPath, + } + + // Setup the 'terraform apply' command + ctx := context.Background() + cmd := exec.CommandContext(ctx, env.tf, tfArgs...) + + // Funnel the stdout/stderr to logging + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, fmt.Errorf("failed to get stderr pipe: %v", err) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("failed to get stdout pipe: %v", err) + } + + // Run 'terraform apply' + cmd.Start() + go tfLog("tf.stderr", stderr) + go tfLog("tf.stdout", stdout) + + err = cmd.Wait() + if err != nil { + return nil, fmt.Errorf("terraform exited with a non-zero status: %v", err) + } + + // Setup and run 'terraform output' to get the module output + cmd = exec.CommandContext(ctx, env.tf, "output", "-json", "-state", env.tfState) + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("terraform exited with a non-zero status: %v", err) + } + + // Parse the json and pull out results + tfOutput := make(map[string]map[string]interface{}) + err = json.Unmarshal(out, &tfOutput) + if err != nil { + return nil, fmt.Errorf("failed to parse terraform output: %v", err) + } + + results := &envResults{} + if nomadAddr, ok := tfOutput["nomad_addr"]; ok { + results.nomadAddr = nomadAddr["value"].(string) + } + + return results, nil +} + +//destroy calls terraform to destroy the environment +func (env *environment) destroy() error { + tfArgs := []string{"destroy", "-auto-approve", "-no-color", + "-state", env.tfState, + "-var", "nomad_binary=", + env.tfPath, + } + cmd := exec.Command(env.tf, tfArgs...) + + // Funnel the stdout/stderr to logging + stderr, err := cmd.StderrPipe() + if err != nil { + return fmt.Errorf("failed to get stderr pipe: %v", err) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return fmt.Errorf("failed to get stdout pipe: %v", err) + } + + // Run 'terraform destroy' + cmd.Start() + go tfLog("tf.stderr", stderr) + go tfLog("tf.stdout", stdout) + + err = cmd.Wait() + if err != nil { + return fmt.Errorf("terraform exited with a non-zero status: %v", err) + } + + return nil +} + +func tfLog(prefix string, r io.ReadCloser) { + defer r.Close() + scanner := bufio.NewScanner(r) + for scanner.Scan() { + log.Printf("[DEBUG] provision.%s: %s", prefix, scanner.Text()) + } + if err := scanner.Err(); err != nil { + log.Printf("[WARN] provision.%s: %v", err) + } + +} diff --git a/e2e/cli/command/run.go b/e2e/cli/command/run.go new file mode 100644 index 000000000..52703832b --- /dev/null +++ b/e2e/cli/command/run.go @@ -0,0 +1,70 @@ +package command + +import ( + "fmt" + "os/exec" + "strings" + + "github.com/mitchellh/cli" +) + +func RunCommandFactory() (cli.Command, error) { + return &Run{}, nil +} + +type Run struct { +} + +func (c *Run) Help() string { + helpText := ` +Usage: nomad-e2e run +` + return strings.TrimSpace(helpText) +} + +func (c *Run) Synopsis() string { + return "Runs the e2e test suite" +} + +func (c *Run) Run(args []string) int { + if err := c.run(); err != nil { + fmt.Println(err) + return 1 + } + return 0 +} + +func (c *Run) run() error { + goBin, err := exec.LookPath("go") + if err != nil { + return err + } + + goArgs := []string{ + "test", + "-json", + "github.com/hashicorp/nomad/e2e", + } + + cmd := exec.Command(goBin, goArgs...) + out, err := cmd.StdoutPipe() + defer out.Close() + if err != nil { + return err + } + + err = cmd.Start() + if err != nil { + return err + } + + dec := NewDecoder(out) + report, err := dec.Decode() + if err != nil { + return err + } + + fmt.Println(report.Summary()) + + return nil +} diff --git a/e2e/cli/command/scanner.go b/e2e/cli/command/scanner.go new file mode 100644 index 000000000..3c96ec789 --- /dev/null +++ b/e2e/cli/command/scanner.go @@ -0,0 +1,229 @@ +package command + +import ( + "encoding/json" + "fmt" + "io" + "strings" + "text/tabwriter" + "time" + + "github.com/fatih/color" +) + +type EventDecoder struct { + r io.Reader + + dec *json.Decoder + report *TestReport +} + +type TestReport struct { + Events []*TestEvent + Suites map[string]*TestSuite + TotalSuites int + TotalFailedSuites int + TotalCases int + TotalFailedCases int + TotalTests int + TotalFailedTests int + Elapsed float64 + Output []string +} + +type TestEvent struct { + Time time.Time // encodes as an RFC3339-format string + Action string + Package string + Test string + Elapsed float64 // seconds + Output string + + eventType string + suiteName string + caseName string + testName string +} + +type TestSuite struct { + Name string + Cases map[string]*TestCase + Failed int + Elapsed float64 + Output []string +} + +type TestCase struct { + Name string + Tests map[string]*Test + Failed int + Elapsed float64 + Output []string +} + +type Test struct { + Name string + Output []string + Failed bool + Elapsed float64 +} + +func NewDecoder(r io.Reader) *EventDecoder { + return &EventDecoder{ + r: r, + dec: json.NewDecoder(r), + report: &TestReport{ + Suites: map[string]*TestSuite{}, + Events: []*TestEvent{}, + }, + } +} + +func (d *EventDecoder) Decode() (*TestReport, error) { + for d.dec.More() { + var e TestEvent + err := d.dec.Decode(&e) + if err != nil { + return nil, err + } + + d.report.record(&e) + } + return d.report, nil +} + +func (r *TestReport) record(event *TestEvent) { + if !strings.HasPrefix(event.Test, "TestE2E") { + return + } + parts := strings.Split(event.Test, "/") + switch len(parts) { + case 1: + r.recordRoot(event) + case 2: + event.suiteName = parts[1] + r.recordSuite(event) + case 3: + event.suiteName = parts[1] + event.caseName = parts[2] + r.recordCase(event, r.Suites[event.suiteName]) + case 4: + event.suiteName = parts[1] + event.caseName = parts[2] + event.testName = strings.Join(parts[3:], "/") + suite := r.Suites[event.suiteName] + r.recordTest(event, suite, suite.Cases[event.caseName]) + } + r.Events = append(r.Events, event) + +} + +func (r *TestReport) recordRoot(event *TestEvent) { + switch event.Action { + case "run": + case "output": + r.Output = append(r.Output, event.Output) + case "pass", "fail": + r.Elapsed = event.Elapsed + } +} +func (r *TestReport) recordSuite(event *TestEvent) { + switch event.Action { + case "run": + r.Suites[event.suiteName] = &TestSuite{ + Name: event.suiteName, + Cases: map[string]*TestCase{}, + } + r.TotalSuites += 1 + case "output": + r.Suites[event.suiteName].Output = append(r.Suites[event.suiteName].Output, event.Output) + case "pass": + r.Suites[event.suiteName].Elapsed = event.Elapsed + case "fail": + r.Suites[event.suiteName].Elapsed = event.Elapsed + r.TotalFailedSuites += 1 + } +} +func (r *TestReport) recordCase(event *TestEvent, suite *TestSuite) { + switch event.Action { + case "run": + suite.Cases[event.caseName] = &TestCase{ + Name: event.caseName, + Tests: map[string]*Test{}, + } + r.TotalCases += 1 + case "output": + suite.Cases[event.caseName].Output = append(suite.Cases[event.caseName].Output, event.Output) + case "pass": + suite.Cases[event.caseName].Elapsed = event.Elapsed + case "fail": + suite.Cases[event.caseName].Elapsed = event.Elapsed + suite.Failed += 1 + r.TotalFailedCases += 1 + } +} +func (r *TestReport) recordTest(event *TestEvent, suite *TestSuite, c *TestCase) { + switch event.Action { + case "run": + c.Tests[event.testName] = &Test{ + Name: event.testName, + } + r.TotalTests += 1 + case "output": + c.Tests[event.testName].Output = append(c.Tests[event.testName].Output, event.Output) + case "pass": + c.Tests[event.testName].Elapsed = event.Elapsed + case "fail": + c.Tests[event.testName].Elapsed = event.Elapsed + c.Tests[event.testName].Failed = true + c.Failed += 1 + r.TotalFailedTests += 1 + } +} + +func (r *TestReport) Summary() string { + green := color.New(color.FgGreen).SprintFunc() + red := color.New(color.FgRed).SprintFunc() + + sb := strings.Builder{} + sb.WriteString( + fmt.Sprintf("Summary: %v/%v suites failed | %v/%v cases failed | %v/%v tests failed\n", + r.TotalFailedSuites, r.TotalSuites, + r.TotalFailedCases, r.TotalCases, + r.TotalFailedTests, r.TotalTests)) + + sb.WriteString("Details:\n") + w := tabwriter.NewWriter(&sb, 0, 0, 1, ' ', tabwriter.AlignRight) + for sname, suite := range r.Suites { + status := red("FAIL") + if suite.Failed == 0 { + status = green("PASS") + } + + fmt.Fprintf(w, "[%s]\t%s\t\t\t (%vs)\n", status, sname, suite.Elapsed) + for cname, c := range suite.Cases { + status := red("FAIL") + if c.Failed == 0 { + status = green("PASS") + } + fmt.Fprintf(w, "[%s]\t↳\t%s\t\t (%vs)\n", status, cname, c.Elapsed) + for tname, test := range c.Tests { + status := red("FAIL") + if !test.Failed { + status = green("PASS") + } + fmt.Fprintf(w, "[%s]\t\t↳\t%s\t (%vs)\n", status, tname, test.Elapsed) + if test.Failed { + for _, line := range test.Output[2:] { + fmt.Fprintf(w, "\t\t\t%s\n", strings.Replace(strings.TrimSpace(line), "\t", " ", -1)) + } + fmt.Fprintln(w, "\t\t\t----------") + } + + } + } + } + + w.Flush() + return sb.String() +} diff --git a/e2e/cli/environments/mock/mock/main.tf b/e2e/cli/environments/mock/mock/main.tf new file mode 100644 index 000000000..5129d951c --- /dev/null +++ b/e2e/cli/environments/mock/mock/main.tf @@ -0,0 +1,5 @@ +variable "nomad_binary" {} + +output "nomad_addr" { + value = "${var.nomad_binary}" +} diff --git a/e2e/cli/environments/mock/mock/terraform.tfvars b/e2e/cli/environments/mock/mock/terraform.tfvars new file mode 100644 index 000000000..19286d33f --- /dev/null +++ b/e2e/cli/environments/mock/mock/terraform.tfvars @@ -0,0 +1 @@ +nomad_addr = "http://localhost:4646" diff --git a/e2e/cli/main.go b/e2e/cli/main.go new file mode 100644 index 000000000..4da5b6873 --- /dev/null +++ b/e2e/cli/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "log" + "os" + + "github.com/hashicorp/nomad/e2e/cli/command" + "github.com/mitchellh/cli" +) + +func main() { + log.SetPrefix("@@@ ==> ") + + c := cli.NewCLI("nomad-e2e", "0.0.1") + c.Args = os.Args[1:] + c.Commands = map[string]cli.CommandFactory{ + "provision": command.ProvisionCommandFactory, + "run": command.RunCommandFactory, + } + + exitStatus, err := c.Run() + if err != nil { + log.Println(err) + } + os.Exit(exitStatus) +} diff --git a/e2e/example/example.go b/e2e/example/example.go index 3884e8b51..09ff6eff7 100644 --- a/e2e/example/example.go +++ b/e2e/example/example.go @@ -1,6 +1,8 @@ package example import ( + "time" + "github.com/hashicorp/nomad/e2e/framework" ) @@ -10,6 +12,7 @@ func init() { CanRunLocal: true, Cases: []framework.TestCase{ new(SimpleExampleTestCase), + new(ExampleLongSetupCase), }, }) } @@ -19,7 +22,24 @@ type SimpleExampleTestCase struct { } func (tc *SimpleExampleTestCase) TestExample(f *framework.F) { + f.T().Log("Logging foo") jobs, _, err := tc.Nomad().Jobs().List(nil) f.NoError(err) f.Empty(jobs) } + +func (tc *SimpleExampleTestCase) TestPassExample(f *framework.F) { + f.T().Log("all good here") +} + +type ExampleLongSetupCase struct { + framework.TC +} + +func (tc *ExampleLongSetupCase) BeforeEach(f *framework.F) { + time.Sleep(5 * time.Second) +} + +func (tc *ExampleLongSetupCase) TestPass(f *framework.F) { + +} From d44f5cbf4e50ddfccf00e0a27246cdc69856164f Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Tue, 24 Jul 2018 12:21:48 -0400 Subject: [PATCH 02/10] e2e/cli: use discover utility for nomad binary --- e2e/cli/command/provision.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/e2e/cli/command/provision.go b/e2e/cli/command/provision.go index b9f5dcb06..dbebb4446 100644 --- a/e2e/cli/command/provision.go +++ b/e2e/cli/command/provision.go @@ -15,6 +15,7 @@ import ( "strings" getter "github.com/hashicorp/go-getter" + "github.com/hashicorp/nomad/helper/discover" "github.com/mitchellh/cli" ) @@ -127,6 +128,12 @@ func (c *Provision) fetchBinary(bin string) (string, error) { return "", fmt.Errorf("failed to create temp dir: %v", err) } + if bin == "" { + bin, err = discover.NomadExecutable() + if err != nil { + return "", fmt.Errorf("failed to discover nomad binary: %v", err) + } + } if err = getter.GetFile(path.Join(nomadBinaryDir, "nomad"), bin); err != nil { return "", fmt.Errorf("failed to get nomad binary: %v", err) } From 8abf4f9e25aa46919ea8fb8a689d000772fd2830 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Thu, 26 Jul 2018 00:03:23 -0400 Subject: [PATCH 03/10] e2e/cli: implemented run logic --- e2e/cli/command/provision.go | 39 +++- e2e/cli/command/run.go | 178 ++++++++++++++++-- .../command/{scanner.go => test_decoder.go} | 5 +- 3 files changed, 197 insertions(+), 25 deletions(-) rename e2e/cli/command/{scanner.go => test_decoder.go} (97%) diff --git a/e2e/cli/command/provision.go b/e2e/cli/command/provision.go index dbebb4446..60d5c3bfc 100644 --- a/e2e/cli/command/provision.go +++ b/e2e/cli/command/provision.go @@ -12,6 +12,7 @@ import ( "os" "os/exec" "path" + "path/filepath" "strings" getter "github.com/hashicorp/go-getter" @@ -87,8 +88,7 @@ func (c *Provision) Run(args []string) int { args = cmdFlags.Args() if len(args) != 2 { - log.Fatalf("expected 2 args, but got: %v", args) - log.Fatal(c.Help()) + log.Fatalf("expected 2 args (provider and environment), but got: %v", args) } env, err := newEnv(envPath, args[0], args[1], tfPath) @@ -101,12 +101,12 @@ func (c *Provision) Run(args []string) int { log.Fatal(err) return 1 } - fmt.Println("Environment successfully destroyed") + log.Println("Environment successfully destroyed") return 0 } // Use go-getter to fetch the nomad binary - nomadPath, err := c.fetchBinary(nomadBinary) + nomadPath, err := fetchBinary(nomadBinary) defer os.RemoveAll(nomadPath) results, err := env.provision(nomadPath) @@ -122,7 +122,7 @@ NOMAD_ADDR=%s } // Fetches the nomad binary and returns the temporary directory where it exists -func (c *Provision) fetchBinary(bin string) (string, error) { +func fetchBinary(bin string) (string, error) { nomadBinaryDir, err := ioutil.TempDir("", "") if err != nil { return "", fmt.Errorf("failed to create temp dir: %v", err) @@ -152,7 +152,9 @@ type environment struct { } type envResults struct { - nomadAddr string + nomadAddr string + consulAddr string + vaultAddr string } func newEnv(envPath, provider, name, tfStatePath string) (*environment, error) { @@ -183,6 +185,31 @@ func newEnv(envPath, provider, name, tfStatePath string) (*environment, error) { return env, nil } +// envsFromGlob allows for the discovery of multiple environments using globs (*). +// ex. aws/* for all environments in aws. +func envsFromGlob(envPath, glob, tfStatePath string) ([]*environment, error) { + results, err := filepath.Glob(filepath.Join(envPath, glob)) + if err != nil { + return nil, err + } + + envs := []*environment{} + + for _, p := range results { + elems := strings.Split(p, "/") + name := elems[len(elems)-1] + provider := elems[len(elems)-2] + env, err := newEnv(envPath, provider, name, tfStatePath) + if err != nil { + return nil, err + } + + envs = append(envs, env) + } + + return envs, nil +} + // provision calls terraform to setup the environment with the given nomad binary func (env *environment) provision(nomadPath string) (*envResults, error) { tfArgs := []string{"apply", "-auto-approve", "-input=false", "-no-color", diff --git a/e2e/cli/command/run.go b/e2e/cli/command/run.go index 52703832b..58c34dd1e 100644 --- a/e2e/cli/command/run.go +++ b/e2e/cli/command/run.go @@ -1,10 +1,16 @@ package command import ( + "flag" "fmt" + "io" + "log" + "os" "os/exec" "strings" + capi "github.com/hashicorp/consul/api" + vapi "github.com/hashicorp/vault/api" "github.com/mitchellh/cli" ) @@ -27,44 +33,180 @@ func (c *Run) Synopsis() string { } func (c *Run) Run(args []string) int { - if err := c.run(); err != nil { - fmt.Println(err) - return 1 + var envPath string + var nomadBinary string + var tfPath string + var slow bool + var run string + var verbose bool + cmdFlags := flag.NewFlagSet("run", flag.ContinueOnError) + cmdFlags.Usage = func() { log.Println(c.Help()) } + cmdFlags.StringVar(&envPath, "env-path", "./environments/", "Path to e2e environment terraform configs") + cmdFlags.StringVar(&nomadBinary, "nomad-binary", "", "") + cmdFlags.StringVar(&tfPath, "tf-path", "", "") + cmdFlags.StringVar(&run, "run", "", "Regex to target specific test suites/cases") + cmdFlags.BoolVar(&slow, "slow", false, "Toggle slow running suites") + cmdFlags.BoolVar(&verbose, "v", false, "Toggle verbose output") + + if err := cmdFlags.Parse(args); err != nil { + log.Fatalf("failed to parse flags: %v", err) + } + + args = cmdFlags.Args() + + if len(args) == 0 { + log.Println("No environments specified, running test suite locally...") + var report *TestReport + var err error + if report, err = c.run(&runOpts{ + slow: slow, + verbose: verbose, + }); err != nil { + log.Fatalf("failed to run test suite: %v", err) + } + if report.TotalFailedTests == 0 { + log.Println("PASSED!") + if verbose { + log.Println(report.Summary()) + } + } else { + log.Println("***FAILED***") + log.Println(report.Summary()) + } + return 0 + } + + environments := []*environment{} + for _, e := range args { + if len(strings.Split(e, "/")) != 2 { + log.Fatalf("argument %s should be formated as /", e) + } + envs, err := envsFromGlob(envPath, e, tfPath) + if err != nil { + log.Fatalf("failed to build environment %s: %v", e, err) + } + environments = append(environments, envs...) + + } + envCount := len(environments) + // Use go-getter to fetch the nomad binary + nomadPath, err := fetchBinary(nomadBinary) + defer os.RemoveAll(nomadPath) + if err != nil { + log.Fatal("failed to fetch nomad binary: %v", err) + } + + log.Printf("Running tests against %d environments...", envCount) + for i, env := range environments { + log.Printf("[%d/%d] provisioning %s environment on %s provider", i+1, envCount, env.name, env.provider) + results, err := env.provision(nomadPath) + if err != nil { + log.Fatalf("failed to provision environment %s/%s: %v", env.provider, env.name, err) + } + + opts := &runOpts{ + provider: env.provider, + env: env.name, + slow: slow, + verbose: verbose, + nomadAddr: results.nomadAddr, + consulAddr: results.consulAddr, + vaultAddr: results.vaultAddr, + } + + var report *TestReport + if report, err = c.run(opts); err != nil { + log.Printf("failed to run tests against environment %s/%s: %v", env.provider, env.name, err) + return 1 + } + if report.TotalFailedTests == 0 { + log.Printf("[%d/%d] %s/%s: PASSED!\n", i, envCount, env.provider, env.name) + if verbose { + log.Printf("[%d/%d] %s/%s: %s", i, envCount, env.provider, env.name, report.Summary()) + } + } else { + log.Printf("[%d/%d] %s/%s: ***FAILED***\n", i, envCount, env.provider, env.name) + log.Printf("[%d/%d] %s/%s: %s", i, envCount, env.provider, env.name, report.Summary()) + } } return 0 } -func (c *Run) run() error { +func (c *Run) run(opts *runOpts) (*TestReport, error) { goBin, err := exec.LookPath("go") if err != nil { - return err + return nil, err } - goArgs := []string{ - "test", - "-json", - "github.com/hashicorp/nomad/e2e", - } - - cmd := exec.Command(goBin, goArgs...) + cmd := exec.Command(goBin, opts.goArgs()...) + cmd.Env = opts.goEnv() out, err := cmd.StdoutPipe() defer out.Close() if err != nil { - return err + return nil, err } err = cmd.Start() if err != nil { - return err + return nil, err + } + + var logger io.Writer + if opts.verbose { + logger = os.Stdout } dec := NewDecoder(out) - report, err := dec.Decode() + report, err := dec.Decode(logger) if err != nil { - return err + return nil, err } - fmt.Println(report.Summary()) + return report, nil - return nil +} + +type runOpts struct { + nomadAddr string + consulAddr string + vaultAddr string + provider string + env string + local bool + slow bool + verbose bool +} + +func (opts *runOpts) goArgs() []string { + a := []string{ + "test", + "-json", + "github.com/hashicorp/nomad/e2e", + "-env=" + opts.env, + "-env.provider=" + opts.provider, + } + + if opts.slow { + a = append(a, "-slow") + } + + if opts.local { + a = append(a, "-local") + } + return a +} + +func (opts *runOpts) goEnv() []string { + env := append(os.Environ(), "NOMAD_E2E=1") + if opts.nomadAddr != "" { + env = append(env, "NOMAD_ADDR="+opts.nomadAddr) + } + if opts.consulAddr != "" { + env = append(env, fmt.Sprintf("%s=%s", capi.HTTPAddrEnvName, opts.consulAddr)) + } + if opts.vaultAddr != "" { + env = append(env, fmt.Sprintf("%s=%s", vapi.EnvVaultAddress, opts.consulAddr)) + } + + return env } diff --git a/e2e/cli/command/scanner.go b/e2e/cli/command/test_decoder.go similarity index 97% rename from e2e/cli/command/scanner.go rename to e2e/cli/command/test_decoder.go index 3c96ec789..58a5d2d0c 100644 --- a/e2e/cli/command/scanner.go +++ b/e2e/cli/command/test_decoder.go @@ -79,7 +79,7 @@ func NewDecoder(r io.Reader) *EventDecoder { } } -func (d *EventDecoder) Decode() (*TestReport, error) { +func (d *EventDecoder) Decode(logger io.Writer) (*TestReport, error) { for d.dec.More() { var e TestEvent err := d.dec.Decode(&e) @@ -88,6 +88,9 @@ func (d *EventDecoder) Decode() (*TestReport, error) { } d.report.record(&e) + if logger != nil { + logger.Write([]byte(e.Output)) + } } return d.report, nil } From bba732b2c3c774951ec1f817b44aa6ed065aedc8 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Thu, 26 Jul 2018 22:26:04 -0400 Subject: [PATCH 04/10] vendor: update github.com/hashicorp/go-hclog --- .../github.com/hashicorp/go-hclog/README.md | 14 ++- vendor/github.com/hashicorp/go-hclog/int.go | 112 ++++++++++++++---- vendor/github.com/hashicorp/go-hclog/log.go | 27 ++++- .../hashicorp/go-hclog/nulllogger.go | 47 ++++++++ vendor/vendor.json | 2 +- 5 files changed, 177 insertions(+), 25 deletions(-) create mode 100644 vendor/github.com/hashicorp/go-hclog/nulllogger.go diff --git a/vendor/github.com/hashicorp/go-hclog/README.md b/vendor/github.com/hashicorp/go-hclog/README.md index 614342b2d..1153e2853 100644 --- a/vendor/github.com/hashicorp/go-hclog/README.md +++ b/vendor/github.com/hashicorp/go-hclog/README.md @@ -10,8 +10,7 @@ interface for use in development and production environments. It provides logging levels that provide decreased output based upon the desired amount of output, unlike the standard library `log` package. -It does not provide `Printf` style logging, only key/value logging that is -exposed as arguments to the logging functions for simplicity. +It provides `Printf` style logging of values via `hclog.Fmt()`. It provides a human readable output mode for use in development as well as JSON output mode for production. @@ -100,6 +99,17 @@ requestLogger.Info("we are transporting a request") This allows sub Loggers to be context specific without having to thread that into all the callers. +### Using `hclog.Fmt()` + +```go +var int totalBandwidth = 200 +appLogger.Info("total bandwidth exceeded", "bandwidth", hclog.Fmt("%d GB/s", totalBandwidth)) +``` + +```text +... [INFO ] my-app: total bandwidth exceeded: bandwidth="200 GB/s" +``` + ### Use this with code that uses the standard library logger If you want to use the standard library's `log.Logger` interface you can wrap diff --git a/vendor/github.com/hashicorp/go-hclog/int.go b/vendor/github.com/hashicorp/go-hclog/int.go index 9f90c2877..7d17d81cb 100644 --- a/vendor/github.com/hashicorp/go-hclog/int.go +++ b/vendor/github.com/hashicorp/go-hclog/int.go @@ -2,14 +2,17 @@ package hclog import ( "bufio" + "encoding" "encoding/json" "fmt" "log" "os" "runtime" + "sort" "strconv" "strings" "sync" + "sync/atomic" "time" ) @@ -39,28 +42,40 @@ func New(opts *LoggerOptions) Logger { level = DefaultLevel } - return &intLogger{ - m: new(sync.Mutex), - json: opts.JSONFormat, - caller: opts.IncludeLocation, - name: opts.Name, - w: bufio.NewWriter(output), - level: level, + mtx := opts.Mutex + if mtx == nil { + mtx = new(sync.Mutex) } + + ret := &intLogger{ + m: mtx, + json: opts.JSONFormat, + caller: opts.IncludeLocation, + name: opts.Name, + timeFormat: TimeFormat, + w: bufio.NewWriter(output), + level: new(int32), + } + if opts.TimeFormat != "" { + ret.timeFormat = opts.TimeFormat + } + atomic.StoreInt32(ret.level, int32(level)) + return ret } // The internal logger implementation. Internal in that it is defined entirely // by this package. type intLogger struct { - json bool - caller bool - name string + json bool + caller bool + name string + timeFormat string // this is a pointer so that it's shared by any derived loggers, since // those derived loggers share the bufio.Writer as well. m *sync.Mutex w *bufio.Writer - level Level + level *int32 implied []interface{} } @@ -75,7 +90,7 @@ const TimeFormat = "2006-01-02T15:04:05.000Z0700" // Log a message and a set of key/value pairs if the given level is at // or more severe that the threshold configured in the Logger. func (z *intLogger) Log(level Level, msg string, args ...interface{}) { - if level < z.level { + if level < Level(atomic.LoadInt32(z.level)) { return } @@ -126,7 +141,7 @@ func trimCallerPath(path string) string { // Non-JSON logging format function func (z *intLogger) log(t time.Time, level Level, msg string, args ...interface{}) { - z.w.WriteString(t.Format(TimeFormat)) + z.w.WriteString(t.Format(z.timeFormat)) z.w.WriteByte(' ') s, ok := _levelToBracket[level] @@ -202,6 +217,8 @@ func (z *intLogger) log(t time.Time, level Level, msg string, args ...interface{ case CapturedStacktrace: stacktrace = st continue FOR + case Format: + val = fmt.Sprintf(st[0].(string), st[1:]...) default: val = fmt.Sprintf("%v", st) } @@ -262,6 +279,8 @@ func (z *intLogger) logJson(t time.Time, level Level, msg string, args ...interf } } + args = append(z.implied, args...) + if args != nil && len(args) > 0 { if len(args)%2 != 0 { cs, ok := args[len(args)-1].(CapturedStacktrace) @@ -279,7 +298,22 @@ func (z *intLogger) logJson(t time.Time, level Level, msg string, args ...interf // without injecting into logs... continue } - vals[args[i].(string)] = args[i+1] + val := args[i+1] + switch sv := val.(type) { + case error: + // Check if val is of type error. If error type doesn't + // implement json.Marshaler or encoding.TextMarshaler + // then set val to err.Error() so that it gets marshaled + switch sv.(type) { + case json.Marshaler, encoding.TextMarshaler: + default: + val = sv.Error() + } + case Format: + val = fmt.Sprintf(sv[0].(string), sv[1:]...) + } + + vals[args[i].(string)] = val } } @@ -316,36 +350,66 @@ func (z *intLogger) Error(msg string, args ...interface{}) { // Indicate that the logger would emit TRACE level logs func (z *intLogger) IsTrace() bool { - return z.level == Trace + return Level(atomic.LoadInt32(z.level)) == Trace } // Indicate that the logger would emit DEBUG level logs func (z *intLogger) IsDebug() bool { - return z.level <= Debug + return Level(atomic.LoadInt32(z.level)) <= Debug } // Indicate that the logger would emit INFO level logs func (z *intLogger) IsInfo() bool { - return z.level <= Info + return Level(atomic.LoadInt32(z.level)) <= Info } // Indicate that the logger would emit WARN level logs func (z *intLogger) IsWarn() bool { - return z.level <= Warn + return Level(atomic.LoadInt32(z.level)) <= Warn } // Indicate that the logger would emit ERROR level logs func (z *intLogger) IsError() bool { - return z.level <= Error + return Level(atomic.LoadInt32(z.level)) <= Error } // Return a sub-Logger for which every emitted log message will contain // the given key/value pairs. This is used to create a context specific // Logger. func (z *intLogger) With(args ...interface{}) Logger { + if len(args)%2 != 0 { + panic("With() call requires paired arguments") + } + var nz intLogger = *z - nz.implied = append(nz.implied, args...) + result := make(map[string]interface{}, len(z.implied)+len(args)) + keys := make([]string, 0, len(z.implied)+len(args)) + + // Read existing args, store map and key for consistent sorting + for i := 0; i < len(z.implied); i += 2 { + key := z.implied[i].(string) + keys = append(keys, key) + result[key] = z.implied[i+1] + } + // Read new args, store map and key for consistent sorting + for i := 0; i < len(args); i += 2 { + key := args[i].(string) + _, exists := result[key] + if !exists { + keys = append(keys, key) + } + result[key] = args[i+1] + } + + // Sort keys to be consistent + sort.Strings(keys) + + nz.implied = make([]interface{}, 0, len(z.implied)+len(args)) + for _, k := range keys { + nz.implied = append(nz.implied, k) + nz.implied = append(nz.implied, result[k]) + } return &nz } @@ -357,6 +421,8 @@ func (z *intLogger) Named(name string) Logger { if nz.name != "" { nz.name = nz.name + "." + name + } else { + nz.name = name } return &nz @@ -373,6 +439,12 @@ func (z *intLogger) ResetNamed(name string) Logger { return &nz } +// Update the logging level on-the-fly. This will affect all subloggers as +// well. +func (z *intLogger) SetLevel(level Level) { + atomic.StoreInt32(z.level, int32(level)) +} + // Create a *log.Logger that will send it's data through this Logger. This // allows packages that expect to be using the standard library log to actually // use this logger. diff --git a/vendor/github.com/hashicorp/go-hclog/log.go b/vendor/github.com/hashicorp/go-hclog/log.go index 6bb16ba75..894e8461b 100644 --- a/vendor/github.com/hashicorp/go-hclog/log.go +++ b/vendor/github.com/hashicorp/go-hclog/log.go @@ -5,6 +5,7 @@ import ( "log" "os" "strings" + "sync" ) var ( @@ -12,7 +13,7 @@ var ( DefaultLevel = Info ) -type Level int +type Level int32 const ( // This is a special level used to indicate that no level has been @@ -36,6 +37,18 @@ const ( Error Level = 5 ) +// When processing a value of this type, the logger automatically treats the first +// argument as a Printf formatting string and passes the rest as the values to be +// formatted. For example: L.Info(Fmt{"%d beans/day", beans}). This is a simple +// convience type for when formatting is required. +type Format []interface{} + +// Fmt returns a Format type. This is a convience function for creating a Format +// type. +func Fmt(str string, args ...interface{}) Format { + return append(Format{str}, args...) +} + // LevelFromString returns a Level type for the named log level, or "NoLevel" if // the level string is invalid. This facilitates setting the log level via // config or environment variable by name in a predictable way. @@ -108,6 +121,10 @@ type Logger interface { // the current name as well. ResetNamed(name string) Logger + // Updates the level. This should affect all sub-loggers as well. If an + // implementation cannot update the level on the fly, it should no-op. + SetLevel(level Level) + // Return a value that conforms to the stdlib log.Logger interface StandardLogger(opts *StandardLoggerOptions) *log.Logger } @@ -130,9 +147,15 @@ type LoggerOptions struct { // Where to write the logs to. Defaults to os.Stdout if nil Output io.Writer + // An optional mutex pointer in case Output is shared + Mutex *sync.Mutex + // Control if the output should be in JSON. JSONFormat bool - // Intclude file and line information in each log line + // Include file and line information in each log line IncludeLocation bool + + // The time format to use instead of the default + TimeFormat string } diff --git a/vendor/github.com/hashicorp/go-hclog/nulllogger.go b/vendor/github.com/hashicorp/go-hclog/nulllogger.go new file mode 100644 index 000000000..0942361a5 --- /dev/null +++ b/vendor/github.com/hashicorp/go-hclog/nulllogger.go @@ -0,0 +1,47 @@ +package hclog + +import ( + "io/ioutil" + "log" +) + +// NewNullLogger instantiates a Logger for which all calls +// will succeed without doing anything. +// Useful for testing purposes. +func NewNullLogger() Logger { + return &nullLogger{} +} + +type nullLogger struct{} + +func (l *nullLogger) Trace(msg string, args ...interface{}) {} + +func (l *nullLogger) Debug(msg string, args ...interface{}) {} + +func (l *nullLogger) Info(msg string, args ...interface{}) {} + +func (l *nullLogger) Warn(msg string, args ...interface{}) {} + +func (l *nullLogger) Error(msg string, args ...interface{}) {} + +func (l *nullLogger) IsTrace() bool { return false } + +func (l *nullLogger) IsDebug() bool { return false } + +func (l *nullLogger) IsInfo() bool { return false } + +func (l *nullLogger) IsWarn() bool { return false } + +func (l *nullLogger) IsError() bool { return false } + +func (l *nullLogger) With(args ...interface{}) Logger { return l } + +func (l *nullLogger) Named(name string) Logger { return l } + +func (l *nullLogger) ResetNamed(name string) Logger { return l } + +func (l *nullLogger) SetLevel(level Level) {} + +func (l *nullLogger) StandardLogger(opts *StandardLoggerOptions) *log.Logger { + return log.New(ioutil.Discard, "", log.LstdFlags) +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 1834b356c..3f9215f66 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -150,7 +150,7 @@ {"path":"github.com/hashicorp/go-envparse","checksumSHA1":"FKmqR4DC3nCXtnT9pe02z5CLNWo=","revision":"310ca1881b22af3522e3a8638c0b426629886196","revisionTime":"2018-01-19T21:58:41Z"}, {"path":"github.com/hashicorp/go-getter","checksumSHA1":"uGH6AI982csQJoRPsSooK7FoWqo=","revision":"3f60ec5cfbb2a39731571b9ddae54b303bb0a969","revisionTime":"2018-04-25T22:41:30Z"}, {"path":"github.com/hashicorp/go-getter/helper/url","checksumSHA1":"9J+kDr29yDrwsdu2ULzewmqGjpA=","revision":"b345bfcec894fb7ff3fdf9b21baf2f56ea423d98","revisionTime":"2018-04-10T17:49:45Z"}, - {"path":"github.com/hashicorp/go-hclog","checksumSHA1":"miVF4/7JP0lRwZvFJGKwZWk7aAQ=","revision":"b4e5765d1e5f00a0550911084f45f8214b5b83b9","revisionTime":"2017-07-16T17:45:23Z"}, + {"path":"github.com/hashicorp/go-hclog","checksumSHA1":"dOP7kCX3dACHc9mU79826N411QA=","revision":"ff2cf002a8dd750586d91dddd4470c341f981fe1","revisionTime":"2018-07-09T16:53:50Z"}, {"path":"github.com/hashicorp/go-immutable-radix","checksumSHA1":"Cas2nprG6pWzf05A2F/OlnjUu2Y=","revision":"8aac2701530899b64bdea735a1de8da899815220","revisionTime":"2017-07-25T22:12:15Z"}, {"path":"github.com/hashicorp/go-memdb","checksumSHA1":"FMAvwDar2bQyYAW4XMFhAt0J5xA=","revision":"20ff6434c1cc49b80963d45bf5c6aa89c78d8d57","revisionTime":"2017-08-31T20:15:40Z"}, {"path":"github.com/hashicorp/go-msgpack/codec","revision":"fa3f63826f7c23912c15263591e65d54d080b458"}, From 77018c677429fac0bbf8d5e0e54b141fa3206f2e Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Thu, 26 Jul 2018 23:11:58 -0400 Subject: [PATCH 05/10] e2e/cli: port logging to use hclog --- e2e/cli/command/meta.go | 22 +++++++++ e2e/cli/command/provision.go | 64 +++++++++++++++---------- e2e/cli/command/run.go | 85 ++++++++++++++++++--------------- e2e/cli/command/test_decoder.go | 7 +-- e2e/cli/main.go | 20 ++++++-- 5 files changed, 128 insertions(+), 70 deletions(-) create mode 100644 e2e/cli/command/meta.go diff --git a/e2e/cli/command/meta.go b/e2e/cli/command/meta.go new file mode 100644 index 000000000..478e57d64 --- /dev/null +++ b/e2e/cli/command/meta.go @@ -0,0 +1,22 @@ +package command + +import ( + "flag" + + hclog "github.com/hashicorp/go-hclog" + "github.com/mitchellh/cli" +) + +type Meta struct { + Ui cli.Ui + logger hclog.Logger + + verbose bool +} + +func (m *Meta) FlagSet(n string) *flag.FlagSet { + f := flag.NewFlagSet(n, flag.ContinueOnError) + + f.BoolVar(&m.verbose, "v", false, "Toggle verbose output") + return f +} diff --git a/e2e/cli/command/provision.go b/e2e/cli/command/provision.go index 60d5c3bfc..42951e9c8 100644 --- a/e2e/cli/command/provision.go +++ b/e2e/cli/command/provision.go @@ -4,11 +4,9 @@ import ( "bufio" "context" "encoding/json" - "flag" "fmt" "io" "io/ioutil" - "log" "os" "os/exec" "path" @@ -16,6 +14,7 @@ import ( "strings" getter "github.com/hashicorp/go-getter" + hclog "github.com/hashicorp/go-hclog" "github.com/hashicorp/nomad/helper/discover" "github.com/mitchellh/cli" ) @@ -24,11 +23,18 @@ func init() { getter.Getters["file"].(*getter.FileGetter).Copy = true } -func ProvisionCommandFactory() (cli.Command, error) { - return &Provision{}, nil +func ProvisionCommandFactory(ui cli.Ui, logger hclog.Logger) cli.CommandFactory { + return func() (cli.Command, error) { + meta := Meta{ + Ui: ui, + logger: logger, + } + return &Provision{Meta: meta}, nil + } } type Provision struct { + Meta } func (c *Provision) Help() string { @@ -75,33 +81,38 @@ func (c *Provision) Run(args []string) int { var nomadBinary string var destroy bool var tfPath string - cmdFlags := flag.NewFlagSet("provision", flag.ContinueOnError) - cmdFlags.Usage = func() { log.Println(c.Help()) } + cmdFlags := c.FlagSet("provision") + cmdFlags.Usage = func() { c.Ui.Output(c.Help()) } cmdFlags.StringVar(&envPath, "env-path", "./environments/", "Path to e2e environment terraform configs") cmdFlags.StringVar(&nomadBinary, "nomad-binary", "", "") cmdFlags.BoolVar(&destroy, "destroy", false, "") cmdFlags.StringVar(&tfPath, "tf-path", "", "") if err := cmdFlags.Parse(args); err != nil { - log.Fatalf("failed to parse flags: %v", err) + c.logger.Error("failed to parse flags:", "error", err) + return 1 + } + if c.verbose { + c.logger.SetLevel(hclog.Debug) } args = cmdFlags.Args() if len(args) != 2 { - log.Fatalf("expected 2 args (provider and environment), but got: %v", args) + c.logger.Error("expected 2 args (provider and environment)", "args", args) } - env, err := newEnv(envPath, args[0], args[1], tfPath) + env, err := newEnv(envPath, args[0], args[1], tfPath, c.logger) if err != nil { - log.Fatal(err) + c.logger.Error("failed to build environment", "error", err) + return 1 } if destroy { if err := env.destroy(); err != nil { - log.Fatal(err) + c.logger.Error("failed to destroy environment", "error", err) return 1 } - log.Println("Environment successfully destroyed") + c.logger.Debug("environment successfully destroyed") return 0 } @@ -111,7 +122,8 @@ func (c *Provision) Run(args []string) int { results, err := env.provision(nomadPath) if err != nil { - log.Fatal(err) + c.logger.Error("", "error", err) + return 1 } fmt.Printf(strings.TrimSpace(` @@ -149,6 +161,7 @@ type environment struct { tf string tfPath string tfState string + logger hclog.Logger } type envResults struct { @@ -157,16 +170,18 @@ type envResults struct { vaultAddr string } -func newEnv(envPath, provider, name, tfStatePath string) (*environment, error) { +func newEnv(envPath, provider, name, tfStatePath string, logger hclog.Logger) (*environment, error) { // Make sure terraform is on the PATH tf, err := exec.LookPath("terraform") if err != nil { return nil, fmt.Errorf("failed to lookup terraform binary: %v", err) } + logger = logger.Named("provision").With("provider", provider, "name", name) + // set the path to the terraform module tfPath := path.Join(envPath, provider, name) - log.Printf("[DEBUG] provision: using tf path %s", tfPath) + logger.Debug("using tf path", "path", tfPath) if _, err := os.Stat(tfPath); os.IsNotExist(err) { return nil, fmt.Errorf("failed to lookup terraform configuration dir %s: %v", tfPath, err) } @@ -181,13 +196,14 @@ func newEnv(envPath, provider, name, tfStatePath string) (*environment, error) { tf: tf, tfPath: tfPath, tfState: tfState, + logger: logger, } return env, nil } // envsFromGlob allows for the discovery of multiple environments using globs (*). // ex. aws/* for all environments in aws. -func envsFromGlob(envPath, glob, tfStatePath string) ([]*environment, error) { +func envsFromGlob(envPath, glob, tfStatePath string, logger hclog.Logger) ([]*environment, error) { results, err := filepath.Glob(filepath.Join(envPath, glob)) if err != nil { return nil, err @@ -199,7 +215,7 @@ func envsFromGlob(envPath, glob, tfStatePath string) ([]*environment, error) { elems := strings.Split(p, "/") name := elems[len(elems)-1] provider := elems[len(elems)-2] - env, err := newEnv(envPath, provider, name, tfStatePath) + env, err := newEnv(envPath, provider, name, tfStatePath, logger) if err != nil { return nil, err } @@ -234,8 +250,8 @@ func (env *environment) provision(nomadPath string) (*envResults, error) { // Run 'terraform apply' cmd.Start() - go tfLog("tf.stderr", stderr) - go tfLog("tf.stdout", stdout) + go tfLog(env.logger.Named("tf.stderr"), stderr) + go tfLog(env.logger.Named("tf.stdout"), stdout) err = cmd.Wait() if err != nil { @@ -285,8 +301,8 @@ func (env *environment) destroy() error { // Run 'terraform destroy' cmd.Start() - go tfLog("tf.stderr", stderr) - go tfLog("tf.stdout", stdout) + go tfLog(env.logger.Named("tf.stderr"), stderr) + go tfLog(env.logger.Named("tf.stdout"), stdout) err = cmd.Wait() if err != nil { @@ -296,14 +312,14 @@ func (env *environment) destroy() error { return nil } -func tfLog(prefix string, r io.ReadCloser) { +func tfLog(logger hclog.Logger, r io.ReadCloser) { defer r.Close() scanner := bufio.NewScanner(r) for scanner.Scan() { - log.Printf("[DEBUG] provision.%s: %s", prefix, scanner.Text()) + logger.Debug(scanner.Text()) } if err := scanner.Err(); err != nil { - log.Printf("[WARN] provision.%s: %v", err) + logger.Error("scan error", "error", err) } } diff --git a/e2e/cli/command/run.go b/e2e/cli/command/run.go index 58c34dd1e..53c35d67d 100644 --- a/e2e/cli/command/run.go +++ b/e2e/cli/command/run.go @@ -1,24 +1,29 @@ package command import ( - "flag" "fmt" - "io" - "log" "os" "os/exec" "strings" capi "github.com/hashicorp/consul/api" + hclog "github.com/hashicorp/go-hclog" vapi "github.com/hashicorp/vault/api" "github.com/mitchellh/cli" ) -func RunCommandFactory() (cli.Command, error) { - return &Run{}, nil +func RunCommandFactory(ui cli.Ui, logger hclog.Logger) cli.CommandFactory { + return func() (cli.Command, error) { + meta := Meta{ + Ui: ui, + logger: logger, + } + return &Run{Meta: meta}, nil + } } type Run struct { + Meta } func (c *Run) Help() string { @@ -38,40 +43,43 @@ func (c *Run) Run(args []string) int { var tfPath string var slow bool var run string - var verbose bool - cmdFlags := flag.NewFlagSet("run", flag.ContinueOnError) - cmdFlags.Usage = func() { log.Println(c.Help()) } + cmdFlags := c.FlagSet("run") + cmdFlags.Usage = func() { c.Ui.Output(c.Help()) } cmdFlags.StringVar(&envPath, "env-path", "./environments/", "Path to e2e environment terraform configs") cmdFlags.StringVar(&nomadBinary, "nomad-binary", "", "") cmdFlags.StringVar(&tfPath, "tf-path", "", "") cmdFlags.StringVar(&run, "run", "", "Regex to target specific test suites/cases") cmdFlags.BoolVar(&slow, "slow", false, "Toggle slow running suites") - cmdFlags.BoolVar(&verbose, "v", false, "Toggle verbose output") if err := cmdFlags.Parse(args); err != nil { - log.Fatalf("failed to parse flags: %v", err) + c.logger.Error("failed to parse flags", "error", err) + return 1 + } + if c.verbose { + c.logger.SetLevel(hclog.Debug) } args = cmdFlags.Args() if len(args) == 0 { - log.Println("No environments specified, running test suite locally...") + c.logger.Info("no environments specified, running test suite locally") var report *TestReport var err error if report, err = c.run(&runOpts{ slow: slow, - verbose: verbose, + verbose: c.verbose, }); err != nil { - log.Fatalf("failed to run test suite: %v", err) + c.logger.Error("failed to run test suite", "error", err) + return 1 } if report.TotalFailedTests == 0 { - log.Println("PASSED!") - if verbose { - log.Println(report.Summary()) + c.Ui.Output("PASSED!") + if c.verbose { + c.Ui.Output(report.Summary()) } } else { - log.Println("***FAILED***") - log.Println(report.Summary()) + c.Ui.Output("***FAILED***") + c.Ui.Output(report.Summary()) } return 0 } @@ -79,11 +87,13 @@ func (c *Run) Run(args []string) int { environments := []*environment{} for _, e := range args { if len(strings.Split(e, "/")) != 2 { - log.Fatalf("argument %s should be formated as /", e) + c.logger.Error("argument should be formated as /", "args", e) + return 1 } - envs, err := envsFromGlob(envPath, e, tfPath) + envs, err := envsFromGlob(envPath, e, tfPath, c.logger) if err != nil { - log.Fatalf("failed to build environment %s: %v", e, err) + c.logger.Error("failed to build environment", "environment", e, "error", err) + return 1 } environments = append(environments, envs...) @@ -93,22 +103,25 @@ func (c *Run) Run(args []string) int { nomadPath, err := fetchBinary(nomadBinary) defer os.RemoveAll(nomadPath) if err != nil { - log.Fatal("failed to fetch nomad binary: %v", err) + c.logger.Error("failed to fetch nomad binary", "error", err) + return 1 } - log.Printf("Running tests against %d environments...", envCount) + c.logger.Debug("starting tests", "totalEnvironments", envCount) for i, env := range environments { - log.Printf("[%d/%d] provisioning %s environment on %s provider", i+1, envCount, env.name, env.provider) + logger := c.logger.With("name", env.name, "provider", env.provider) + logger.Debug("provisioning environment") results, err := env.provision(nomadPath) if err != nil { - log.Fatalf("failed to provision environment %s/%s: %v", env.provider, env.name, err) + logger.Error("failed to provision environment", "error", err) + return 1 } opts := &runOpts{ provider: env.provider, env: env.name, slow: slow, - verbose: verbose, + verbose: c.verbose, nomadAddr: results.nomadAddr, consulAddr: results.consulAddr, vaultAddr: results.vaultAddr, @@ -116,17 +129,18 @@ func (c *Run) Run(args []string) int { var report *TestReport if report, err = c.run(opts); err != nil { - log.Printf("failed to run tests against environment %s/%s: %v", env.provider, env.name, err) + logger.Error("failed to run tests against environment", "error", err) return 1 } if report.TotalFailedTests == 0 { - log.Printf("[%d/%d] %s/%s: PASSED!\n", i, envCount, env.provider, env.name) - if verbose { - log.Printf("[%d/%d] %s/%s: %s", i, envCount, env.provider, env.name, report.Summary()) + + c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: PASSED!\n", i+1, envCount, env.provider, env.name)) + if c.verbose { + c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: %s", i+1, envCount, env.provider, env.name, report.Summary())) } } else { - log.Printf("[%d/%d] %s/%s: ***FAILED***\n", i, envCount, env.provider, env.name) - log.Printf("[%d/%d] %s/%s: %s", i, envCount, env.provider, env.name, report.Summary()) + c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: ***FAILED***\n", i+1, envCount, env.provider, env.name)) + c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: %s", i+1, envCount, env.provider, env.name, report.Summary())) } } return 0 @@ -151,13 +165,8 @@ func (c *Run) run(opts *runOpts) (*TestReport, error) { return nil, err } - var logger io.Writer - if opts.verbose { - logger = os.Stdout - } - dec := NewDecoder(out) - report, err := dec.Decode(logger) + report, err := dec.Decode(c.logger.Named("run.gotest")) if err != nil { return nil, err } diff --git a/e2e/cli/command/test_decoder.go b/e2e/cli/command/test_decoder.go index 58a5d2d0c..776b7778f 100644 --- a/e2e/cli/command/test_decoder.go +++ b/e2e/cli/command/test_decoder.go @@ -9,6 +9,7 @@ import ( "time" "github.com/fatih/color" + hclog "github.com/hashicorp/go-hclog" ) type EventDecoder struct { @@ -79,7 +80,7 @@ func NewDecoder(r io.Reader) *EventDecoder { } } -func (d *EventDecoder) Decode(logger io.Writer) (*TestReport, error) { +func (d *EventDecoder) Decode(logger hclog.Logger) (*TestReport, error) { for d.dec.More() { var e TestEvent err := d.dec.Decode(&e) @@ -88,8 +89,8 @@ func (d *EventDecoder) Decode(logger io.Writer) (*TestReport, error) { } d.report.record(&e) - if logger != nil { - logger.Write([]byte(e.Output)) + if logger != nil && e.Output != "" { + logger.Debug(strings.TrimRight(e.Output, "\n")) } } return d.report, nil diff --git a/e2e/cli/main.go b/e2e/cli/main.go index 4da5b6873..f2340d5f0 100644 --- a/e2e/cli/main.go +++ b/e2e/cli/main.go @@ -1,26 +1,36 @@ package main import ( - "log" "os" + hclog "github.com/hashicorp/go-hclog" "github.com/hashicorp/nomad/e2e/cli/command" "github.com/mitchellh/cli" ) func main() { - log.SetPrefix("@@@ ==> ") + + ui := &cli.BasicUi{ + Reader: os.Stdin, + Writer: os.Stdout, + ErrorWriter: os.Stderr, + } + + logger := hclog.New(&hclog.LoggerOptions{ + Name: "nomad-e2e", + Output: &cli.UiWriter{ui}, + }) c := cli.NewCLI("nomad-e2e", "0.0.1") c.Args = os.Args[1:] c.Commands = map[string]cli.CommandFactory{ - "provision": command.ProvisionCommandFactory, - "run": command.RunCommandFactory, + "provision": command.ProvisionCommandFactory(ui, logger), + "run": command.RunCommandFactory(ui, logger), } exitStatus, err := c.Run() if err != nil { - log.Println(err) + logger.Error("command exited with non-zero status", "status", exitStatus, "error", err) } os.Exit(exitStatus) } From 01dd1f5d65f9479fc2774a1c049220f42fae9772 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Thu, 26 Jul 2018 23:49:53 -0400 Subject: [PATCH 06/10] e2e/cli: fix formatting --- e2e/cli/command/run.go | 4 ++-- e2e/example/example.go | 13 ++++--------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/e2e/cli/command/run.go b/e2e/cli/command/run.go index 53c35d67d..76001baa4 100644 --- a/e2e/cli/command/run.go +++ b/e2e/cli/command/run.go @@ -134,12 +134,12 @@ func (c *Run) Run(args []string) int { } if report.TotalFailedTests == 0 { - c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: PASSED!\n", i+1, envCount, env.provider, env.name)) + c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: PASSED!", i+1, envCount, env.provider, env.name)) if c.verbose { c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: %s", i+1, envCount, env.provider, env.name, report.Summary())) } } else { - c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: ***FAILED***\n", i+1, envCount, env.provider, env.name)) + c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: ***FAILED***", i+1, envCount, env.provider, env.name)) c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: %s", i+1, envCount, env.provider, env.name, report.Summary())) } } diff --git a/e2e/example/example.go b/e2e/example/example.go index 09ff6eff7..c2d2956d4 100644 --- a/e2e/example/example.go +++ b/e2e/example/example.go @@ -1,8 +1,6 @@ package example import ( - "time" - "github.com/hashicorp/nomad/e2e/framework" ) @@ -28,8 +26,9 @@ func (tc *SimpleExampleTestCase) TestExample(f *framework.F) { f.Empty(jobs) } -func (tc *SimpleExampleTestCase) TestPassExample(f *framework.F) { - f.T().Log("all good here") +func (tc *SimpleExampleTestCase) TestParallelExample(f *framework.F) { + f.T().Log("this one can run in parallel with other tests") + f.T().Parallel() } type ExampleLongSetupCase struct { @@ -37,9 +36,5 @@ type ExampleLongSetupCase struct { } func (tc *ExampleLongSetupCase) BeforeEach(f *framework.F) { - time.Sleep(5 * time.Second) -} - -func (tc *ExampleLongSetupCase) TestPass(f *framework.F) { - + f.T().Log("Logging before each") } From 69220df24a82ab9043e43a27fa5976be74883383 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Tue, 31 Jul 2018 11:41:20 -0400 Subject: [PATCH 07/10] e2e/cli: code review comments, restructing and cleanup --- e2e/cli/command/environment.go | 222 +++++++++++++++++++++++++++++++++ e2e/cli/command/meta.go | 7 ++ e2e/cli/command/provision.go | 218 ++------------------------------ e2e/cli/command/run.go | 69 +++++----- e2e/cli/command/util.go | 55 ++++++++ e2e/cli/main.go | 17 ++- 6 files changed, 348 insertions(+), 240 deletions(-) create mode 100644 e2e/cli/command/environment.go create mode 100644 e2e/cli/command/util.go diff --git a/e2e/cli/command/environment.go b/e2e/cli/command/environment.go new file mode 100644 index 000000000..da288adb7 --- /dev/null +++ b/e2e/cli/command/environment.go @@ -0,0 +1,222 @@ +package command + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "os/signal" + "path" + "path/filepath" + "strings" + "syscall" + "time" + + hclog "github.com/hashicorp/go-hclog" +) + +// environment captures all the information needed to execute terraform +// in order to setup a test environment +type environment struct { + provider string // provider ex. aws + name string // environment name ex. generic + + tf string // location of terraform binary + tfPath string // path to terraform configuration + tfState string // path to terraform state file + logger hclog.Logger +} + +func (env *environment) canonicalName() string { + return fmt.Sprintf("%s/%s", env.provider, env.name) +} + +// envResults are the fields returned after provisioning a test environment +type envResults struct { + nomadAddr string + consulAddr string + vaultAddr string +} + +// newEnv takes a path to the environments directory, environment name and provider, +// path to terraform state file and a logger and builds the environment stuct used +// to initial terraform calls +func newEnv(envPath, provider, name, tfStatePath string, logger hclog.Logger) (*environment, error) { + // Make sure terraform is on the PATH + tf, err := exec.LookPath("terraform") + if err != nil { + return nil, fmt.Errorf("failed to lookup terraform binary: %v", err) + } + + logger = logger.Named("provision").With("provider", provider, "name", name) + + // set the path to the terraform module + tfPath := path.Join(envPath, provider, name) + logger.Debug("using tf path", "path", tfPath) + if _, err := os.Stat(tfPath); os.IsNotExist(err) { + return nil, fmt.Errorf("failed to lookup terraform configuration dir %s: %v", tfPath, err) + } + + // set the path to state file + tfState := path.Join(tfStatePath, fmt.Sprintf("e2e.%s.%s.tfstate", provider, name)) + + env := &environment{ + provider: provider, + name: name, + tf: tf, + tfPath: tfPath, + tfState: tfState, + logger: logger, + } + return env, nil +} + +// envsFromGlob allows for the discovery of multiple environments using globs (*). +// ex. aws/* for all environments in aws. +func envsFromGlob(envPath, glob, tfStatePath string, logger hclog.Logger) ([]*environment, error) { + results, err := filepath.Glob(filepath.Join(envPath, glob)) + if err != nil { + return nil, err + } + + envs := []*environment{} + + for _, p := range results { + elems := strings.Split(p, "/") + name := elems[len(elems)-1] + provider := elems[len(elems)-2] + env, err := newEnv(envPath, provider, name, tfStatePath, logger) + if err != nil { + return nil, err + } + + envs = append(envs, env) + } + + return envs, nil +} + +// provision calls terraform to setup the environment with the given nomad binary +func (env *environment) provision(nomadPath string) (*envResults, error) { + tfArgs := []string{"apply", "-auto-approve", "-input=false", "-no-color", + "-state", env.tfState, + "-var", fmt.Sprintf("nomad_binary=%s", path.Join(nomadPath, "nomad")), + env.tfPath, + } + + // Setup the 'terraform apply' command + ctx := context.Background() + cmd := exec.CommandContext(ctx, env.tf, tfArgs...) + + // Funnel the stdout/stderr to logging + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, fmt.Errorf("failed to get stderr pipe: %v", err) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("failed to get stdout pipe: %v", err) + } + + // Run 'terraform apply' + cmd.Start() + go tfLog(env.logger.Named("tf.stderr"), stderr) + go tfLog(env.logger.Named("tf.stdout"), stdout) + + sigChan := make(chan os.Signal) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + + cmdChan := make(chan error) + go func() { + cmdChan <- cmd.Wait() + }() + + // if an interrupt is received before terraform finished, forward signal to + // child pid + select { + case sig := <-sigChan: + env.logger.Error("interrupt received, forwarding signal to child process", + "pid", cmd.Process.Pid) + cmd.Process.Signal(sig) + if err := procWaitTimeout(cmd.Process, 5*time.Second); err != nil { + env.logger.Error("child process did not exit in time, killing forcefully", + "pid", cmd.Process.Pid) + cmd.Process.Kill() + } + return nil, fmt.Errorf("interrupt received") + case err := <-cmdChan: + if err != nil { + return nil, fmt.Errorf("terraform exited with a non-zero status: %v", err) + } + } + + // Setup and run 'terraform output' to get the module output + cmd = exec.CommandContext(ctx, env.tf, "output", "-json", "-state", env.tfState) + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("terraform exited with a non-zero status: %v", err) + } + + // Parse the json and pull out results + tfOutput := make(map[string]map[string]interface{}) + err = json.Unmarshal(out, &tfOutput) + if err != nil { + return nil, fmt.Errorf("failed to parse terraform output: %v", err) + } + + results := &envResults{} + if nomadAddr, ok := tfOutput["nomad_addr"]; ok { + results.nomadAddr = nomadAddr["value"].(string) + } else { + return nil, fmt.Errorf("'nomad_addr' field expected in terraform output, but was missing") + } + + return results, nil +} + +// destroy calls terraform to destroy the environment +func (env *environment) destroy() error { + tfArgs := []string{"destroy", "-auto-approve", "-no-color", + "-state", env.tfState, + "-var", "nomad_binary=", + env.tfPath, + } + cmd := exec.Command(env.tf, tfArgs...) + + // Funnel the stdout/stderr to logging + stderr, err := cmd.StderrPipe() + if err != nil { + return fmt.Errorf("failed to get stderr pipe: %v", err) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return fmt.Errorf("failed to get stdout pipe: %v", err) + } + + // Run 'terraform destroy' + cmd.Start() + go tfLog(env.logger.Named("tf.stderr"), stderr) + go tfLog(env.logger.Named("tf.stdout"), stdout) + + err = cmd.Wait() + if err != nil { + return fmt.Errorf("terraform exited with a non-zero status: %v", err) + } + + return nil +} + +func tfLog(logger hclog.Logger, r io.ReadCloser) { + defer r.Close() + scanner := bufio.NewScanner(r) + for scanner.Scan() { + logger.Debug(scanner.Text()) + } + if err := scanner.Err(); err != nil { + logger.Error("scan error", "error", err) + } + +} diff --git a/e2e/cli/command/meta.go b/e2e/cli/command/meta.go index 478e57d64..8dbd3cdda 100644 --- a/e2e/cli/command/meta.go +++ b/e2e/cli/command/meta.go @@ -14,6 +14,13 @@ type Meta struct { verbose bool } +func NewMeta(ui cli.Ui, logger hclog.Logger) Meta { + return Meta{ + Ui: ui, + logger: logger, + } +} + func (m *Meta) FlagSet(n string) *flag.FlagSet { f := flag.NewFlagSet(n, flag.ContinueOnError) diff --git a/e2e/cli/command/provision.go b/e2e/cli/command/provision.go index 42951e9c8..926beaf05 100644 --- a/e2e/cli/command/provision.go +++ b/e2e/cli/command/provision.go @@ -1,34 +1,25 @@ package command import ( - "bufio" - "context" - "encoding/json" "fmt" - "io" - "io/ioutil" "os" - "os/exec" - "path" - "path/filepath" "strings" getter "github.com/hashicorp/go-getter" hclog "github.com/hashicorp/go-hclog" - "github.com/hashicorp/nomad/helper/discover" "github.com/mitchellh/cli" ) +const ( + DefaultEnvironmentsPath = "./environments/" +) + func init() { getter.Getters["file"].(*getter.FileGetter).Copy = true } -func ProvisionCommandFactory(ui cli.Ui, logger hclog.Logger) cli.CommandFactory { +func ProvisionCommandFactory(meta Meta) cli.CommandFactory { return func() (cli.Command, error) { - meta := Meta{ - Ui: ui, - logger: logger, - } return &Provision{Meta: meta}, nil } } @@ -51,7 +42,7 @@ Provision Options: -env-path Sets the path for where to search for test environment configuration. - This defaults to './environments/'. + This defaults to './environments/'. -nomad-binary Sets the target nomad-binary to use when provisioning a nomad cluster. @@ -83,7 +74,7 @@ func (c *Provision) Run(args []string) int { var tfPath string cmdFlags := c.FlagSet("provision") cmdFlags.Usage = func() { c.Ui.Output(c.Help()) } - cmdFlags.StringVar(&envPath, "env-path", "./environments/", "Path to e2e environment terraform configs") + cmdFlags.StringVar(&envPath, "env-path", DefaultEnvironmentsPath, "Path to e2e environment terraform configs") cmdFlags.StringVar(&nomadBinary, "nomad-binary", "", "") cmdFlags.BoolVar(&destroy, "destroy", false, "") cmdFlags.StringVar(&tfPath, "tf-path", "", "") @@ -119,6 +110,10 @@ func (c *Provision) Run(args []string) int { // Use go-getter to fetch the nomad binary nomadPath, err := fetchBinary(nomadBinary) defer os.RemoveAll(nomadPath) + if err != nil { + c.logger.Error("failed to fetch nomad binary", "error", err) + return 1 + } results, err := env.provision(nomadPath) if err != nil { @@ -132,194 +127,3 @@ NOMAD_ADDR=%s return 0 } - -// Fetches the nomad binary and returns the temporary directory where it exists -func fetchBinary(bin string) (string, error) { - nomadBinaryDir, err := ioutil.TempDir("", "") - if err != nil { - return "", fmt.Errorf("failed to create temp dir: %v", err) - } - - if bin == "" { - bin, err = discover.NomadExecutable() - if err != nil { - return "", fmt.Errorf("failed to discover nomad binary: %v", err) - } - } - if err = getter.GetFile(path.Join(nomadBinaryDir, "nomad"), bin); err != nil { - return "", fmt.Errorf("failed to get nomad binary: %v", err) - } - - return nomadBinaryDir, nil -} - -type environment struct { - path string - provider string - name string - - tf string - tfPath string - tfState string - logger hclog.Logger -} - -type envResults struct { - nomadAddr string - consulAddr string - vaultAddr string -} - -func newEnv(envPath, provider, name, tfStatePath string, logger hclog.Logger) (*environment, error) { - // Make sure terraform is on the PATH - tf, err := exec.LookPath("terraform") - if err != nil { - return nil, fmt.Errorf("failed to lookup terraform binary: %v", err) - } - - logger = logger.Named("provision").With("provider", provider, "name", name) - - // set the path to the terraform module - tfPath := path.Join(envPath, provider, name) - logger.Debug("using tf path", "path", tfPath) - if _, err := os.Stat(tfPath); os.IsNotExist(err) { - return nil, fmt.Errorf("failed to lookup terraform configuration dir %s: %v", tfPath, err) - } - - // set the path to state file - tfState := path.Join(tfStatePath, fmt.Sprintf("e2e.%s.%s.tfstate", provider, name)) - - env := &environment{ - path: envPath, - provider: provider, - name: name, - tf: tf, - tfPath: tfPath, - tfState: tfState, - logger: logger, - } - return env, nil -} - -// envsFromGlob allows for the discovery of multiple environments using globs (*). -// ex. aws/* for all environments in aws. -func envsFromGlob(envPath, glob, tfStatePath string, logger hclog.Logger) ([]*environment, error) { - results, err := filepath.Glob(filepath.Join(envPath, glob)) - if err != nil { - return nil, err - } - - envs := []*environment{} - - for _, p := range results { - elems := strings.Split(p, "/") - name := elems[len(elems)-1] - provider := elems[len(elems)-2] - env, err := newEnv(envPath, provider, name, tfStatePath, logger) - if err != nil { - return nil, err - } - - envs = append(envs, env) - } - - return envs, nil -} - -// provision calls terraform to setup the environment with the given nomad binary -func (env *environment) provision(nomadPath string) (*envResults, error) { - tfArgs := []string{"apply", "-auto-approve", "-input=false", "-no-color", - "-state", env.tfState, - "-var", fmt.Sprintf("nomad_binary=%s", path.Join(nomadPath, "nomad")), - env.tfPath, - } - - // Setup the 'terraform apply' command - ctx := context.Background() - cmd := exec.CommandContext(ctx, env.tf, tfArgs...) - - // Funnel the stdout/stderr to logging - stderr, err := cmd.StderrPipe() - if err != nil { - return nil, fmt.Errorf("failed to get stderr pipe: %v", err) - } - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, fmt.Errorf("failed to get stdout pipe: %v", err) - } - - // Run 'terraform apply' - cmd.Start() - go tfLog(env.logger.Named("tf.stderr"), stderr) - go tfLog(env.logger.Named("tf.stdout"), stdout) - - err = cmd.Wait() - if err != nil { - return nil, fmt.Errorf("terraform exited with a non-zero status: %v", err) - } - - // Setup and run 'terraform output' to get the module output - cmd = exec.CommandContext(ctx, env.tf, "output", "-json", "-state", env.tfState) - out, err := cmd.Output() - if err != nil { - return nil, fmt.Errorf("terraform exited with a non-zero status: %v", err) - } - - // Parse the json and pull out results - tfOutput := make(map[string]map[string]interface{}) - err = json.Unmarshal(out, &tfOutput) - if err != nil { - return nil, fmt.Errorf("failed to parse terraform output: %v", err) - } - - results := &envResults{} - if nomadAddr, ok := tfOutput["nomad_addr"]; ok { - results.nomadAddr = nomadAddr["value"].(string) - } - - return results, nil -} - -//destroy calls terraform to destroy the environment -func (env *environment) destroy() error { - tfArgs := []string{"destroy", "-auto-approve", "-no-color", - "-state", env.tfState, - "-var", "nomad_binary=", - env.tfPath, - } - cmd := exec.Command(env.tf, tfArgs...) - - // Funnel the stdout/stderr to logging - stderr, err := cmd.StderrPipe() - if err != nil { - return fmt.Errorf("failed to get stderr pipe: %v", err) - } - stdout, err := cmd.StdoutPipe() - if err != nil { - return fmt.Errorf("failed to get stdout pipe: %v", err) - } - - // Run 'terraform destroy' - cmd.Start() - go tfLog(env.logger.Named("tf.stderr"), stderr) - go tfLog(env.logger.Named("tf.stdout"), stdout) - - err = cmd.Wait() - if err != nil { - return fmt.Errorf("terraform exited with a non-zero status: %v", err) - } - - return nil -} - -func tfLog(logger hclog.Logger, r io.ReadCloser) { - defer r.Close() - scanner := bufio.NewScanner(r) - for scanner.Scan() { - logger.Debug(scanner.Text()) - } - if err := scanner.Err(); err != nil { - logger.Error("scan error", "error", err) - } - -} diff --git a/e2e/cli/command/run.go b/e2e/cli/command/run.go index 76001baa4..c7b1f84b9 100644 --- a/e2e/cli/command/run.go +++ b/e2e/cli/command/run.go @@ -12,12 +12,8 @@ import ( "github.com/mitchellh/cli" ) -func RunCommandFactory(ui cli.Ui, logger hclog.Logger) cli.CommandFactory { +func RunCommandFactory(meta Meta) cli.CommandFactory { return func() (cli.Command, error) { - meta := Meta{ - Ui: ui, - logger: logger, - } return &Run{Meta: meta}, nil } } @@ -45,7 +41,7 @@ func (c *Run) Run(args []string) int { var run string cmdFlags := c.FlagSet("run") cmdFlags.Usage = func() { c.Ui.Output(c.Help()) } - cmdFlags.StringVar(&envPath, "env-path", "./environments/", "Path to e2e environment terraform configs") + cmdFlags.StringVar(&envPath, "env-path", DefaultEnvironmentsPath, "Path to e2e environment terraform configs") cmdFlags.StringVar(&nomadBinary, "nomad-binary", "", "") cmdFlags.StringVar(&tfPath, "tf-path", "", "") cmdFlags.StringVar(&run, "run", "", "Regex to target specific test suites/cases") @@ -63,22 +59,21 @@ func (c *Run) Run(args []string) int { if len(args) == 0 { c.logger.Info("no environments specified, running test suite locally") - var report *TestReport - var err error - if report, err = c.run(&runOpts{ + report, err := c.runTest(&runOpts{ slow: slow, verbose: c.verbose, - }); err != nil { + }) + if err != nil { c.logger.Error("failed to run test suite", "error", err) return 1 } - if report.TotalFailedTests == 0 { - c.Ui.Output("PASSED!") - if c.verbose { - c.Ui.Output(report.Summary()) - } - } else { - c.Ui.Output("***FAILED***") + if report.TotalFailedTests > 0 { + c.Ui.Error("***FAILED***") + c.Ui.Error(report.Summary()) + return 1 + } + c.Ui.Output("PASSED!") + if c.verbose { c.Ui.Output(report.Summary()) } return 0 @@ -98,7 +93,7 @@ func (c *Run) Run(args []string) int { environments = append(environments, envs...) } - envCount := len(environments) + // Use go-getter to fetch the nomad binary nomadPath, err := fetchBinary(nomadBinary) defer os.RemoveAll(nomadPath) @@ -107,7 +102,9 @@ func (c *Run) Run(args []string) int { return 1 } + envCount := len(environments) c.logger.Debug("starting tests", "totalEnvironments", envCount) + failedEnvs := map[string]*TestReport{} for i, env := range environments { logger := c.logger.With("name", env.name, "provider", env.provider) logger.Debug("provisioning environment") @@ -128,25 +125,35 @@ func (c *Run) Run(args []string) int { } var report *TestReport - if report, err = c.run(opts); err != nil { + if report, err = c.runTest(opts); err != nil { logger.Error("failed to run tests against environment", "error", err) return 1 } - if report.TotalFailedTests == 0 { + if report.TotalFailedTests > 0 { + c.Ui.Error(fmt.Sprintf("[%d/%d] %s: ***FAILED***", i+1, envCount, env.canonicalName())) + c.Ui.Error(fmt.Sprintf("[%d/%d] %s: %s", i+1, envCount, env.canonicalName(), report.Summary())) + failedEnvs[env.canonicalName()] = report + } - c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: PASSED!", i+1, envCount, env.provider, env.name)) - if c.verbose { - c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: %s", i+1, envCount, env.provider, env.name, report.Summary())) - } - } else { - c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: ***FAILED***", i+1, envCount, env.provider, env.name)) - c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: %s", i+1, envCount, env.provider, env.name, report.Summary())) + c.Ui.Output(fmt.Sprintf("[%d/%d] %s: PASSED!", i+1, envCount, env.canonicalName())) + if c.verbose { + c.Ui.Output(fmt.Sprintf("[%d/%d] %s: %s", i+1, envCount, env.canonicalName(), report.Summary())) } } + + if len(failedEnvs) > 0 { + c.Ui.Error(fmt.Sprintf("The following environments ***FAILED***")) + for name, report := range failedEnvs { + c.Ui.Error(fmt.Sprintf(" [%s]: %d out of %d suite failures", + name, report.TotalFailedSuites, report.TotalSuites)) + } + return 1 + } + c.Ui.Output("All Environments PASSED!") return 0 } -func (c *Run) run(opts *runOpts) (*TestReport, error) { +func (c *Run) runTest(opts *runOpts) (*TestReport, error) { goBin, err := exec.LookPath("go") if err != nil { return nil, err @@ -175,6 +182,8 @@ func (c *Run) run(opts *runOpts) (*TestReport, error) { } +// runOpts contains fields used to build the arguments and environment variabled +// nessicary to run go test and initialize the e2e framework type runOpts struct { nomadAddr string consulAddr string @@ -186,6 +195,8 @@ type runOpts struct { verbose bool } +// goArgs returns the list of arguments passed to the go command to start the +// e2e test framework func (opts *runOpts) goArgs() []string { a := []string{ "test", @@ -205,6 +216,8 @@ func (opts *runOpts) goArgs() []string { return a } +// goEnv returns the list of environment variabled passed to the go command to start +// the e2e test framework func (opts *runOpts) goEnv() []string { env := append(os.Environ(), "NOMAD_E2E=1") if opts.nomadAddr != "" { diff --git a/e2e/cli/command/util.go b/e2e/cli/command/util.go new file mode 100644 index 000000000..daf9ed30e --- /dev/null +++ b/e2e/cli/command/util.go @@ -0,0 +1,55 @@ +package command + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "runtime" + "time" + + getter "github.com/hashicorp/go-getter" + "github.com/hashicorp/nomad/helper/discover" +) + +// Fetches the nomad binary and returns the temporary directory where it exists +func fetchBinary(bin string) (string, error) { + nomadBinaryDir, err := ioutil.TempDir("", "") + if err != nil { + return "", fmt.Errorf("failed to create temp dir: %v", err) + } + + if bin == "" { + bin, err = discover.NomadExecutable() + if err != nil { + return "", fmt.Errorf("failed to discover nomad binary: %v", err) + } + } + + dest := path.Join(nomadBinaryDir, "nomad") + if runtime.GOOS == "windows" { + dest = dest + ".exe" + } + + if err = getter.GetFile(dest, bin); err != nil { + return "", fmt.Errorf("failed to get nomad binary: %v", err) + } + + return nomadBinaryDir, nil +} + +func procWaitTimeout(p *os.Process, d time.Duration) error { + stop := make(chan struct{}) + + go func() { + p.Wait() + stop <- struct{}{} + }() + + select { + case <-stop: + return nil + case <-time.NewTimer(d).C: + return fmt.Errorf("timeout waiting for process %d to exit", p.Pid) + } +} diff --git a/e2e/cli/main.go b/e2e/cli/main.go index f2340d5f0..18670e202 100644 --- a/e2e/cli/main.go +++ b/e2e/cli/main.go @@ -8,6 +8,11 @@ import ( "github.com/mitchellh/cli" ) +const ( + NomadE2ECli = "nomad-e2e" + NomadE2ECliVersion = "0.0.1" +) + func main() { ui := &cli.BasicUi{ @@ -17,15 +22,17 @@ func main() { } logger := hclog.New(&hclog.LoggerOptions{ - Name: "nomad-e2e", - Output: &cli.UiWriter{ui}, + Name: NomadE2ECli, + Output: &cli.UiWriter{Ui: ui}, }) - c := cli.NewCLI("nomad-e2e", "0.0.1") + c := cli.NewCLI(NomadE2ECli, NomadE2ECliVersion) c.Args = os.Args[1:] + + meta := command.NewMeta(ui, logger) c.Commands = map[string]cli.CommandFactory{ - "provision": command.ProvisionCommandFactory(ui, logger), - "run": command.RunCommandFactory(ui, logger), + "provision": command.ProvisionCommandFactory(meta), + "run": command.RunCommandFactory(meta), } exitStatus, err := c.Run() From 44652f455f5ce7dc7c6791b01aaca1e5fa1aa02d Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Tue, 31 Jul 2018 12:01:54 -0400 Subject: [PATCH 08/10] e2e/cli: add run command usage docs --- e2e/cli/command/meta.go | 10 ++++++++++ e2e/cli/command/provision.go | 12 ++++++------ e2e/cli/command/run.go | 24 +++++++++++++++++++++++- 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/e2e/cli/command/meta.go b/e2e/cli/command/meta.go index 8dbd3cdda..3ad4b8ffe 100644 --- a/e2e/cli/command/meta.go +++ b/e2e/cli/command/meta.go @@ -2,6 +2,7 @@ package command import ( "flag" + "strings" hclog "github.com/hashicorp/go-hclog" "github.com/mitchellh/cli" @@ -27,3 +28,12 @@ func (m *Meta) FlagSet(n string) *flag.FlagSet { f.BoolVar(&m.verbose, "v", false, "Toggle verbose output") return f } + +// generalOptionsUsage return the help string for the global options +func generalOptionsUsage() string { + helpText := ` + -verbose + Enables verbose logging. +` + return strings.TrimSpace(helpText) +} diff --git a/e2e/cli/command/provision.go b/e2e/cli/command/provision.go index 926beaf05..29c127786 100644 --- a/e2e/cli/command/provision.go +++ b/e2e/cli/command/provision.go @@ -38,6 +38,10 @@ Usage: nomad-e2e provision The output is a list of environment variables used to configure various api clients such as Nomad, Consul and Vault. +General Options: + +` + generalOptionsUsage() + ` + Provision Options: -env-path @@ -49,10 +53,6 @@ Provision Options: The binary is retrieved by go-getter and can therefore be a local file path, remote http url, or other support go-getter uri. - -nomad-checksum - If set, will ensure the binary from -nomad-binary matches the given - checksum. - -destroy If set, will destroy the target environment. @@ -121,9 +121,9 @@ func (c *Provision) Run(args []string) int { return 1 } - fmt.Printf(strings.TrimSpace(` + c.Ui.Output(strings.TrimSpace(fmt.Sprintf(` NOMAD_ADDR=%s - `), results.nomadAddr) + `, results.nomadAddr))) return 0 } diff --git a/e2e/cli/command/run.go b/e2e/cli/command/run.go index c7b1f84b9..e7bdec7d4 100644 --- a/e2e/cli/command/run.go +++ b/e2e/cli/command/run.go @@ -24,7 +24,29 @@ type Run struct { func (c *Run) Help() string { helpText := ` -Usage: nomad-e2e run +Usage: nomad-e2e run (/)... + + Two modes exist when using the run command. + + When no arguments are given to the run command, it will launch + the e2e test suite against the nomad cluster specified by the + NOMAD_ADDR environment variable. If this is not set, it defaults + to 'http://localhost:4646' + + Multiple arguments may be given to specify one or more environments to + provision and run the e2e tests against. These are given in the form of + /. Globs are support, for example 'aws/*' would run tests + against all of the environments under the aws provider. When using this mode, + all of the provision flags are supported. + +General Options: + +` + generalOptionsUsage() + ` + +Run Options: + + -slow + If set, will only run test suites marked as slow. ` return strings.TrimSpace(helpText) } From 2d31a59bfd4bfabf821caa433e4f22add7dab937 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Tue, 31 Jul 2018 15:21:47 -0400 Subject: [PATCH 09/10] e2e/cli: add -run option to mimic go test --- e2e/cli/command/meta.go | 2 +- e2e/cli/command/run.go | 18 +++++++++++++++++- e2e/cli/command/test_decoder.go | 1 - 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/e2e/cli/command/meta.go b/e2e/cli/command/meta.go index 3ad4b8ffe..2d9c66292 100644 --- a/e2e/cli/command/meta.go +++ b/e2e/cli/command/meta.go @@ -25,7 +25,7 @@ func NewMeta(ui cli.Ui, logger hclog.Logger) Meta { func (m *Meta) FlagSet(n string) *flag.FlagSet { f := flag.NewFlagSet(n, flag.ContinueOnError) - f.BoolVar(&m.verbose, "v", false, "Toggle verbose output") + f.BoolVar(&m.verbose, "verbose", false, "Toggle verbose output") return f } diff --git a/e2e/cli/command/run.go b/e2e/cli/command/run.go index e7bdec7d4..b0d55e13a 100644 --- a/e2e/cli/command/run.go +++ b/e2e/cli/command/run.go @@ -45,6 +45,12 @@ General Options: Run Options: + -run regex + Sets a regular expression for what tests to run. Uses '/' as a separator + to allow hierarchy between Suite/Case/Test. + + Example '-run MyTestSuite' would only run tests under the MyTestSuite suite. + -slow If set, will only run test suites marked as slow. ` @@ -82,6 +88,7 @@ func (c *Run) Run(args []string) int { if len(args) == 0 { c.logger.Info("no environments specified, running test suite locally") report, err := c.runTest(&runOpts{ + run: run, slow: slow, verbose: c.verbose, }) @@ -140,6 +147,7 @@ func (c *Run) Run(args []string) int { provider: env.provider, env: env.name, slow: slow, + run: run, verbose: c.verbose, nomadAddr: results.nomadAddr, consulAddr: results.consulAddr, @@ -212,6 +220,7 @@ type runOpts struct { vaultAddr string provider string env string + run string local bool slow bool verbose bool @@ -223,10 +232,17 @@ func (opts *runOpts) goArgs() []string { a := []string{ "test", "-json", + } + + if opts.run != "" { + a = append(a, "-run=TestE2E/"+opts.run) + } + + a = append(a, []string{ "github.com/hashicorp/nomad/e2e", "-env=" + opts.env, "-env.provider=" + opts.provider, - } + }...) if opts.slow { a = append(a, "-slow") diff --git a/e2e/cli/command/test_decoder.go b/e2e/cli/command/test_decoder.go index 776b7778f..c3b178f07 100644 --- a/e2e/cli/command/test_decoder.go +++ b/e2e/cli/command/test_decoder.go @@ -40,7 +40,6 @@ type TestEvent struct { Elapsed float64 // seconds Output string - eventType string suiteName string caseName string testName string From fff5ae622bc7cdf0675da925bc052d77a481f9a4 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Thu, 2 Aug 2018 13:29:12 -0400 Subject: [PATCH 10/10] e2e/cli: comment fixups --- e2e/cli/command/run.go | 2 +- e2e/cli/command/util.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/cli/command/run.go b/e2e/cli/command/run.go index b0d55e13a..ebcd29fcf 100644 --- a/e2e/cli/command/run.go +++ b/e2e/cli/command/run.go @@ -29,7 +29,7 @@ Usage: nomad-e2e run (/)... Two modes exist when using the run command. When no arguments are given to the run command, it will launch - the e2e test suite against the nomad cluster specified by the + the e2e test suite against the Nomad cluster specified by the NOMAD_ADDR environment variable. If this is not set, it defaults to 'http://localhost:4646' diff --git a/e2e/cli/command/util.go b/e2e/cli/command/util.go index daf9ed30e..f2e3f4a23 100644 --- a/e2e/cli/command/util.go +++ b/e2e/cli/command/util.go @@ -12,7 +12,7 @@ import ( "github.com/hashicorp/nomad/helper/discover" ) -// Fetches the nomad binary and returns the temporary directory where it exists +// fetchBinary fetches the nomad binary and returns the temporary directory where it exists func fetchBinary(bin string) (string, error) { nomadBinaryDir, err := ioutil.TempDir("", "") if err != nil {