From ec0d5e995050188f255dc13dabd572f75f1619f6 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Fri, 9 Sep 2022 07:17:02 -0500 Subject: [PATCH] build: make ec2info command usable from GNUMakefile The ec2info was never intuitive to run - needing to set the AWS envinronment variables, cd'ing into tools/ec2info, and knowing to invoke the command. This PR makes it so we can run ec2info just by running make ec2info The command now also checks for the AWS environment variables being set, and provides a useful error if they are not. --- GNUmakefile | 5 ++ tools/ec2info/aws.go | 124 -------------------------- tools/ec2info/main.go | 187 ++++++++++++++++++++++++++++++++++++---- tools/ec2info/output.go | 48 ----------- 4 files changed, 173 insertions(+), 191 deletions(-) delete mode 100644 tools/ec2info/aws.go delete mode 100644 tools/ec2info/output.go diff --git a/GNUmakefile b/GNUmakefile index f60aac9d6..3e77c5a34 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -425,3 +425,8 @@ endif missing: ## Check for packages not being tested @echo "==> Checking for packages not being tested ..." @go run -modfile tools/go.mod tools/missing/main.go .github/workflows/test-core.yaml + +.PHONY: ec2info +ec2info: ## Generate AWS EC2 CPU specification table + @echo "==> Generating AWS EC2 specifications ..." + @go run -modfile tools/go.mod tools/ec2info/main.go diff --git a/tools/ec2info/aws.go b/tools/ec2info/aws.go deleted file mode 100644 index 6382885c5..000000000 --- a/tools/ec2info/aws.go +++ /dev/null @@ -1,124 +0,0 @@ -package main - -import ( - "fmt" - "log" - "sort" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ec2" -) - -func clientForRegion(region string) (*ec2.EC2, error) { - sess, err := session.NewSession(&aws.Config{ - Region: ®ion, - }) - if err != nil { - return nil, err - } - return ec2.New(sess), nil -} - -func getRegions(client *ec2.EC2) ([]*ec2.Region, error) { - all := false // beyond account access - regions, err := client.DescribeRegions(&ec2.DescribeRegionsInput{ - AllRegions: &all, - }) - if err != nil { - return nil, err - } - return regions.Regions, nil -} - -type specs struct { - Cores int - Speed float64 -} - -func (s specs) String() string { - return fmt.Sprintf("(%d %.2f)", s.Cores, s.Speed) -} - -func getData(regions []*ec2.Region, verbose bool) (map[string]map[string]specs, error) { - data := make(map[string]map[string]specs) - - for _, region := range regions { - rData, rProblems, err := getDataForRegion(*region.RegionName) - if err != nil { - return nil, err - } - data[*region.RegionName] = rData - - if verbose { - log.Println("region", *region.RegionName, "got data for", len(rData), "instance types", len(rProblems), "incomplete") - instanceProblems(rProblems) - } - } - - return data, nil -} - -func instanceProblems(problems map[string]string) { - types := make([]string, 0, len(problems)) - for k := range problems { - types = append(types, k) - } - sort.Strings(types) - for _, iType := range types { - log.Println(" ->", iType, problems[iType]) - } -} - -func getDataForRegion(region string) (map[string]specs, map[string]string, error) { - client, err := clientForRegion(region) - if err != nil { - return nil, nil, err - } - - data := make(map[string]specs) - problems := make(map[string]string) - regionInfoPage(client, true, region, nil, data, problems) - return data, problems, nil -} - -func regionInfoPage(client *ec2.EC2, first bool, region string, token *string, data map[string]specs, problems map[string]string) { - if first || token != nil { - output, err := client.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{ - NextToken: token, - }) - if err != nil { - log.Fatal(err) - } - - // recursively accumulate each page of data - regionInfoAccumulate(output, data, problems) - regionInfoPage(client, false, region, output.NextToken, data, problems) - } -} - -func regionInfoAccumulate(output *ec2.DescribeInstanceTypesOutput, data map[string]specs, problems map[string]string) { - for _, iType := range output.InstanceTypes { - switch { - - case iType.ProcessorInfo == nil: - fallthrough - case iType.ProcessorInfo.SustainedClockSpeedInGhz == nil: - problems[*iType.InstanceType] = "missing clock Speed" - continue - - case iType.VCpuInfo == nil: - fallthrough - case iType.VCpuInfo.DefaultVCpus == nil: - problems[*iType.InstanceType] = "missing virtual cpu Cores" - continue - - default: - data[*iType.InstanceType] = specs{ - Speed: *iType.ProcessorInfo.SustainedClockSpeedInGhz, - Cores: int(*iType.VCpuInfo.DefaultVCpus), - } - continue - } - } -} diff --git a/tools/ec2info/main.go b/tools/ec2info/main.go index 8e2f17294..aa707735d 100644 --- a/tools/ec2info/main.go +++ b/tools/ec2info/main.go @@ -6,29 +6,24 @@ // // Requires AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN. // -// Options -// --package : configure package name of generated output file -// --region : configure initial region from which to lookup all regions -// --outfile : configure filepath of generated output file -// --verbose : print log messages while running +// Usage (invoke from Nomad's makefile) // -// Usage -// $ go run . +// make ec2info package main import ( - "flag" + "fmt" + "io" "log" -) + "os" + "os/exec" + "sort" + "text/template" -func args() (string, string, string, bool) { - pkg := flag.String("package", "fingerprint", "generate package name") - region := flag.String("region", "us-west-1", "initial region for listing regions") - outfile := flag.String("output", "../../client/fingerprint/env_aws_cpu.go", "output filepath") - verbose := flag.Bool("verbose", true, "print extra information while running") - flag.Parse() - return *pkg, *region, *outfile, *verbose -} + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" +) func check(err error) { if err != nil { @@ -37,7 +32,7 @@ func check(err error) { } func main() { - pkg, region, output, verbose := args() + pkg, region, output := "fingerprint", "us-west-1", "client/fingerprint/env_aws_cpu.go" client, err := clientForRegion(region) check(err) @@ -45,7 +40,7 @@ func main() { regions, err := getRegions(client) check(err) - data, err := getData(regions, verbose) + data, err := getData(regions) check(err) flat := flatten(data) @@ -59,3 +54,157 @@ func main() { check(write(f, flat, pkg)) check(format(output)) } + +func clientForRegion(region string) (*ec2.EC2, error) { + sess, err := session.NewSession(&aws.Config{ + Region: ®ion, + }) + if err != nil { + return nil, err + } + return ec2.New(sess), nil +} + +func getRegions(client *ec2.EC2) ([]*ec2.Region, error) { + all := false // beyond account access + regions, err := client.DescribeRegions(&ec2.DescribeRegionsInput{ + AllRegions: &all, + }) + if err != nil { + log.Println("failed to create AWS session; make sure environment is setup") + log.Println("must have environment variables AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN") + log.Println("or ~/.aws/credentials configured properly") + return nil, err + } + return regions.Regions, nil +} + +type specs struct { + Cores int + Speed float64 +} + +func (s specs) String() string { + return fmt.Sprintf("(%d %.2f)", s.Cores, s.Speed) +} + +func getData(regions []*ec2.Region) (map[string]map[string]specs, error) { + data := make(map[string]map[string]specs) + + for _, region := range regions { + rData, rProblems, err := getDataForRegion(*region.RegionName) + if err != nil { + return nil, err + } + data[*region.RegionName] = rData + + log.Println("region", *region.RegionName, "got data for", len(rData), "instance types", len(rProblems), "incomplete") + instanceProblems(rProblems) + } + + return data, nil +} + +func instanceProblems(problems map[string]string) { + types := make([]string, 0, len(problems)) + for k := range problems { + types = append(types, k) + } + sort.Strings(types) + for _, iType := range types { + log.Println(" ->", iType, problems[iType]) + } +} + +func getDataForRegion(region string) (map[string]specs, map[string]string, error) { + client, err := clientForRegion(region) + if err != nil { + return nil, nil, err + } + + data := make(map[string]specs) + problems := make(map[string]string) + regionInfoPage(client, true, region, nil, data, problems) + return data, problems, nil +} + +func regionInfoPage(client *ec2.EC2, first bool, region string, token *string, data map[string]specs, problems map[string]string) { + if first || token != nil { + output, err := client.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{ + NextToken: token, + }) + if err != nil { + log.Fatal(err) + } + + // recursively accumulate each page of data + regionInfoAccumulate(output, data, problems) + regionInfoPage(client, false, region, output.NextToken, data, problems) + } +} + +func regionInfoAccumulate(output *ec2.DescribeInstanceTypesOutput, data map[string]specs, problems map[string]string) { + for _, iType := range output.InstanceTypes { + switch { + + case iType.ProcessorInfo == nil: + fallthrough + case iType.ProcessorInfo.SustainedClockSpeedInGhz == nil: + problems[*iType.InstanceType] = "missing clock Speed" + continue + + case iType.VCpuInfo == nil: + fallthrough + case iType.VCpuInfo.DefaultVCpus == nil: + problems[*iType.InstanceType] = "missing virtual cpu Cores" + continue + + default: + data[*iType.InstanceType] = specs{ + Speed: *iType.ProcessorInfo.SustainedClockSpeedInGhz, + Cores: int(*iType.VCpuInfo.DefaultVCpus), + } + continue + } + } +} + +// open the output file for writing. +func open(output string) (io.ReadWriteCloser, error) { + return os.Create(output) +} + +// flatten region data, assuming instance type is the same across regions. +func flatten(data map[string]map[string]specs) map[string]specs { + result := make(map[string]specs) + for _, m := range data { + for iType, specifications := range m { + result[iType] = specifications + } + } + return result +} + +type Template struct { + Package string + Data map[string]specs +} + +// write the data using the cpu_table.go.template to w. +func write(w io.Writer, data map[string]specs, pkg string) error { + tmpl, err := template.ParseFiles("tools/ec2info/cpu_table.go.template") + if err != nil { + return err + } + return tmpl.Execute(w, Template{ + Package: pkg, + Data: data, + }) +} + +// format the file using gofmt. +func format(file string) error { + cmd := exec.Command("gofmt", "-w", file) + _, err := cmd.CombinedOutput() + return err +} diff --git a/tools/ec2info/output.go b/tools/ec2info/output.go deleted file mode 100644 index 75d27be27..000000000 --- a/tools/ec2info/output.go +++ /dev/null @@ -1,48 +0,0 @@ -package main - -import ( - "io" - "os" - "os/exec" - "text/template" -) - -// open the output file for writing. -func open(output string) (io.ReadWriteCloser, error) { - return os.Create(output) -} - -// flatten region data, assuming instance type is the same across regions. -func flatten(data map[string]map[string]specs) map[string]specs { - result := make(map[string]specs) - for _, m := range data { - for iType, specs := range m { - result[iType] = specs - } - } - return result -} - -type Template struct { - Package string - Data map[string]specs -} - -// write the data using the cpu_table.go.template to w. -func write(w io.Writer, data map[string]specs, pkg string) error { - tmpl, err := template.ParseFiles("cpu_table.go.template") - if err != nil { - return err - } - return tmpl.Execute(w, Template{ - Package: pkg, - Data: data, - }) -} - -// format the file using gofmt. -func format(file string) error { - cmd := exec.Command("gofmt", "-w", file) - _, err := cmd.CombinedOutput() - return err -}