jobspec: new HCL API

This commit is contained in:
Mitchell Hashimoto 2015-11-08 22:57:39 -08:00
parent 3103e426f7
commit 3f8765f532
2 changed files with 199 additions and 198 deletions

View File

@ -9,14 +9,16 @@ import (
"regexp"
"strconv"
"strings"
"time"
"github.com/hashicorp/hcl"
hclobj "github.com/hashicorp/hcl/hcl"
"github.com/hashicorp/hcl/hcl/ast"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/mitchellh/mapstructure"
)
var reDynamicPorts *regexp.Regexp = regexp.MustCompile("^[a-zA-Z0-9_]+$")
var errDynamicPorts = fmt.Errorf("DynamicPort label does not conform to naming requirements %s", reDynamicPorts.String())
// Parse parses the job spec from the given io.Reader.
//
// Due to current internal limitations, the entire contents of the
@ -29,20 +31,26 @@ func Parse(r io.Reader) (*structs.Job, error) {
}
// Parse the buffer
obj, err := hcl.Parse(buf.String())
root, err := hcl.Parse(buf.String())
if err != nil {
return nil, fmt.Errorf("error parsing: %s", err)
}
buf.Reset()
// Top-level item should be a list
list, ok := root.Node.(*ast.ObjectList)
if !ok {
return nil, fmt.Errorf("error parsing: root should be an object")
}
var job structs.Job
// Parse the job out
jobO := obj.Get("job", false)
if jobO == nil {
matches := list.Filter("job")
if len(matches.Items) == 0 {
return nil, fmt.Errorf("'job' stanza not found")
}
if err := parseJob(&job, jobO); err != nil {
if err := parseJob(&job, matches); err != nil {
return nil, fmt.Errorf("error parsing 'job': %s", err)
}
@ -65,17 +73,18 @@ func ParseFile(path string) (*structs.Job, error) {
return Parse(f)
}
func parseJob(result *structs.Job, obj *hclobj.Object) error {
if obj.Len() > 1 {
func parseJob(result *structs.Job, list *ast.ObjectList) error {
list = list.Children()
if len(list.Items) != 1 {
return fmt.Errorf("only one 'job' block allowed")
}
// Get our job object
obj = obj.Elem(true)[0]
obj := list.Items[0]
// Decode the full thing into a map[string]interface for ease
var m map[string]interface{}
if err := hcl.DecodeObject(&m, obj); err != nil {
if err := hcl.DecodeObject(&m, obj.Val); err != nil {
return err
}
delete(m, "constraint")
@ -83,8 +92,8 @@ func parseJob(result *structs.Job, obj *hclobj.Object) error {
delete(m, "update")
// Set the ID and name to the object key
result.ID = obj.Key
result.Name = obj.Key
result.ID = obj.Keys[0].Token.Value().(string)
result.Name = result.ID
// Defaults
result.Priority = 50
@ -96,15 +105,23 @@ func parseJob(result *structs.Job, obj *hclobj.Object) error {
return err
}
// Value should be an object
var listVal *ast.ObjectList
if ot, ok := obj.Val.(*ast.ObjectType); ok {
listVal = ot.List
} else {
return fmt.Errorf("job '%s' value: should be an object", result.ID)
}
// Parse constraints
if o := obj.Get("constraint", false); o != nil {
if o := listVal.Filter("constraint"); len(o.Items) > 0 {
if err := parseConstraints(&result.Constraints, o); err != nil {
return err
}
}
// If we have an update strategy, then parse that
if o := obj.Get("update", false); o != nil {
if o := listVal.Filter("update"); len(o.Items) > 0 {
if err := parseUpdate(&result.Update, o); err != nil {
return err
}
@ -112,10 +129,10 @@ func parseJob(result *structs.Job, obj *hclobj.Object) error {
// Parse out meta fields. These are in HCL as a list so we need
// to iterate over them and merge them.
if metaO := obj.Get("meta", false); metaO != nil {
for _, o := range metaO.Elem(false) {
if metaO := listVal.Filter("meta"); len(metaO.Items) > 0 {
for _, o := range metaO.Elem().Items {
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o); err != nil {
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return err
}
if err := mapstructure.WeakDecode(m, &result.Meta); err != nil {
@ -125,7 +142,7 @@ func parseJob(result *structs.Job, obj *hclobj.Object) error {
}
// If we have tasks outside, create TaskGroups for them
if o := obj.Get("task", false); o != nil {
if o := listVal.Filter("task"); len(o.Items) > 0 {
var tasks []*structs.Task
if err := parseTasks(&tasks, o); err != nil {
return err
@ -143,7 +160,7 @@ func parseJob(result *structs.Job, obj *hclobj.Object) error {
}
// Parse the task groups
if o := obj.Get("group", false); o != nil {
if o := listVal.Filter("group"); len(o.Items) > 0 {
if err := parseGroups(result, o); err != nil {
return fmt.Errorf("error parsing 'group': %s", err)
}
@ -152,30 +169,34 @@ func parseJob(result *structs.Job, obj *hclobj.Object) error {
return nil
}
func parseGroups(result *structs.Job, obj *hclobj.Object) error {
// Get all the maps of keys to the actual object
objects := make(map[string]*hclobj.Object)
for _, o1 := range obj.Elem(false) {
for _, o2 := range o1.Elem(true) {
if _, ok := objects[o2.Key]; ok {
return fmt.Errorf(
"group '%s' defined more than once",
o2.Key)
}
objects[o2.Key] = o2
}
}
if len(objects) == 0 {
func parseGroups(result *structs.Job, list *ast.ObjectList) error {
list = list.Children()
if len(list.Items) == 0 {
return nil
}
// Go through each object and turn it into an actual result.
collection := make([]*structs.TaskGroup, 0, len(objects))
for n, o := range objects {
collection := make([]*structs.TaskGroup, 0, len(list.Items))
seen := make(map[string]struct{})
for _, item := range list.Items {
n := item.Keys[0].Token.Value().(string)
// Make sure we haven't already found this
if _, ok := seen[n]; ok {
return fmt.Errorf("group '%s' defined more than once", n)
}
seen[n] = struct{}{}
// We need this later
var listVal *ast.ObjectList
if ot, ok := item.Val.(*ast.ObjectType); ok {
listVal = ot.List
} else {
return fmt.Errorf("group '%s': should be an object", n)
}
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o); err != nil {
if err := hcl.DecodeObject(&m, item.Val); err != nil {
return err
}
delete(m, "constraint")
@ -196,23 +217,26 @@ func parseGroups(result *structs.Job, obj *hclobj.Object) error {
}
// Parse constraints
if o := o.Get("constraint", false); o != nil {
if o := listVal.Filter("constraint"); len(o.Items) > 0 {
if err := parseConstraints(&g.Constraints, o); err != nil {
return err
}
}
g.RestartPolicy = structs.NewRestartPolicy(result.Type)
if err := parseRestartPolicy(g.RestartPolicy, o); err != nil {
return err
// Parse restart policy
if o := listVal.Filter("restart"); len(o.Items) > 0 {
if err := parseRestartPolicy(g.RestartPolicy, o); err != nil {
return err
}
}
// Parse out meta fields. These are in HCL as a list so we need
// to iterate over them and merge them.
if metaO := o.Get("meta", false); metaO != nil {
for _, o := range metaO.Elem(false) {
if metaO := listVal.Filter("meta"); len(metaO.Items) > 0 {
for _, o := range metaO.Elem().Items {
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o); err != nil {
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return err
}
if err := mapstructure.WeakDecode(m, &g.Meta); err != nil {
@ -222,7 +246,7 @@ func parseGroups(result *structs.Job, obj *hclobj.Object) error {
}
// Parse tasks
if o := o.Get("task", false); o != nil {
if o := listVal.Filter("task"); len(o.Items) > 0 {
if err := parseTasks(&g.Tasks, o); err != nil {
return err
}
@ -235,46 +259,44 @@ func parseGroups(result *structs.Job, obj *hclobj.Object) error {
return nil
}
func parseRestartPolicy(result *structs.RestartPolicy, obj *hclobj.Object) error {
var restartHclObj *hclobj.Object
var m map[string]interface{}
if restartHclObj = obj.Get("restart", false); restartHclObj == nil {
func parseRestartPolicy(final *structs.RestartPolicy, list *ast.ObjectList) error {
list = list.Elem()
if len(list.Items) == 0 {
return nil
}
if err := hcl.DecodeObject(&m, restartHclObj); err != nil {
if len(list.Items) != 1 {
return fmt.Errorf("only one 'restart' block allowed")
}
// Get our job object
obj := list.Items[0]
var m map[string]interface{}
if err := hcl.DecodeObject(&m, obj.Val); err != nil {
return err
}
if delay, ok := m["delay"]; ok {
d, err := toDuration(delay)
if err != nil {
return fmt.Errorf("Invalid Delay time in restart policy: %v", err)
}
result.Delay = d
var result structs.RestartPolicy
dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
DecodeHook: mapstructure.StringToTimeDurationHookFunc(),
WeaklyTypedInput: true,
Result: &result,
})
if err != nil {
return err
}
if err := dec.Decode(m); err != nil {
return err
}
if interval, ok := m["interval"]; ok {
i, err := toDuration(interval)
if err != nil {
return fmt.Errorf("Invalid Interval time in restart policy: %v", err)
}
result.Interval = i
}
if attempts, ok := m["attempts"]; ok {
a, err := toInteger(attempts)
if err != nil {
return fmt.Errorf("Invalid value in attempts: %v", err)
}
result.Attempts = a
}
*final = result
return nil
}
func parseConstraints(result *[]*structs.Constraint, obj *hclobj.Object) error {
for _, o := range obj.Elem(false) {
func parseConstraints(result *[]*structs.Constraint, list *ast.ObjectList) error {
for _, o := range list.Elem().Items {
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o); err != nil {
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return err
}
m["LTarget"] = m["attribute"]
@ -324,30 +346,33 @@ func parseConstraints(result *[]*structs.Constraint, obj *hclobj.Object) error {
return nil
}
func parseTasks(result *[]*structs.Task, obj *hclobj.Object) error {
// Get all the maps of keys to the actual object
objects := make([]*hclobj.Object, 0, 5)
set := make(map[string]struct{})
for _, o1 := range obj.Elem(false) {
for _, o2 := range o1.Elem(true) {
if _, ok := set[o2.Key]; ok {
return fmt.Errorf(
"group '%s' defined more than once",
o2.Key)
}
objects = append(objects, o2)
set[o2.Key] = struct{}{}
}
}
if len(objects) == 0 {
func parseTasks(result *[]*structs.Task, list *ast.ObjectList) error {
list = list.Children()
if len(list.Items) == 0 {
return nil
}
for _, o := range objects {
// Go through each object and turn it into an actual result.
seen := make(map[string]struct{})
for _, item := range list.Items {
n := item.Keys[0].Token.Value().(string)
// Make sure we haven't already found this
if _, ok := seen[n]; ok {
return fmt.Errorf("task '%s' defined more than once", n)
}
seen[n] = struct{}{}
// We need this later
var listVal *ast.ObjectList
if ot, ok := item.Val.(*ast.ObjectType); ok {
listVal = ot.List
} else {
return fmt.Errorf("group '%s': should be an object", n)
}
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o); err != nil {
if err := hcl.DecodeObject(&m, item.Val); err != nil {
return err
}
delete(m, "config")
@ -358,16 +383,16 @@ func parseTasks(result *[]*structs.Task, obj *hclobj.Object) error {
// Build the task
var t structs.Task
t.Name = o.Key
t.Name = n
if err := mapstructure.WeakDecode(m, &t); err != nil {
return err
}
// If we have env, then parse them
if o := o.Get("env", false); o != nil {
for _, o := range o.Elem(false) {
if o := listVal.Filter("env"); len(o.Items) > 0 {
for _, o := range o.Elem().Items {
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o); err != nil {
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return err
}
if err := mapstructure.WeakDecode(m, &t.Env); err != nil {
@ -377,10 +402,10 @@ func parseTasks(result *[]*structs.Task, obj *hclobj.Object) error {
}
// If we have config, then parse that
if o := o.Get("config", false); o != nil {
for _, o := range o.Elem(false) {
if o := listVal.Filter("config"); len(o.Items) > 0 {
for _, o := range o.Elem().Items {
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o); err != nil {
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return err
}
if err := mapstructure.WeakDecode(m, &t.Config); err != nil {
@ -390,7 +415,7 @@ func parseTasks(result *[]*structs.Task, obj *hclobj.Object) error {
}
// Parse constraints
if o := o.Get("constraint", false); o != nil {
if o := listVal.Filter("constraint"); len(o.Items) > 0 {
if err := parseConstraints(&t.Constraints, o); err != nil {
return err
}
@ -398,10 +423,10 @@ func parseTasks(result *[]*structs.Task, obj *hclobj.Object) error {
// Parse out meta fields. These are in HCL as a list so we need
// to iterate over them and merge them.
if metaO := o.Get("meta", false); metaO != nil {
for _, o := range metaO.Elem(false) {
if metaO := listVal.Filter("meta"); len(metaO.Items) > 0 {
for _, o := range metaO.Elem().Items {
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o); err != nil {
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return err
}
if err := mapstructure.WeakDecode(m, &t.Meta); err != nil {
@ -411,7 +436,7 @@ func parseTasks(result *[]*structs.Task, obj *hclobj.Object) error {
}
// If we have resources, then parse that
if o := o.Get("resources", false); o != nil {
if o := listVal.Filter("resources"); len(o.Items) > 0 {
var r structs.Resources
if err := parseResources(&r, o); err != nil {
return fmt.Errorf("task '%s': %s", t.Name, err)
@ -426,121 +451,95 @@ func parseTasks(result *[]*structs.Task, obj *hclobj.Object) error {
return nil
}
var reDynamicPorts *regexp.Regexp = regexp.MustCompile("^[a-zA-Z0-9_]+$")
var errDynamicPorts = fmt.Errorf("DynamicPort label does not conform to naming requirements %s", reDynamicPorts.String())
func parseResources(result *structs.Resources, obj *hclobj.Object) error {
if obj.Len() > 1 {
func parseResources(result *structs.Resources, list *ast.ObjectList) error {
list = list.Elem()
if len(list.Items) == 0 {
return nil
}
if len(list.Items) > 1 {
return fmt.Errorf("only one 'resource' block allowed per task")
}
for _, o := range obj.Elem(false) {
// Get our resource object
o := list.Items[0]
// We need this later
var listVal *ast.ObjectList
if ot, ok := o.Val.(*ast.ObjectType); ok {
listVal = ot.List
} else {
return fmt.Errorf("resource: should be an object")
}
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return err
}
delete(m, "network")
if err := mapstructure.WeakDecode(m, result); err != nil {
return err
}
// Parse the network resources
if o := listVal.Filter("network"); len(o.Items) > 0 {
if len(o.Items) > 1 {
return fmt.Errorf("only one 'network' resource allowed")
}
var r structs.NetworkResource
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o); err != nil {
if err := hcl.DecodeObject(&m, o.Items[0].Val); err != nil {
return err
}
delete(m, "network")
if err := mapstructure.WeakDecode(m, result); err != nil {
if err := mapstructure.WeakDecode(m, &r); err != nil {
return err
}
// Parse the network resources
if o := o.Get("network", false); o != nil {
if o.Len() > 1 {
return fmt.Errorf("only one 'network' resource allowed")
// Keep track of labels we've already seen so we can ensure there
// are no collisions when we turn them into environment variables.
// lowercase:NomalCase so we can get the first for the error message
seenLabel := map[string]string{}
for _, label := range r.DynamicPorts {
if !reDynamicPorts.MatchString(label) {
return errDynamicPorts
}
first, seen := seenLabel[strings.ToLower(label)]
if seen {
return fmt.Errorf("Found a port label collision: `%s` overlaps with previous `%s`", label, first)
} else {
seenLabel[strings.ToLower(label)] = label
}
var r structs.NetworkResource
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o); err != nil {
return err
}
if err := mapstructure.WeakDecode(m, &r); err != nil {
return err
}
// Keep track of labels we've already seen so we can ensure there
// are no collisions when we turn them into environment variables.
// lowercase:NomalCase so we can get the first for the error message
seenLabel := map[string]string{}
for _, label := range r.DynamicPorts {
if !reDynamicPorts.MatchString(label) {
return errDynamicPorts
}
first, seen := seenLabel[strings.ToLower(label)]
if seen {
return fmt.Errorf("Found a port label collision: `%s` overlaps with previous `%s`", label, first)
} else {
seenLabel[strings.ToLower(label)] = label
}
}
result.Networks = []*structs.NetworkResource{&r}
}
result.Networks = []*structs.NetworkResource{&r}
}
return nil
}
func parseUpdate(result *structs.UpdateStrategy, obj *hclobj.Object) error {
if obj.Len() > 1 {
func parseUpdate(result *structs.UpdateStrategy, list *ast.ObjectList) error {
list = list.Elem()
if len(list.Items) > 1 {
return fmt.Errorf("only one 'update' block allowed per job")
}
for _, o := range obj.Elem(false) {
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o); err != nil {
return err
}
for _, key := range []string{"stagger", "Stagger"} {
if raw, ok := m[key]; ok {
staggerTime, err := toDuration(raw)
if err != nil {
return fmt.Errorf("Invalid stagger time: %v", err)
}
m[key] = staggerTime
}
}
// Get our resource object
o := list.Items[0]
if err := mapstructure.WeakDecode(m, result); err != nil {
return err
}
}
return nil
}
func toDuration(value interface{}) (time.Duration, error) {
var dur time.Duration
var err error
switch v := value.(type) {
case string:
dur, err = time.ParseDuration(v)
case int:
dur = time.Duration(v) * time.Second
default:
err = fmt.Errorf("Invalid time %s", value)
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return err
}
return dur, err
}
func toInteger(value interface{}) (int, error) {
var integer int
var err error
switch v := value.(type) {
case string:
var i int64
i, err = strconv.ParseInt(v, 10, 32)
integer = int(i)
case int:
integer = v
default:
err = fmt.Errorf("Value: %v can't be parsed into int", value)
dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
DecodeHook: mapstructure.StringToTimeDurationHookFunc(),
WeaklyTypedInput: true,
Result: result,
})
if err != nil {
return err
}
return integer, err
return dec.Decode(m)
}

View File

@ -228,6 +228,8 @@ func TestParse(t *testing.T) {
}
for _, tc := range cases {
t.Logf("Testing parse: %s", tc.File)
path, err := filepath.Abs(filepath.Join("./test-fixtures", tc.File))
if err != nil {
t.Fatalf("file: %s\n\n%s", tc.File, err)