Upgrade hcl2 to validate arrays for unknown values

This upgrades hcl2 library dependency to pick up
https://github.com/hashicorp/hcl2/pull/113 .

Prior to this change, parsing and decoding array attributes containing
invalid errors (e.g. references to unknown variables) are silently
dropped, with `cty.Unknown` being assigned to the bad element.  Rather
than showing a type/meaningful error from hcl2, we get a very decrypted
error message from msgpack layer trying to handle `cty.unknown`.

This ensures that we propagate diagnostics correctly and report
meaningful errors to users.

Fixes https://github.com/hashicorp/nomad/issues/5694
Fixes https://github.com/hashicorp/nomad/issues/5680
This commit is contained in:
Mahmood Ali 2019-06-17 12:28:14 -04:00
parent cb92b5d162
commit c1f6c7b457
14 changed files with 3601 additions and 3547 deletions

View file

@ -417,6 +417,17 @@ Token:
p.recoverAfterBodyItem()
}
// We must never produce a nil body, since the caller may attempt to
// do analysis of a partial result when there's an error, so we'll
// insert a placeholder if we otherwise failed to produce a valid
// body due to one of the syntax error paths above.
if body == nil && diags.HasErrors() {
body = &Body{
SrcRange: hcl.RangeBetween(oBrace.Range, cBraceRange),
EndRange: cBraceRange,
}
}
return &Block{
Type: blockType,
Labels: labels,

View file

@ -2,10 +2,10 @@ package hclsyntax
import (
"fmt"
"github.com/apparentlymart/go-textseg/textseg"
"strings"
"unicode"
"github.com/apparentlymart/go-textseg/textseg"
"github.com/hashicorp/hcl2/hcl"
"github.com/zclconf/go-cty/cty"
)

File diff suppressed because it is too large Load diff

View file

@ -63,8 +63,16 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To
BeginHeredocTmpl = '<<' ('-')? Ident Newline;
Comment = (
("#" (any - EndOfLine)* EndOfLine) |
("//" (any - EndOfLine)* EndOfLine) |
# The :>> operator in these is a "finish-guarded concatenation",
# which terminates the sequence on its left when it completes
# the sequence on its right.
# In the single-line comment cases this is allowing us to make
# the trailing EndOfLine optional while still having the overall
# pattern terminate. In the multi-line case it ensures that
# the first comment in the file ends at the first */, rather than
# gobbling up all of the "any*" until the _final_ */ in the file.
("#" (any - EndOfLine)* :>> EndOfLine?) |
("//" (any - EndOfLine)* :>> EndOfLine?) |
("/*" any* :>> "*/")
);
@ -218,29 +226,35 @@ func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []To
TemplateInterp = "${" ("~")?;
TemplateControl = "%{" ("~")?;
EndStringTmpl = '"';
StringLiteralChars = (AnyUTF8 - ("\r"|"\n"));
NewlineChars = ("\r"|"\n");
NewlineCharsSeq = NewlineChars+;
StringLiteralChars = (AnyUTF8 - NewlineChars);
TemplateIgnoredNonBrace = (^'{' %{ fhold; });
TemplateNotInterp = '$' (TemplateIgnoredNonBrace | TemplateInterp);
TemplateNotControl = '%' (TemplateIgnoredNonBrace | TemplateControl);
QuotedStringLiteralWithEsc = ('\\' StringLiteralChars) | (StringLiteralChars - ("$" | '%' | '"' | "\\"));
TemplateStringLiteral = (
('$' ^'{' %{ fhold; }) |
('%' ^'{' %{ fhold; }) |
('\\' StringLiteralChars) |
(StringLiteralChars - ("$" | '%' | '"'))
)+;
(TemplateNotInterp) |
(TemplateNotControl) |
(QuotedStringLiteralWithEsc)+
);
HeredocStringLiteral = (
('$' ^'{' %{ fhold; }) |
('%' ^'{' %{ fhold; }) |
(StringLiteralChars - ("$" | '%'))
)*;
(TemplateNotInterp) |
(TemplateNotControl) |
(StringLiteralChars - ("$" | '%'))*
);
BareStringLiteral = (
('$' ^'{') |
('%' ^'{') |
(StringLiteralChars - ("$" | '%'))
)* Newline?;
(TemplateNotInterp) |
(TemplateNotControl) |
(StringLiteralChars - ("$" | '%'))*
) Newline?;
stringTemplate := |*
TemplateInterp => beginTemplateInterp;
TemplateControl => beginTemplateControl;
EndStringTmpl => endStringTemplate;
TemplateStringLiteral => { token(TokenQuotedLit); };
NewlineCharsSeq => { token(TokenQuotedNewline); };
AnyUTF8 => { token(TokenInvalid); };
BrokenUTF8 => { token(TokenBadUTF8); };
*|;

View file

@ -279,7 +279,11 @@ func (b *Body) JustAttributes() (hcl.Attributes, hcl.Diagnostics) {
}
func (b *Body) MissingItemRange() hcl.Range {
return b.EndRange
return hcl.Range{
Filename: b.SrcRange.Filename,
Start: b.SrcRange.Start,
End: b.SrcRange.Start,
}
}
// Attributes is the collection of attribute definitions within a body.

View file

@ -85,17 +85,18 @@ const (
// things that might work in other languages they are familiar with, or
// simply make incorrect assumptions about the HCL language.
TokenBitwiseAnd TokenType = '&'
TokenBitwiseOr TokenType = '|'
TokenBitwiseNot TokenType = '~'
TokenBitwiseXor TokenType = '^'
TokenStarStar TokenType = '➚'
TokenApostrophe TokenType = '\''
TokenBacktick TokenType = '`'
TokenSemicolon TokenType = ';'
TokenTabs TokenType = '␉'
TokenInvalid TokenType = '<27>'
TokenBadUTF8 TokenType = '💩'
TokenBitwiseAnd TokenType = '&'
TokenBitwiseOr TokenType = '|'
TokenBitwiseNot TokenType = '~'
TokenBitwiseXor TokenType = '^'
TokenStarStar TokenType = '➚'
TokenApostrophe TokenType = '\''
TokenBacktick TokenType = '`'
TokenSemicolon TokenType = ';'
TokenTabs TokenType = '␉'
TokenInvalid TokenType = '<27>'
TokenBadUTF8 TokenType = '💩'
TokenQuotedNewline TokenType = '␤'
// TokenNil is a placeholder for when a token is required but none is
// available, e.g. when reporting errors. The scanner will never produce
@ -285,6 +286,13 @@ func checkInvalidTokens(tokens Tokens) hcl.Diagnostics {
toldBadUTF8++
}
case TokenQuotedNewline:
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid multi-line string",
Detail: "Quoted strings may not be split over multiple lines. To produce a multi-line string, either use the \\n escape to represent a newline character or use the \"heredoc\" multi-line template syntax.",
Subject: &tok.Range,
})
case TokenInvalid:
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,

View file

@ -4,7 +4,67 @@ package hclsyntax
import "strconv"
const _TokenType_name = "TokenNilTokenNewlineTokenBangTokenPercentTokenBitwiseAndTokenApostropheTokenOParenTokenCParenTokenStarTokenPlusTokenCommaTokenMinusTokenDotTokenSlashTokenColonTokenSemicolonTokenLessThanTokenEqualTokenGreaterThanTokenQuestionTokenCommentTokenOHeredocTokenIdentTokenNumberLitTokenQuotedLitTokenStringLitTokenOBrackTokenCBrackTokenBitwiseXorTokenBacktickTokenCHeredocTokenOBraceTokenBitwiseOrTokenCBraceTokenBitwiseNotTokenOQuoteTokenCQuoteTokenTemplateControlTokenEllipsisTokenFatArrowTokenTemplateSeqEndTokenAndTokenOrTokenTemplateInterpTokenEqualOpTokenNotEqualTokenLessThanEqTokenGreaterThanEqTokenEOFTokenTabsTokenStarStarTokenInvalidTokenBadUTF8"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[TokenOBrace-123]
_ = x[TokenCBrace-125]
_ = x[TokenOBrack-91]
_ = x[TokenCBrack-93]
_ = x[TokenOParen-40]
_ = x[TokenCParen-41]
_ = x[TokenOQuote-171]
_ = x[TokenCQuote-187]
_ = x[TokenOHeredoc-72]
_ = x[TokenCHeredoc-104]
_ = x[TokenStar-42]
_ = x[TokenSlash-47]
_ = x[TokenPlus-43]
_ = x[TokenMinus-45]
_ = x[TokenPercent-37]
_ = x[TokenEqual-61]
_ = x[TokenEqualOp-8788]
_ = x[TokenNotEqual-8800]
_ = x[TokenLessThan-60]
_ = x[TokenLessThanEq-8804]
_ = x[TokenGreaterThan-62]
_ = x[TokenGreaterThanEq-8805]
_ = x[TokenAnd-8743]
_ = x[TokenOr-8744]
_ = x[TokenBang-33]
_ = x[TokenDot-46]
_ = x[TokenComma-44]
_ = x[TokenEllipsis-8230]
_ = x[TokenFatArrow-8658]
_ = x[TokenQuestion-63]
_ = x[TokenColon-58]
_ = x[TokenTemplateInterp-8747]
_ = x[TokenTemplateControl-955]
_ = x[TokenTemplateSeqEnd-8718]
_ = x[TokenQuotedLit-81]
_ = x[TokenStringLit-83]
_ = x[TokenNumberLit-78]
_ = x[TokenIdent-73]
_ = x[TokenComment-67]
_ = x[TokenNewline-10]
_ = x[TokenEOF-9220]
_ = x[TokenBitwiseAnd-38]
_ = x[TokenBitwiseOr-124]
_ = x[TokenBitwiseNot-126]
_ = x[TokenBitwiseXor-94]
_ = x[TokenStarStar-10138]
_ = x[TokenApostrophe-39]
_ = x[TokenBacktick-96]
_ = x[TokenSemicolon-59]
_ = x[TokenTabs-9225]
_ = x[TokenInvalid-65533]
_ = x[TokenBadUTF8-128169]
_ = x[TokenQuotedNewline-9252]
_ = x[TokenNil-0]
}
const _TokenType_name = "TokenNilTokenNewlineTokenBangTokenPercentTokenBitwiseAndTokenApostropheTokenOParenTokenCParenTokenStarTokenPlusTokenCommaTokenMinusTokenDotTokenSlashTokenColonTokenSemicolonTokenLessThanTokenEqualTokenGreaterThanTokenQuestionTokenCommentTokenOHeredocTokenIdentTokenNumberLitTokenQuotedLitTokenStringLitTokenOBrackTokenCBrackTokenBitwiseXorTokenBacktickTokenCHeredocTokenOBraceTokenBitwiseOrTokenCBraceTokenBitwiseNotTokenOQuoteTokenCQuoteTokenTemplateControlTokenEllipsisTokenFatArrowTokenTemplateSeqEndTokenAndTokenOrTokenTemplateInterpTokenEqualOpTokenNotEqualTokenLessThanEqTokenGreaterThanEqTokenEOFTokenTabsTokenQuotedNewlineTokenStarStarTokenInvalidTokenBadUTF8"
var _TokenType_map = map[TokenType]string{
0: _TokenType_name[0:8],
@ -57,9 +117,10 @@ var _TokenType_map = map[TokenType]string{
8805: _TokenType_name[577:595],
9220: _TokenType_name[595:603],
9225: _TokenType_name[603:612],
10138: _TokenType_name[612:625],
65533: _TokenType_name[625:637],
128169: _TokenType_name[637:649],
9252: _TokenType_name[612:630],
10138: _TokenType_name[630:643],
65533: _TokenType_name[643:655],
128169: _TokenType_name[655:667],
}
func (i TokenType) String() string {

View file

@ -153,7 +153,7 @@ func byteCanStartKeyword(b byte) bool {
// in the parser, where we can generate better diagnostics.
// So e.g. we want to be able to say:
// unrecognized keyword "True". Did you mean "true"?
case b >= 'a' || b <= 'z' || b >= 'A' || b <= 'Z':
case isAlphabetical(b):
return true
default:
return false
@ -167,7 +167,7 @@ Byte:
for i = 0; i < len(buf); i++ {
b := buf[i]
switch {
case (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_':
case isAlphabetical(b) || b == '_':
p.Pos.Byte++
p.Pos.Column++
default:
@ -291,3 +291,7 @@ func posRange(start, end pos) hcl.Range {
func (t token) GoString() string {
return fmt.Sprintf("json.token{json.%s, []byte(%q), %#v}", t.Type, t.Bytes, t.Range)
}
func isAlphabetical(b byte) bool {
return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z')
}

View file

@ -416,12 +416,14 @@ func (e *expression) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) {
case *booleanVal:
return cty.BoolVal(v.Value), nil
case *arrayVal:
var diags hcl.Diagnostics
vals := []cty.Value{}
for _, jsonVal := range v.Values {
val, _ := (&expression{src: jsonVal}).Value(ctx)
val, valDiags := (&expression{src: jsonVal}).Value(ctx)
vals = append(vals, val)
diags = append(diags, valDiags...)
}
return cty.TupleVal(vals), nil
return cty.TupleVal(vals), diags
case *objectVal:
var diags hcl.Diagnostics
attrs := map[string]cty.Value{}

View file

@ -2,6 +2,7 @@ package hcl
import (
"fmt"
"math/big"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/convert"
@ -84,6 +85,27 @@ func Index(collection, key cty.Value, srcRange *Range) (cty.Value, Diagnostics)
}
}
if has.False() {
// We have a more specialized error message for the situation of
// using a fractional number to index into a sequence, because
// that will tend to happen if the user is trying to use division
// to calculate an index and not realizing that HCL does float
// division rather than integer division.
if (ty.IsListType() || ty.IsTupleType()) && key.Type().Equals(cty.Number) {
if key.IsKnown() && !key.IsNull() {
bf := key.AsBigFloat()
if _, acc := bf.Int(nil); acc != big.Exact {
return cty.DynamicVal, Diagnostics{
{
Severity: DiagError,
Summary: "Invalid index",
Detail: fmt.Sprintf("The given key does not identify an element in this collection value: indexing a sequence requires a whole number, but the given index (%g) has a fractional part.", bf),
Subject: srcRange,
},
}
}
}
}
return cty.DynamicVal, Diagnostics{
{
Severity: DiagError,

View file

@ -31,6 +31,9 @@ type Pos struct {
Byte int
}
// InitialPos is a suitable position to use to mark the start of a file.
var InitialPos = Pos{Byte: 0, Line: 1, Column: 1}
// Range represents a span of characters between two positions in a source
// file.
//

View file

@ -29,8 +29,8 @@ type RangeScanner struct {
err error // error from last scan, if any
}
// Create a new RangeScanner for the given buffer, producing ranges for the
// given filename.
// NewRangeScanner creates a new RangeScanner for the given buffer, producing
// ranges for the given filename.
//
// Since ranges have grapheme-cluster granularity rather than byte granularity,
// the scanner will produce incorrect results if the given SplitFunc creates
@ -39,15 +39,19 @@ type RangeScanner struct {
// around individual UTF-8 sequences, which will split any multi-sequence
// grapheme clusters.
func NewRangeScanner(b []byte, filename string, cb bufio.SplitFunc) *RangeScanner {
return NewRangeScannerFragment(b, filename, InitialPos, cb)
}
// NewRangeScannerFragment is like NewRangeScanner but the ranges it produces
// will be offset by the given starting position, which is appropriate for
// sub-slices of a file, whereas NewRangeScanner assumes it is scanning an
// entire file.
func NewRangeScannerFragment(b []byte, filename string, start Pos, cb bufio.SplitFunc) *RangeScanner {
return &RangeScanner{
filename: filename,
b: b,
cb: cb,
pos: Pos{
Byte: 0,
Line: 1,
Column: 1,
},
pos: start,
}
}

View file

@ -79,13 +79,13 @@ func formatIndent(lines []formatLine) {
netBrackets := 0
for _, token := range line.lead {
netBrackets += tokenBracketChange(token)
}
for _, token := range line.assign {
netBrackets += tokenBracketChange(token)
if token.Type == hclsyntax.TokenOHeredoc {
inHeredoc = true
}
}
for _, token := range line.assign {
netBrackets += tokenBracketChange(token)
}
switch {
case netBrackets > 0:
@ -248,8 +248,8 @@ func spaceAfterToken(subject, before, after *Token) bool {
// Don't use spaces around attribute access dots
return false
case after.Type == hclsyntax.TokenComma:
// No space right before a comma in an argument list
case after.Type == hclsyntax.TokenComma || after.Type == hclsyntax.TokenEllipsis:
// No space right before a comma or ... in an argument list
return false
case subject.Type == hclsyntax.TokenComma:

12
vendor/vendor.json vendored
View file

@ -217,12 +217,12 @@
{"path":"github.com/hashicorp/hcl/json/parser","checksumSHA1":"138aCV5n8n7tkGYMsMVQQnnLq+0=","revision":"6e968a3fcdcbab092f5307fd0d85479d5af1e4dc","revisionTime":"2016-11-01T18:00:25Z"},
{"path":"github.com/hashicorp/hcl/json/scanner","checksumSHA1":"YdvFsNOMSWMLnY6fcliWQa0O5Fw=","revision":"6e968a3fcdcbab092f5307fd0d85479d5af1e4dc","revisionTime":"2016-11-01T18:00:25Z"},
{"path":"github.com/hashicorp/hcl/json/token","checksumSHA1":"fNlXQCQEnb+B3k5UDL/r15xtSJY=","revision":"6e968a3fcdcbab092f5307fd0d85479d5af1e4dc","revisionTime":"2016-11-01T18:00:25Z"},
{"path":"github.com/hashicorp/hcl2/gohcl","checksumSHA1":"RFEjfMQWPAVILXE2PhL6wDW8Zg4=","revision":"fdf8e232b64f68d5335fa1be449b0160dadacdb5","revisionTime":"2019-03-05T17:45:54Z","version":"master","versionExact":"master"},
{"path":"github.com/hashicorp/hcl2/hcl","checksumSHA1":"bUO4KS1yjAWa6miewgbUUsxYVfo=","revision":"fdf8e232b64f68d5335fa1be449b0160dadacdb5","revisionTime":"2019-03-05T17:45:54Z","version":"master","versionExact":"master"},
{"path":"github.com/hashicorp/hcl2/hcl/hclsyntax","checksumSHA1":"uMWQk/2xJyIqL6ILq83VYVxkuY8=","revision":"fdf8e232b64f68d5335fa1be449b0160dadacdb5","revisionTime":"2019-03-05T17:45:54Z","version":"master","versionExact":"master"},
{"path":"github.com/hashicorp/hcl2/hcl/json","checksumSHA1":"Vagtn0ywFboMp7br/xvzeuNUFNc=","revision":"fdf8e232b64f68d5335fa1be449b0160dadacdb5","revisionTime":"2019-03-05T17:45:54Z","version":"master","versionExact":"master"},
{"path":"github.com/hashicorp/hcl2/hcldec","checksumSHA1":"6JRj4T/iQxIe/CoKXHDjPuupmL8=","revision":"fdf8e232b64f68d5335fa1be449b0160dadacdb5","revisionTime":"2019-03-05T17:45:54Z","version":"master","versionExact":"master"},
{"path":"github.com/hashicorp/hcl2/hclwrite","checksumSHA1":"DQLzlvDUtHL1DkYDZrMx2vfKJUg=","revision":"fdf8e232b64f68d5335fa1be449b0160dadacdb5","revisionTime":"2019-03-05T17:45:54Z","version":"master","versionExact":"master"},
{"path":"github.com/hashicorp/hcl2/gohcl","checksumSHA1":"RFEjfMQWPAVILXE2PhL6wDW8Zg4=","revision":"4fba5e1a75e382aed7f7a7993f2c4836a5e1cd52","revisionTime":"2019-06-17T16:00:22Z","version":"master","versionExact":"master"},
{"path":"github.com/hashicorp/hcl2/hcl","checksumSHA1":"1jDEGh+P7Cu8Lz39VudY6rRS6Jw=","revision":"4fba5e1a75e382aed7f7a7993f2c4836a5e1cd52","revisionTime":"2019-06-17T16:00:22Z","version":"master","versionExact":"master"},
{"path":"github.com/hashicorp/hcl2/hcl/hclsyntax","checksumSHA1":"6FZRBZj+je9sMM5wrhM5phUXxZU=","revision":"4fba5e1a75e382aed7f7a7993f2c4836a5e1cd52","revisionTime":"2019-06-17T16:00:22Z","version":"master","versionExact":"master"},
{"path":"github.com/hashicorp/hcl2/hcl/json","checksumSHA1":"l5KBeDDM1gf7yFth7WGhckpF+oc=","revision":"4fba5e1a75e382aed7f7a7993f2c4836a5e1cd52","revisionTime":"2019-06-17T16:00:22Z","version":"master","versionExact":"master"},
{"path":"github.com/hashicorp/hcl2/hcldec","checksumSHA1":"6JRj4T/iQxIe/CoKXHDjPuupmL8=","revision":"4fba5e1a75e382aed7f7a7993f2c4836a5e1cd52","revisionTime":"2019-06-17T16:00:22Z","version":"master","versionExact":"master"},
{"path":"github.com/hashicorp/hcl2/hclwrite","checksumSHA1":"Hgs10IsC8LdotUKLavgOL35Mbw4=","revision":"4fba5e1a75e382aed7f7a7993f2c4836a5e1cd52","revisionTime":"2019-06-17T16:00:22Z","version":"master","versionExact":"master"},
{"path":"github.com/hashicorp/logutils","checksumSHA1":"vt+P9D2yWDO3gdvdgCzwqunlhxU=","revision":"0dc08b1671f34c4250ce212759ebd880f743d883"},
{"path":"github.com/hashicorp/memberlist","checksumSHA1":"yAu2gPVXIh28yJ2If5gZPrf04kU=","revision":"1a62499c21db33d57691001d5e08a71ec857b18f","revisionTime":"2019-01-03T22:22:36Z"},
{"path":"github.com/hashicorp/net-rpc-msgpackrpc","checksumSHA1":"qnlqWJYV81ENr61SZk9c65R1mDo=","revision":"a14192a58a694c123d8fe5481d4a4727d6ae82f3"},