From 835ceb0b9780138960dcfa35639fd28c22aa7d53 Mon Sep 17 00:00:00 2001 From: Bazaah Date: Wed, 2 Nov 2022 10:49:10 +0000 Subject: [PATCH 1/3] lib/scanner: fix missing token production for zero indents So, the underlying bug here is caused because we check for zero indents *before* unwinding the indentation stack. Take the following example: ``` 0|Map1: 1|- Map2: value 2| Map2.1: value 3|Map3: ... ---------- 0123456789 ``` At (1,0) we encounter to indentation level increases -- map, sequence -- but zero actual indentation increase until (1,3) ('M' in 'Map2'). So, when we arrive at (3,0), Scanner::unroll_indent/2 only unrolls one indent level... the second one (sequence) which contains the information we use to determine whether an indent level has a zero indent that needs to be added to the queue. Thus, when we hit the call to Scanner::pop_zero_indent_sequence/2 it looks at the current indent (map), which it correctly decides isn't zero indented. The fix is simple, just reorder the operations to check for zero indents before we unroll. That said, I probably should create some additional test cases to check that we handle these edge cases correctly. 1. Zero indent before and after real indent: (Z,N,Z) 2. Double zero indent -- not sure this is possible in YAML (Z,Z) Issue: #54 --- src/scanner/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanner/mod.rs b/src/scanner/mod.rs index a00e471..f53d791 100644 --- a/src/scanner/mod.rs +++ b/src/scanner/mod.rs @@ -139,8 +139,8 @@ impl Scanner self.expire_stale_saved_key()?; // Handle indentation unrolling - self.pop_zero_indent_sequence(*base, tokens)?; self.unroll_indent(tokens, self.stats.column)?; + self.pop_zero_indent_sequence(*base, tokens)?; // Is it the end of a stream? if base.is_empty() || self.state == StreamState::Done From a354d52d7c020119f716c5969b258d8c05caa533 Mon Sep 17 00:00:00 2001 From: Bazaah Date: Mon, 31 Oct 2022 21:19:11 +0000 Subject: [PATCH 2/3] scanner/test: add complex::zero_indent_multilevel_coalesce This test reproduces a bug report, which lead to the fix in 835ceb0. Issue: #54 --- src/scanner/tests/complex.rs | 70 ++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/src/scanner/tests/complex.rs b/src/scanner/tests/complex.rs index 4030ba7..24a14eb 100644 --- a/src/scanner/tests/complex.rs +++ b/src/scanner/tests/complex.rs @@ -185,3 +185,73 @@ fn plain() @ None ); } + +/// Check we handle zero indented indents that could be +/// incorrectly coalesced with normal indentation levels +#[test] +fn zero_indent_multilevel_coalesce() +{ + let data = r#" +Objs: +- UnitConfigName: Enemy_Lizalfos_Dark + HashId: 0x43ef248b +- UnitConfigName: Item_Fish_21 + HashId: 0x453cc5d0 # Last Ok +Rails: # Error at the beginning of this line +- Blah: SomeRail + HashId: 0x24f8f8f8 +"#; + + let mut s = ScanIter::new(data); + + tokens!(s => + | StreamStart(StreamEncoding::UTF8), + | BlockMappingStart, + | Key, + | Scalar(cow!("Objs"), Plain), + | Value, + | BlockSequenceStart, + | BlockEntry, + | BlockMappingStart, + | Key, + | Scalar(cow!("UnitConfigName"), Plain), + | Value, + | Scalar(cow!("Enemy_Lizalfos_Dark"), Plain), + | Key, + | Scalar(cow!("HashId"), Plain), + | Value, + | Scalar(cow!("0x43ef248b"), Plain), + | BlockEnd, + | BlockEntry, + | BlockMappingStart, + | Key, + | Scalar(cow!("UnitConfigName"), Plain), + | Value, + | Scalar(cow!("Item_Fish_21"), Plain), + | Key, + | Scalar(cow!("HashId"), Plain), + | Value, + | Scalar(cow!("0x453cc5d0"), Plain), + | BlockEnd => "expected END of 'UnitConfigName: Item_Fish_21' map", + | BlockEnd => "expected END of 'Objs' zero indented sequence", + | Key, + | Scalar(cow!("Rails"), Plain), + | Value, + | BlockSequenceStart, + | BlockEntry, + | BlockMappingStart, + | Key, + | Scalar(cow!("Blah"), Plain), + | Value, + | Scalar(cow!("SomeRail"), Plain), + | Key, + | Scalar(cow!("HashId"), Plain), + | Value, + | Scalar(cow!("0x24f8f8f8"), Plain), + | BlockEnd, + | BlockEnd, + | BlockEnd, + | StreamEnd, + @ None + ); +} From dacb74524959cee8be4290322924ea0af2a9c6b4 Mon Sep 17 00:00:00 2001 From: Bazaah Date: Wed, 2 Nov 2022 10:50:15 +0000 Subject: [PATCH 3/3] scanner/test: add complex::zero_indent_multilevel Check an additional edge case around zero indent handling, specifically the Zero, Normal, Zero (indent) pattern. Issue: #54 --- src/scanner/tests/complex.rs | 64 ++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/scanner/tests/complex.rs b/src/scanner/tests/complex.rs index 24a14eb..f10045a 100644 --- a/src/scanner/tests/complex.rs +++ b/src/scanner/tests/complex.rs @@ -255,3 +255,67 @@ Rails: # Error at the beginning of this line @ None ); } + +/// This test ensures that we catch zero indents on both +/// sides of a normal indentation decrease +#[test] +fn zero_indent_multilevel() +{ + let data = r#" +Z1: +- Z2: + - N1: + - N2: + - Z3: + - end +"#; + + let mut s = ScanIter::new(data); + + tokens!(s => + | StreamStart(StreamEncoding::UTF8), + | BlockMappingStart => "expected START of Z1 mapping", + | Key, + | Scalar(cow!("Z1"), Plain), + | Value, + | BlockSequenceStart => "expected START of zero indent sequence 1", + | BlockEntry, + | BlockMappingStart => "expected START of Z2 mapping", + | Key, + | Scalar(cow!("Z2"), Plain), + | Value, + | BlockSequenceStart => "expected START of zero indent sequence 2", + | BlockEntry, + | BlockMappingStart => "expected START of N1 mapping", + | Key, + | Scalar(cow!("N1"), Plain), + | Value, + | BlockSequenceStart => "expected START of normal indent sequence 1", + | BlockEntry, + | BlockMappingStart => "expected START of N2 mapping", + | Key, + | Scalar(cow!("N2"), Plain), + | Value, + | BlockSequenceStart => "expected START of normal indent sequence 2", + | BlockEntry, + | BlockMappingStart => "expected START of Z3 mapping", + | Key, + | Scalar(cow!("Z3"), Plain), + | Value, + | BlockSequenceStart => "expected START of zero indent sequence 3", + | BlockEntry, + | Scalar(cow!("end"), Plain), + | BlockEnd => "expected END of zero indent sequence 3", + | BlockEnd => "expected END of Z3 mapping", + | BlockEnd => "expected END of normal indent sequence 2", + | BlockEnd => "expected END of N2 mapping", + | BlockEnd => "expected END of normal indent sequence 1", + | BlockEnd => "expected END of N1 mapping", + | BlockEnd => "expected END of zero indent sequence 2", + | BlockEnd => "expected END of Z2 mapping", + | BlockEnd => "expected END of zero indent sequence 1", + | BlockEnd => "expected END of Z1 mapping", + | StreamEnd, + @ None + ); +}