dolysis/yary
3
0
Fork 0
Code Issues Pull requests Releases Activity

lib/scanner: ensure Scanner does not exit before Key resolution

Browse source 
just loop until we either:

1. Have produced >1 tokens
AND
2. A key isn't possible
OR
3. We've produced stream end
This commit is contained in:
Paul Stemmet 2021-07-31 20:18:06 +00:00 committed by Paul Stemmet
parent ebda074d66
commit 6825e3ebd4
1 changed files with 12 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
src/scanner/mod.rs
View file

@ -68,20 +68,23 @@ impl Scanner
/// returning the number added.
pub fn scan_tokens<'de>(&mut self, base: &'de str, tokens: &mut Tokens<'de>) -> Result<usize>
{
if let Some(mut buffer) = base
.get(self.offset..)
.filter(|_| self.state != StreamState::Done)
let mut num_tokens = 0;
let starting_tokens = tokens.len();
while self.state != StreamState::Done
&& (starting_tokens == tokens.len() || self.key.possible())
{
let existing_tokens = tokens.len();
if let Some(mut buffer) = base.get(self.offset..)
{
self.scan_next_token(&mut buffer, tokens)?;
self.scan_next_token(&mut buffer, tokens)?;
self.offset = base.len() - buffer.len();
self.offset = base.len() - buffer.len();
return Ok(tokens.len() - existing_tokens);
num_tokens = tokens.len() - starting_tokens;
}
}
Ok(0)
Ok(num_tokens)
}
fn scan_next_token<'de>(&mut self, base: &mut &'de str, tokens: &mut Tokens<'de>)