Feature/scanner/option #27
|
@ -125,6 +125,12 @@ impl Scanner
|
||||||
return self.fetch_stream_end(*base, tokens);
|
return self.fetch_stream_end(*base, tokens);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 4 characters is the longest token we can encounter, one
|
||||||
|
// of:
|
||||||
|
// - '--- '
|
||||||
|
// - '... '
|
||||||
|
cache!(~base, 4, opts)?;
|
||||||
|
|
||||||
// Fetch the next token(s)
|
// Fetch the next token(s)
|
||||||
match base.as_bytes()
|
match base.as_bytes()
|
||||||
{
|
{
|
||||||
|
|
Loading…
Reference in a new issue