lib/scanner: add feature gated test harness for tokens!

In essence, this allows us to test the Scanner's ability to handle
chunked byte streams, hooking directly into the existing test suite.

It has three levels large, medium and small where large is probably the
smallest buffer size + increment that could be considered reasonable
(4k/64), with the smaller two testing absurd buffers (8/8 and 1/1).

src/scanner/mod.rs

@@ -1121,12 +1121,22 @@ mod tests
     mod tag;
     mod whitespace;
 
+    #[cfg(feature = "test_buffer")]
+    mod str_reader;
+
     use super::*;
     use crate::token::{ScalarStyle::*, Token::*};
 
     struct ScanIter<'de>
     {
-        data:   &'de str,
+        #[cfg(feature = "test_buffer")]
+        data: str_reader::StrReader<'de>,
+        #[cfg(feature = "test_buffer")]
+        opts: Flags,
+
+        #[cfg(not(feature = "test_buffer"))]
+        data: &'de str,
+
         scan:   Scanner,
         tokens: Tokens<'de>,
 
@@ -1138,7 +1148,14 @@ mod tests
         pub fn new(data: &'de str) -> Self
         {
             Self {
+                #[cfg(feature = "test_buffer")]
+                data: str_reader::StrReader::new(data, str_reader::StrReader::BUF_SIZE),
+                #[cfg(feature = "test_buffer")]
+                opts: O_ZEROED | O_EXTENDABLE,
+
+                #[cfg(not(feature = "test_buffer"))]
                 data,
+
                 scan: Scanner::new(),
                 tokens: Tokens::new(),
                 done: false,
@@ -1149,12 +1166,7 @@ mod tests
         {
             if (!self.done) && self.tokens.is_empty()
             {
-                if let 0 = self
-                    .scan
-                    .scan_tokens(O_ZEROED, self.data, &mut self.tokens)?
-                {
-                    self.done = true
-                }
+                self.get_next_token()?;
             }
 
             if !self.done
@@ -1166,6 +1178,52 @@ mod tests
                 Ok(None)
             }
         }
+
+        #[cfg(feature = "test_buffer")]
+        fn get_next_token(&mut self) -> Result<()>
+        {
+            let count = loop
+            {
+                match self
+                    .scan
+                    .scan_tokens(self.opts, self.data.read(), &mut self.tokens)
+                {
+                    Ok(count) => break count,
+                    Err(e) if e == ScanError::Extend =>
+                    {
+                        self.data.expand(str_reader::StrReader::BUF_EXTEND);
+
+                        if !self.data.expandable()
+                        {
+                            self.opts.remove(O_EXTENDABLE)
+                        }
+
+                        continue;
+                    },
+                    Err(e) => return Err(e),
+                };
+            };
+
+            if count == 0
+            {
+                self.done = true
+            }
+
+            Ok(())
+        }
+
+        #[cfg(not(feature = "test_buffer"))]
+        fn get_next_token(&mut self) -> Result<()>
+        {
+            if let 0 = self
+                .scan
+                .scan_tokens(O_ZEROED, self.data, &mut self.tokens)?
+            {
+                self.done = true
+            }
+
+            Ok(())
+        }
     }
 
     impl<'de> Iterator for ScanIter<'de>

src/scanner/tests/str_reader.rs

@@ -0,0 +1,65 @@
+use cfg_if::cfg_if;
+
+#[derive(Debug, Clone)]
+pub(super) struct StrReader<'de>
+{
+    s:    &'de str,
+    size: usize,
+}
+
+impl<'de> StrReader<'de>
+{
+    cfg_if! {
+        if #[cfg(feature = "test_buffer_large")]
+        {
+            pub const BUF_SIZE: usize = 4 * 1024;
+            pub const BUF_EXTEND: usize = 64;
+        }
+        else if #[cfg(feature = "test_buffer_medium")]
+        {
+            pub const BUF_SIZE: usize = 8;
+            pub const BUF_EXTEND: usize = 8;
+        }
+        else if #[cfg(feature = "test_buffer_small")]
+        {
+            pub const BUF_SIZE: usize = 1;
+            pub const BUF_EXTEND: usize = 1;
+        }
+    }
+
+    pub fn new(s: &'de str, size: usize) -> Self
+    {
+        let size = std::cmp::min(s.len(), size);
+
+        Self { s, size }
+    }
+
+    pub fn read(&self) -> &'de str
+    {
+        &self.s[..self.size]
+    }
+
+    pub fn expand(&mut self, size: usize)
+    {
+        let new = self.size + size;
+
+        match self.s.len() > new
+        {
+            true => self.size = new,
+            false => self.size = self.s.len(),
+        }
+    }
+
+    pub fn expandable(&self) -> bool
+    {
+        self.size < self.s.len()
+    }
+}
+
+impl std::fmt::Display for StrReader<'_>
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result
+    {
+        self.s.fmt(f)
+    }
+}