From 0e5a0f79494d20496336ce9e68fecf7de074a592 Mon Sep 17 00:00:00 2001
From: Bazaah <github@luxolus.com>
Date: Thu, 9 Sep 2021 18:56:55 +0000
Subject: [PATCH] lib/scanner: add feature gated test harness for tokens!

In essence, this allows us to test the Scanner's ability to handle
chunked byte streams, hooking directly into the existing test suite.

It has three levels large, medium and small where large is probably the
smallest buffer size + increment that could be considered reasonable
(4k/64), with the smaller two testing absurd buffers (8/8 and 1/1).
---
 src/scanner/mod.rs              | 72 +++++++++++++++++++++++++++++----
 src/scanner/tests/str_reader.rs | 65 +++++++++++++++++++++++++++++
 2 files changed, 130 insertions(+), 7 deletions(-)
 create mode 100644 src/scanner/tests/str_reader.rs

diff --git a/src/scanner/mod.rs b/src/scanner/mod.rs
index 86fa5d2..468cd84 100644
--- a/src/scanner/mod.rs
+++ b/src/scanner/mod.rs
@@ -1121,12 +1121,22 @@ mod tests
     mod tag;
     mod whitespace;
 
+    #[cfg(feature = "test_buffer")]
+    mod str_reader;
+
     use super::*;
     use crate::token::{ScalarStyle::*, Token::*};
 
     struct ScanIter<'de>
     {
-        data:   &'de str,
+        #[cfg(feature = "test_buffer")]
+        data: str_reader::StrReader<'de>,
+        #[cfg(feature = "test_buffer")]
+        opts: Flags,
+
+        #[cfg(not(feature = "test_buffer"))]
+        data: &'de str,
+
         scan:   Scanner,
         tokens: Tokens<'de>,
 
@@ -1138,7 +1148,14 @@ mod tests
         pub fn new(data: &'de str) -> Self
         {
             Self {
+                #[cfg(feature = "test_buffer")]
+                data: str_reader::StrReader::new(data, str_reader::StrReader::BUF_SIZE),
+                #[cfg(feature = "test_buffer")]
+                opts: O_ZEROED | O_EXTENDABLE,
+
+                #[cfg(not(feature = "test_buffer"))]
                 data,
+
                 scan: Scanner::new(),
                 tokens: Tokens::new(),
                 done: false,
@@ -1149,12 +1166,7 @@ mod tests
         {
             if (!self.done) && self.tokens.is_empty()
             {
-                if let 0 = self
-                    .scan
-                    .scan_tokens(O_ZEROED, self.data, &mut self.tokens)?
-                {
-                    self.done = true
-                }
+                self.get_next_token()?;
             }
 
             if !self.done
@@ -1166,6 +1178,52 @@ mod tests
                 Ok(None)
             }
         }
+
+        #[cfg(feature = "test_buffer")]
+        fn get_next_token(&mut self) -> Result<()>
+        {
+            let count = loop
+            {
+                match self
+                    .scan
+                    .scan_tokens(self.opts, self.data.read(), &mut self.tokens)
+                {
+                    Ok(count) => break count,
+                    Err(e) if e == ScanError::Extend =>
+                    {
+                        self.data.expand(str_reader::StrReader::BUF_EXTEND);
+
+                        if !self.data.expandable()
+                        {
+                            self.opts.remove(O_EXTENDABLE)
+                        }
+
+                        continue;
+                    },
+                    Err(e) => return Err(e),
+                };
+            };
+
+            if count == 0
+            {
+                self.done = true
+            }
+
+            Ok(())
+        }
+
+        #[cfg(not(feature = "test_buffer"))]
+        fn get_next_token(&mut self) -> Result<()>
+        {
+            if let 0 = self
+                .scan
+                .scan_tokens(O_ZEROED, self.data, &mut self.tokens)?
+            {
+                self.done = true
+            }
+
+            Ok(())
+        }
     }
 
     impl<'de> Iterator for ScanIter<'de>
diff --git a/src/scanner/tests/str_reader.rs b/src/scanner/tests/str_reader.rs
new file mode 100644
index 0000000..1e3376a
--- /dev/null
+++ b/src/scanner/tests/str_reader.rs
@@ -0,0 +1,65 @@
+use cfg_if::cfg_if;
+
+#[derive(Debug, Clone)]
+pub(super) struct StrReader<'de>
+{
+    s:    &'de str,
+    size: usize,
+}
+
+impl<'de> StrReader<'de>
+{
+    cfg_if! {
+        if #[cfg(feature = "test_buffer_large")]
+        {
+            pub const BUF_SIZE: usize = 4 * 1024;
+            pub const BUF_EXTEND: usize = 64;
+        }
+        else if #[cfg(feature = "test_buffer_medium")]
+        {
+            pub const BUF_SIZE: usize = 8;
+            pub const BUF_EXTEND: usize = 8;
+        }
+        else if #[cfg(feature = "test_buffer_small")]
+        {
+            pub const BUF_SIZE: usize = 1;
+            pub const BUF_EXTEND: usize = 1;
+        }
+    }
+
+    pub fn new(s: &'de str, size: usize) -> Self
+    {
+        let size = std::cmp::min(s.len(), size);
+
+        Self { s, size }
+    }
+
+    pub fn read(&self) -> &'de str
+    {
+        &self.s[..self.size]
+    }
+
+    pub fn expand(&mut self, size: usize)
+    {
+        let new = self.size + size;
+
+        match self.s.len() > new
+        {
+            true => self.size = new,
+            false => self.size = self.s.len(),
+        }
+    }
+
+    pub fn expandable(&self) -> bool
+    {
+        self.size < self.s.len()
+    }
+}
+
+impl std::fmt::Display for StrReader<'_>
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result
+    {
+        self.s.fmt(f)
+    }
+}