Coalesce text and simple spaces for high performance

Co-Authored-By: Martin Haug <mhaug@live.de>

1 files changed, 12 insertions, 3 deletions

diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs
index 970c0dd6..0c125b4b 100644
--- a/src/parse/tokens.rs
+++ b/src/parse/tokens.rs
@@ -202,9 +202,18 @@ impl<'s> Tokens<'s> {
             '~' | '*' | '_' | '`' | '$' | '-' | '\\'
         };
 
-        self.s.eat_until(|c| {
-            TABLE.get(c as usize).copied().unwrap_or_else(|| c.is_whitespace())
-        });
+        loop {
+            self.s.eat_until(|c| {
+                TABLE.get(c as usize).copied().unwrap_or_else(|| c.is_whitespace())
+            });
+
+            let mut s = self.s;
+            if !(s.eat_if(' ') && s.check_or(false, char::is_alphanumeric)) {
+                break;
+            }
+
+            self.s.eat();
+        }
 
         NodeKind::Text(self.s.eaten_from(start).into())
     }