summaryrefslogtreecommitdiff
path: root/src/parse/tokens.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/parse/tokens.rs')
-rw-r--r--src/parse/tokens.rs15
1 files changed, 12 insertions, 3 deletions
diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs
index 970c0dd6..0c125b4b 100644
--- a/src/parse/tokens.rs
+++ b/src/parse/tokens.rs
@@ -202,9 +202,18 @@ impl<'s> Tokens<'s> {
'~' | '*' | '_' | '`' | '$' | '-' | '\\'
};
- self.s.eat_until(|c| {
- TABLE.get(c as usize).copied().unwrap_or_else(|| c.is_whitespace())
- });
+ loop {
+ self.s.eat_until(|c| {
+ TABLE.get(c as usize).copied().unwrap_or_else(|| c.is_whitespace())
+ });
+
+ let mut s = self.s;
+ if !(s.eat_if(' ') && s.check_or(false, char::is_alphanumeric)) {
+ break;
+ }
+
+ self.s.eat();
+ }
NodeKind::Text(self.s.eaten_from(start).into())
}