Do not parse special spaces to Space Token (#3267)

1 files changed, 11 insertions, 2 deletions

diff --git a/crates/typst-syntax/src/lexer.rs b/crates/typst-syntax/src/lexer.rs
index 4800c8de..a9ffb216 100644
--- a/crates/typst-syntax/src/lexer.rs
+++ b/crates/typst-syntax/src/lexer.rs
@@ -90,7 +90,7 @@ impl Lexer<'_> {
         self.error = None;
         let start = self.s.cursor();
         match self.s.eat() {
-            Some(c) if c.is_whitespace() => self.whitespace(start, c),
+            Some(c) if is_space(c, self.mode) => self.whitespace(start, c),
             Some('/') if self.s.eat_if('/') => self.line_comment(),
             Some('/') if self.s.eat_if('*') => self.block_comment(),
             Some('*') if self.s.eat_if('/') => {
@@ -108,7 +108,7 @@ impl Lexer<'_> {
     }
 
     fn whitespace(&mut self, start: usize, c: char) -> SyntaxKind {
-        let more = self.s.eat_while(char::is_whitespace);
+        let more = self.s.eat_while(|c| is_space(c, self.mode));
         let newlines = match c {
             ' ' if more.is_empty() => 0,
             _ => count_newlines(self.s.from(start)),
@@ -628,6 +628,15 @@ fn keyword(ident: &str) -> Option<SyntaxKind> {
     })
 }
 
+/// Whether a character will become a Space token in Typst
+#[inline]
+fn is_space(character: char, mode: LexMode) -> bool {
+    match mode {
+        LexMode::Markup => matches!(character, ' ' | '\t') || is_newline(character),
+        _ => character.is_whitespace(),
+    }
+}
+
 /// Whether a character is interpreted as a newline by Typst.
 #[inline]
 pub fn is_newline(character: char) -> bool {