Fix newline parsing behavior in code mode (#3780)

2 files changed, 18 insertions, 3 deletions

diff --git a/crates/typst-syntax/src/lexer.rs b/crates/typst-syntax/src/lexer.rs
index aacbee62..f1d29fb3 100644
--- a/crates/typst-syntax/src/lexer.rs
+++ b/crates/typst-syntax/src/lexer.rs
@@ -88,8 +88,10 @@ impl Lexer<'_> {
     }
 }
 
-/// Shared.
+/// Shared methods with all [`LexMode`].
 impl Lexer<'_> {
+    /// Proceed to the next token and return its [`SyntaxKind`]. Note the
+    /// token could be a [trivia](SyntaxKind::is_trivia).
     pub fn next(&mut self) -> SyntaxKind {
         if self.mode == LexMode::Raw {
             let Some((kind, end)) = self.raw.pop() else {
@@ -121,6 +123,7 @@ impl Lexer<'_> {
         }
     }
 
+    /// Eat whitespace characters greedily.
     fn whitespace(&mut self, start: usize, c: char) -> SyntaxKind {
         let more = self.s.eat_while(|c| is_space(c, self.mode));
         let newlines = match c {
@@ -760,7 +763,7 @@ impl ScannerExt for Scanner<'_> {
     }
 }
 
-/// Whether a character will become a Space token in Typst
+/// Whether a character will become a [`SyntaxKind::Space`] token.
 #[inline]
 fn is_space(character: char, mode: LexMode) -> bool {
     match mode {
diff --git a/crates/typst-syntax/src/parser.rs b/crates/typst-syntax/src/parser.rs
index 50032898..17f08153 100644
--- a/crates/typst-syntax/src/parser.rs
+++ b/crates/typst-syntax/src/parser.rs
@@ -1748,15 +1748,27 @@ impl<'s> Parser<'s> {
         }
     }
 
+    fn next_non_trivia(lexer: &mut Lexer<'s>) -> SyntaxKind {
+        loop {
+            let next = lexer.next();
+            // Loop is terminatable, because SyntaxKind::Eof is not a trivia.
+            if !next.is_trivia() {
+                break next;
+            }
+        }
+    }
+
     fn lex(&mut self) {
         self.current_start = self.lexer.cursor();
         self.current = self.lexer.next();
+
+        // Special cases to handle newlines in code mode.
         if self.lexer.mode() == LexMode::Code
             && self.lexer.newline()
             && match self.newline_modes.last() {
                 Some(NewlineMode::Continue) => false,
                 Some(NewlineMode::Contextual) => !matches!(
-                    self.lexer.clone().next(),
+                    Self::next_non_trivia(&mut self.lexer.clone()),
                     SyntaxKind::Else | SyntaxKind::Dot
                 ),
                 Some(NewlineMode::Stop) => true,