Let the lexer respect linebreaks within inline raw (#3756)

2 files changed, 22 insertions, 6 deletions

diff --git a/crates/typst-syntax/src/kind.rs b/crates/typst-syntax/src/kind.rs
index c34f6002..c84e5358 100644
--- a/crates/typst-syntax/src/kind.rs
+++ b/crates/typst-syntax/src/kind.rs
@@ -32,7 +32,7 @@ pub enum SyntaxKind {
     RawLang,
     /// A raw delimiter consisting of 1 or 3+ backticks: `` ` ``.
     RawDelim,
-    /// A sequence of whitespace to ignore in a raw block: `    `.
+    /// A sequence of whitespace to ignore in a raw text: `    `.
     RawTrimmed,
     /// A hyperlink: `https://typst.org`.
     Link,
diff --git a/crates/typst-syntax/src/lexer.rs b/crates/typst-syntax/src/lexer.rs
index f1d29fb3..20cd0d60 100644
--- a/crates/typst-syntax/src/lexer.rs
+++ b/crates/typst-syntax/src/lexer.rs
@@ -275,10 +275,7 @@ impl Lexer<'_> {
         if backticks >= 3 {
             self.blocky_raw(start, end, backticks);
         } else {
-            // Single backtick needs no trimming or extra fancyness.
-            self.s.jump(end - backticks);
-            self.push_raw(SyntaxKind::Text);
-            self.s.jump(end);
+            self.inline_raw(start, end, backticks);
         }
 
         // Closing delimiter.
@@ -356,6 +353,25 @@ impl Lexer<'_> {
         self.s.jump(end);
     }
 
+    fn inline_raw(&mut self, start: usize, end: usize, backticks: usize) {
+        self.s.jump(start + backticks);
+
+        while self.s.cursor() < end - backticks {
+            if self.s.at(is_newline) {
+                self.push_raw(SyntaxKind::Text);
+                self.s.eat_newline();
+                self.push_raw(SyntaxKind::RawTrimmed);
+                continue;
+            }
+            self.s.eat();
+        }
+        self.push_raw(SyntaxKind::Text);
+
+        self.s.jump(end);
+    }
+
+    /// Push the current cursor that marks the end of a raw segment of
+    /// the given `kind`.
     fn push_raw(&mut self, kind: SyntaxKind) {
         let end = self.s.cursor();
         self.raw.push((kind, end));
@@ -821,7 +837,7 @@ pub fn link_prefix(text: &str) -> (&str, bool) {
     (s.before(), brackets.is_empty())
 }
 
-/// Split text at newlines.
+/// Split text at newlines. These newline characters are not kept.
 pub fn split_newlines(text: &str) -> Vec<&str> {
     let mut s = Scanner::new(text);
     let mut lines = Vec::new();