Syntax functions 🚀

This adds overridable functions that markup desugars into. Specifically:
- \ desugars into linebreak
- Two newlines desugar into parbreak
- * desugars into strong
- _ desugars into emph
- = .. desugars into heading
- `..` desugars into raw

2 files changed, 76 insertions, 77 deletions

diff --git a/src/parse/mod.rs b/src/parse/mod.rs
index ceb8a206..b4727fe9 100644
--- a/src/parse/mod.rs
+++ b/src/parse/mod.rs
@@ -31,7 +31,7 @@ fn tree(p: &mut Parser) -> Tree {
     let mut tree = vec![];
     while !p.eof() {
         if let Some(node) = node(p, &mut at_start) {
-            if !matches!(node, Node::Parbreak | Node::Space) {
+            if !matches!(node, Node::Parbreak(_) | Node::Space) {
                 at_start = false;
             }
             tree.push(node);
@@ -43,19 +43,24 @@ fn tree(p: &mut Parser) -> Tree {
 /// Parse a syntax node.
 fn node(p: &mut Parser, at_start: &mut bool) -> Option<Node> {
     let token = p.peek()?;
+    let span = p.peek_span();
     let node = match token {
         // Whitespace.
         Token::Space(newlines) => {
             *at_start |= newlines > 0;
-            if newlines < 2 { Node::Space } else { Node::Parbreak }
+            if newlines < 2 {
+                Node::Space
+            } else {
+                Node::Parbreak(span)
+            }
         }
 
         // Text.
         Token::Text(text) => Node::Text(text.into()),
 
         // Markup.
-        Token::Star => Node::Strong,
-        Token::Underscore => Node::Emph,
+        Token::Star => Node::Strong(span),
+        Token::Underscore => Node::Emph(span),
         Token::Eq => {
             if *at_start {
                 return Some(heading(p));
@@ -64,7 +69,7 @@ fn node(p: &mut Parser, at_start: &mut bool) -> Option<Node> {
             }
         }
         Token::Tilde => Node::Text("\u{00A0}".into()),
-        Token::Backslash => Node::Linebreak,
+        Token::Backslash => Node::Linebreak(span),
         Token::Raw(t) => raw(p, t),
         Token::UnicodeEscape(t) => Node::Text(unicode_escape(p, t)),
 
@@ -120,28 +125,33 @@ fn heading(p: &mut Parser) -> Node {
     p.assert(Token::Eq);
 
     // Count depth.
-    let mut level: usize = 0;
+    let mut level: usize = 1;
     while p.eat_if(Token::Eq) {
         level += 1;
     }
 
-    if level > 5 {
+    if level > 6 {
         p.diag(warning!(start .. p.end(), "should not exceed depth 6"));
-        level = 5;
+        level = 6;
     }
 
     // Parse the heading contents.
-    let mut contents = vec![];
+    let mut tree = vec![];
     while p.check(|t| !matches!(t, Token::Space(n) if n >= 1)) {
-        contents.extend(node(p, &mut false));
+        tree.extend(node(p, &mut false));
     }
 
-    Node::Heading(HeadingNode { level, contents })
+    Node::Heading(HeadingNode {
+        span: p.span(start),
+        level,
+        contents: Rc::new(tree),
+    })
 }
 
 /// Handle a raw block.
 fn raw(p: &mut Parser, token: RawToken) -> Node {
-    let raw = resolve::resolve_raw(token.text, token.backticks, p.start());
+    let span = p.peek_span();
+    let raw = resolve::resolve_raw(span, token.text, token.backticks);
     if !token.terminated {
         p.diag(error!(p.peek_span().end, "expected backtick(s)"));
     }
@@ -280,17 +290,18 @@ fn primary(p: &mut Parser, atomic: bool) -> Option<Expr> {
 
 /// Parse a literal.
 fn literal(p: &mut Parser) -> Option<Expr> {
-    let kind = match p.peek()? {
+    let span = p.peek_span();
+    let expr = match p.peek()? {
         // Basic values.
-        Token::None => LitKind::None,
-        Token::Bool(b) => LitKind::Bool(b),
-        Token::Int(i) => LitKind::Int(i),
-        Token::Float(f) => LitKind::Float(f),
-        Token::Length(val, unit) => LitKind::Length(val, unit),
-        Token::Angle(val, unit) => LitKind::Angle(val, unit),
-        Token::Percent(p) => LitKind::Percent(p),
-        Token::Color(color) => LitKind::Color(color),
-        Token::Str(token) => LitKind::Str({
+        Token::None => Expr::None(span),
+        Token::Bool(b) => Expr::Bool(span, b),
+        Token::Int(i) => Expr::Int(span, i),
+        Token::Float(f) => Expr::Float(span, f),
+        Token::Length(val, unit) => Expr::Length(span, val, unit),
+        Token::Angle(val, unit) => Expr::Angle(span, val, unit),
+        Token::Percent(p) => Expr::Percent(span, p),
+        Token::Color(color) => Expr::Color(span, color),
+        Token::Str(token) => Expr::Str(span, {
             if !token.terminated {
                 p.expected_at("quote", p.peek_span().end);
             }
@@ -298,7 +309,8 @@ fn literal(p: &mut Parser) -> Option<Expr> {
         }),
         _ => return None,
     };
-    Some(Expr::Lit(Lit { span: p.eat_span(), kind }))
+    p.eat();
+    Some(expr)
 }
 
 /// Parse something that starts with a parenthesis, which can be either of:
diff --git a/src/parse/resolve.rs b/src/parse/resolve.rs
index 1f33198a..b3fdef4a 100644
--- a/src/parse/resolve.rs
+++ b/src/parse/resolve.rs
@@ -1,5 +1,5 @@
 use super::{is_newline, Scanner};
-use crate::syntax::{Ident, Pos, RawNode};
+use crate::syntax::{Ident, RawNode, Span};
 
 /// Resolve all escape sequences in a string.
 pub fn resolve_string(string: &str) -> String {
@@ -47,19 +47,17 @@ pub fn resolve_hex(sequence: &str) -> Option<char> {
 }
 
 /// Resolve the language tag and trims the raw text.
-pub fn resolve_raw(text: &str, backticks: usize, start: Pos) -> RawNode {
+pub fn resolve_raw(span: Span, text: &str, backticks: usize) -> RawNode {
     if backticks > 1 {
         let (tag, inner) = split_at_lang_tag(text);
-        let (lines, had_newline) = trim_and_split_raw(inner);
-        RawNode {
-            lang: Ident::new(tag, start .. start + tag.len()),
-            lines,
-            block: had_newline,
-        }
+        let (text, block) = trim_and_split_raw(inner);
+        let lang = Ident::new(tag, span.start .. span.start + tag.len());
+        RawNode { span, lang, text, block }
     } else {
         RawNode {
+            span,
             lang: None,
-            lines: split_lines(text),
+            text: split_lines(text).join("\n"),
             block: false,
         }
     }
@@ -77,7 +75,7 @@ fn split_at_lang_tag(raw: &str) -> (&str, &str) {
 /// Trim raw text and splits it into lines.
 ///
 /// Returns whether at least one newline was contained in `raw`.
-fn trim_and_split_raw(mut raw: &str) -> (Vec<String>, bool) {
+fn trim_and_split_raw(mut raw: &str) -> (String, bool) {
     // Trims one space at the start.
     raw = raw.strip_prefix(' ').unwrap_or(raw);
 
@@ -87,8 +85,8 @@ fn trim_and_split_raw(mut raw: &str) -> (Vec<String>, bool) {
     }
 
     let mut lines = split_lines(raw);
-    let had_newline = lines.len() > 1;
     let is_whitespace = |line: &String| line.chars().all(char::is_whitespace);
+    let had_newline = lines.len() > 1;
 
     // Trims a sequence of whitespace followed by a newline at the start.
     if lines.first().map_or(false, is_whitespace) {
@@ -100,7 +98,7 @@ fn trim_and_split_raw(mut raw: &str) -> (Vec<String>, bool) {
         lines.pop();
     }
 
-    (lines, had_newline)
+    (lines.join("\n"), had_newline)
 }
 
 /// Split a string into a vector of lines
@@ -171,64 +169,53 @@ mod tests {
             raw: &str,
             backticks: usize,
             lang: Option<&str>,
-            lines: &[&str],
+            text: &str,
             block: bool,
         ) {
-            Span::without_cmp(|| assert_eq!(resolve_raw(raw, backticks, Pos(0)), RawNode {
-                lang: lang.and_then(|id| Ident::new(id, 0)),
-                lines: lines.iter().map(ToString::to_string).collect(),
-                block,
-            }));
+            Span::without_cmp(|| {
+                assert_eq!(resolve_raw(Span::ZERO, raw, backticks), RawNode {
+                    span: Span::ZERO,
+                    lang: lang.and_then(|id| Ident::new(id, 0)),
+                    text: text.into(),
+                    block,
+                });
+            });
         }
 
         // Just one backtick.
-        test("py",     1, None, &["py"],     false);
-        test("1\n2",   1, None, &["1", "2"], false);
-        test("1\r\n2", 1, None, &["1", "2"], false);
+        test("py",     1, None, "py",     false);
+        test("1\n2",   1, None, "1\n2", false);
+        test("1\r\n2", 1, None, "1\n2", false);
 
         // More than one backtick with lang tag.
-        test("js alert()",     2, Some("js"), &["alert()"],        false);
-        test("py quit(\n\n)",  3, Some("py"), &["quit(", "", ")"], true);
-        test("♥",              2, None,       &[],                 false);
+        test("js alert()",     2, Some("js"), "alert()",        false);
+        test("py quit(\n\n)",  3, Some("py"), "quit(\n\n)", true);
+        test("♥",              2, None,       "",               false);
 
         // Trimming of whitespace (tested more thoroughly in separate test).
-        test(" a",   2, None, &["a"],  false);
-        test("  a",  2, None, &[" a"], false);
-        test(" \na", 2, None, &["a"],  true);
+        test(" a",   2, None, "a",  false);
+        test("  a",  2, None, " a", false);
+        test(" \na", 2, None, "a",  true);
     }
 
     #[test]
     fn test_trim_raw() {
         #[track_caller]
-        fn test(text: &str, expected: Vec<&str>) {
+        fn test(text: &str, expected: &str) {
             assert_eq!(trim_and_split_raw(text).0, expected);
         }
 
-        test(" hi",          vec!["hi"]);
-        test("  hi",         vec![" hi"]);
-        test("\nhi",         vec!["hi"]);
-        test("    \n hi",    vec![" hi"]);
-        test("hi` ",         vec!["hi`"]);
-        test("hi`  ",        vec!["hi` "]);
-        test("hi`   ",       vec!["hi`  "]);
-        test("hi ",          vec!["hi "]);
-        test("hi  ",         vec!["hi  "]);
-        test("hi\n",         vec!["hi"]);
-        test("hi \n   ",     vec!["hi "]);
-        test("  \n hi \n  ", vec![" hi "]);
-    }
-
-    #[test]
-    fn test_split_lines() {
-        #[track_caller]
-        fn test(text: &str, expected: Vec<&str>) {
-            assert_eq!(split_lines(text), expected);
-        }
-
-        test("raw\ntext",  vec!["raw", "text"]);
-        test("a\r\nb",     vec!["a", "b"]);
-        test("a\n\nb",     vec!["a", "", "b"]);
-        test("a\r\x0Bb",   vec!["a", "", "b"]);
-        test("a\r\n\r\nb", vec!["a", "", "b"]);
+        test(" hi",          "hi");
+        test("  hi",         " hi");
+        test("\nhi",         "hi");
+        test("    \n hi",    " hi");
+        test("hi` ",         "hi`");
+        test("hi`  ",        "hi` ");
+        test("hi`   ",       "hi`  ");
+        test("hi ",          "hi ");
+        test("hi  ",         "hi  ");
+        test("hi\n",         "hi");
+        test("hi \n   ",     "hi ");
+        test("  \n hi \n  ", " hi ");
     }
 }