More math syntax

7 files changed, 646 insertions, 565 deletions

diff --git a/src/syntax/ast.rs b/src/syntax/ast.rs
index c44fa2a0..55586feb 100644
--- a/src/syntax/ast.rs
+++ b/src/syntax/ast.rs
@@ -107,8 +107,6 @@ pub enum MarkupNode {
     Enum(EnumItem),
     /// An item in a description list: `/ Term: Details`.
     Desc(DescItem),
-    /// A math formula: `$x$`, `$ x^2 $`.
-    Math(Math),
     /// An expression.
     Expr(Expr),
 }
@@ -132,7 +130,6 @@ impl AstNode for MarkupNode {
             SyntaxKind::ListItem => node.cast().map(Self::List),
             SyntaxKind::EnumItem => node.cast().map(Self::Enum),
             SyntaxKind::DescItem => node.cast().map(Self::Desc),
-            SyntaxKind::Math => node.cast().map(Self::Math),
             _ => node.cast().map(Self::Expr),
         }
     }
@@ -155,7 +152,6 @@ impl AstNode for MarkupNode {
             Self::List(v) => v.as_untyped(),
             Self::Enum(v) => v.as_untyped(),
             Self::Desc(v) => v.as_untyped(),
-            Self::Math(v) => v.as_untyped(),
             Self::Expr(v) => v.as_untyped(),
         }
     }
@@ -447,6 +443,9 @@ pub enum MathNode {
     Escape(Escape),
     /// An atom: `x`, `+`, `12`.
     Atom(Atom),
+    /// Symbol notation: `:arrow:l:` or `arrow:l`. Notations without any colons
+    /// are parsed as identifier expression and handled during evaluation.
+    Symbol(Symbol),
     /// A base with optional sub- and superscripts: `a_1^2`.
     Script(Script),
     /// A fraction: `x/2`.
@@ -466,6 +465,7 @@ impl AstNode for MathNode {
             SyntaxKind::Linebreak => node.cast().map(Self::Linebreak),
             SyntaxKind::Escape(_) => node.cast().map(Self::Escape),
             SyntaxKind::Atom(_) => node.cast().map(Self::Atom),
+            SyntaxKind::Symbol(_) => node.cast().map(Self::Symbol),
             SyntaxKind::Script => node.cast().map(Self::Script),
             SyntaxKind::Frac => node.cast().map(Self::Frac),
             SyntaxKind::Align => node.cast().map(Self::Align),
@@ -480,6 +480,7 @@ impl AstNode for MathNode {
             Self::Linebreak(v) => v.as_untyped(),
             Self::Escape(v) => v.as_untyped(),
             Self::Atom(v) => v.as_untyped(),
+            Self::Symbol(v) => v.as_untyped(),
             Self::Script(v) => v.as_untyped(),
             Self::Frac(v) => v.as_untyped(),
             Self::Align(v) => v.as_untyped(),
@@ -574,6 +575,8 @@ pub enum Expr {
     Code(CodeBlock),
     /// A content block: `[*Hi* there!]`.
     Content(ContentBlock),
+    /// A math formula: `$x$`, `$ x^2 $`.
+    Math(Math),
     /// A grouped expression: `(1 + 2)`.
     Parenthesized(Parenthesized),
     /// An array: `(1, "hi", 12cm)`.
@@ -622,6 +625,7 @@ impl AstNode for Expr {
             SyntaxKind::Ident(_) => node.cast().map(Self::Ident),
             SyntaxKind::CodeBlock => node.cast().map(Self::Code),
             SyntaxKind::ContentBlock => node.cast().map(Self::Content),
+            SyntaxKind::Math => node.cast().map(Self::Math),
             SyntaxKind::Parenthesized => node.cast().map(Self::Parenthesized),
             SyntaxKind::Array => node.cast().map(Self::Array),
             SyntaxKind::Dict => node.cast().map(Self::Dict),
@@ -651,6 +655,7 @@ impl AstNode for Expr {
             Self::Lit(v) => v.as_untyped(),
             Self::Code(v) => v.as_untyped(),
             Self::Content(v) => v.as_untyped(),
+            Self::Math(v) => v.as_untyped(),
             Self::Ident(v) => v.as_untyped(),
             Self::Array(v) => v.as_untyped(),
             Self::Dict(v) => v.as_untyped(),
diff --git a/src/syntax/highlight.rs b/src/syntax/highlight.rs
index d4da7b3e..3fed905f 100644
--- a/src/syntax/highlight.rs
+++ b/src/syntax/highlight.rs
@@ -163,8 +163,6 @@ pub enum Category {
     ListMarker,
     /// A term in a description list.
     ListTerm,
-    /// A full math formula.
-    Math,
     /// The delimiters of a math formula.
     MathDelimiter,
     /// An operator with special meaning in a math formula.
@@ -300,15 +298,17 @@ impl Category {
             SyntaxKind::EnumItem => Some(Category::ListItem),
             SyntaxKind::EnumNumbering(_) => Some(Category::ListMarker),
             SyntaxKind::DescItem => Some(Category::ListItem),
-            SyntaxKind::Math => Some(Category::Math),
+            SyntaxKind::Math => None,
             SyntaxKind::Atom(_) => None,
             SyntaxKind::Script => None,
             SyntaxKind::Frac => None,
             SyntaxKind::Align => None,
 
             SyntaxKind::Ident(_) => match parent.kind() {
-                SyntaxKind::Markup { .. } => Some(Category::Interpolated),
-                SyntaxKind::Math => Some(Category::Interpolated),
+                SyntaxKind::Markup { .. }
+                | SyntaxKind::Math
+                | SyntaxKind::Script
+                | SyntaxKind::Frac => Some(Category::Interpolated),
                 SyntaxKind::FuncCall => Some(Category::Function),
                 SyntaxKind::MethodCall if i > 0 => Some(Category::Function),
                 SyntaxKind::Closure if i == 0 => Some(Category::Function),
@@ -378,7 +378,6 @@ impl Category {
             Self::Emph => "markup.italic.typst",
             Self::Link => "markup.underline.link.typst",
             Self::Raw => "markup.raw.typst",
-            Self::Math => "string.other.math.typst",
             Self::MathDelimiter => "punctuation.definition.math.typst",
             Self::MathOperator => "keyword.operator.math.typst",
             Self::Heading => "markup.heading.typst",
diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs
index 2ef49322..c461a589 100644
--- a/src/syntax/mod.rs
+++ b/src/syntax/mod.rs
@@ -24,19 +24,4 @@ use incremental::reparse;
 use parser::*;
 
 #[cfg(test)]
-mod tests {
-    use std::fmt::Debug;
-
-    #[track_caller]
-    pub fn check<T>(text: &str, found: T, expected: T)
-    where
-        T: Debug + PartialEq,
-    {
-        if found != expected {
-            println!("source:   {text:?}");
-            println!("expected: {expected:#?}");
-            println!("found:    {found:#?}");
-            panic!("test failed");
-        }
-    }
-}
+mod tests;
diff --git a/src/syntax/parser.rs b/src/syntax/parser.rs
index ff4a4952..3e133fe1 100644
--- a/src/syntax/parser.rs
+++ b/src/syntax/parser.rs
@@ -235,17 +235,9 @@ impl<'s> Parser<'s> {
     pub fn start_group(&mut self, kind: Group) {
         self.groups.push(GroupEntry { kind, prev_mode: self.tokens.mode() });
         self.tokens.set_mode(match kind {
-            Group::Strong | Group::Emph => TokenMode::Markup,
-            Group::Bracket => match self.tokens.mode() {
-                TokenMode::Math => TokenMode::Math,
-                _ => TokenMode::Markup,
-            },
-            Group::Brace | Group::Paren => match self.tokens.mode() {
-                TokenMode::Math => TokenMode::Math,
-                _ => TokenMode::Code,
-            },
-            Group::Math => TokenMode::Math,
-            Group::Expr | Group::Imports => TokenMode::Code,
+            Group::Bracket | Group::Strong | Group::Emph => TokenMode::Markup,
+            Group::Math | Group::MathRow(_, _) => TokenMode::Math,
+            Group::Brace | Group::Paren | Group::Expr | Group::Imports => TokenMode::Code,
         });
 
         match kind {
@@ -255,6 +247,7 @@ impl<'s> Parser<'s> {
             Group::Strong => self.assert(SyntaxKind::Star),
             Group::Emph => self.assert(SyntaxKind::Underscore),
             Group::Math => self.assert(SyntaxKind::Dollar),
+            Group::MathRow(l, _) => self.assert(SyntaxKind::Atom(l.into())),
             Group::Expr => self.repeek(),
             Group::Imports => self.repeek(),
         }
@@ -279,6 +272,7 @@ impl<'s> Parser<'s> {
             Group::Strong => Some((SyntaxKind::Star, true)),
             Group::Emph => Some((SyntaxKind::Underscore, true)),
             Group::Math => Some((SyntaxKind::Dollar, true)),
+            Group::MathRow(_, r) => Some((SyntaxKind::Atom(r.into()), true)),
             Group::Expr => Some((SyntaxKind::Semicolon, false)),
             Group::Imports => None,
         } {
@@ -344,9 +338,17 @@ impl<'s> Parser<'s> {
             Some(SyntaxKind::RightParen) => self.inside(Group::Paren),
             Some(SyntaxKind::Star) => self.inside(Group::Strong),
             Some(SyntaxKind::Underscore) => self.inside(Group::Emph),
-            Some(SyntaxKind::Dollar) => self.inside(Group::Math),
+            Some(SyntaxKind::Dollar) => {
+                self.groups.last().map(|group| group.kind) == Some(Group::Math)
+            }
             Some(SyntaxKind::Semicolon) => self.inside(Group::Expr),
             Some(SyntaxKind::From) => self.inside(Group::Imports),
+            Some(SyntaxKind::Atom(s)) => match s.as_str() {
+                ")" => self.inside(Group::MathRow('(', ')')),
+                "}" => self.inside(Group::MathRow('{', '}')),
+                "]" => self.inside(Group::MathRow('[', ']')),
+                _ => false,
+            },
             Some(SyntaxKind::Space { newlines }) => self.space_ends_group(*newlines),
             Some(_) => false,
             None => true,
@@ -531,6 +533,8 @@ pub enum Group {
     Emph,
     /// A group surrounded by dollar signs: `$...$`.
     Math,
+    /// A group surrounded by math delimiters.
+    MathRow(char, char),
     /// A group ended by a semicolon or a line break: `;`, `\n`.
     Expr,
     /// A group for import items, ended by a semicolon, line break or `from`.
diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs
index 59e066a6..5bd5e63b 100644
--- a/src/syntax/parsing.rs
+++ b/src/syntax/parsing.rs
@@ -268,7 +268,7 @@ fn markup_node(p: &mut Parser, at_start: &mut bool) {
         | SyntaxKind::Include
         | SyntaxKind::Break
         | SyntaxKind::Continue
-        | SyntaxKind::Return => markup_expr(p),
+        | SyntaxKind::Return => embedded_expr(p),
 
         // Code and content block.
         SyntaxKind::LeftBrace => code_block(p),
@@ -359,7 +359,7 @@ fn desc_item(p: &mut Parser, at_start: bool) -> ParseResult {
     Ok(())
 }
 
-fn markup_expr(p: &mut Parser) {
+fn embedded_expr(p: &mut Parser) {
     // Does the expression need termination or can content follow directly?
     let stmt = matches!(
         p.peek(),
@@ -437,36 +437,63 @@ fn math_node_prec(p: &mut Parser, min_prec: usize, stop: Option<SyntaxKind>) {
 fn math_primary(p: &mut Parser) {
     let Some(token) = p.peek() else { return };
     match token {
-        // Spaces, atoms and expressions.
+        // Spaces and expressions.
         SyntaxKind::Space { .. }
         | SyntaxKind::Linebreak
         | SyntaxKind::Escape(_)
-        | SyntaxKind::Atom(_)
-        | SyntaxKind::Ident(_) => p.eat(),
+        | SyntaxKind::Str(_)
+        | SyntaxKind::Symbol(_) => p.eat(),
 
-        // Groups.
-        SyntaxKind::LeftParen => math_group(p, Group::Paren, '(', ')'),
-        SyntaxKind::LeftBracket => math_group(p, Group::Bracket, '[', ']'),
-        SyntaxKind::LeftBrace => math_group(p, Group::Brace, '{', '}'),
+        // Atoms.
+        SyntaxKind::Atom(s) => match s.as_str() {
+            "(" => math_group(p, Group::MathRow('(', ')')),
+            "{" => math_group(p, Group::MathRow('{', '}')),
+            "[" => math_group(p, Group::MathRow('[', ']')),
+            _ => p.eat(),
+        },
 
         // Alignment indactor.
         SyntaxKind::Amp => math_align(p),
 
+        // Identifiers and math calls.
+        SyntaxKind::Ident(_) => {
+            let marker = p.marker();
+            p.eat();
+
+            // Parenthesis or bracket means this is a function call.
+            if matches!(p.peek_direct(), Some(SyntaxKind::Atom(s)) if s == "(") {
+                marker.perform(p, SyntaxKind::FuncCall, math_args);
+            }
+        }
+
+        // Hashtag + keyword / identifier.
+        SyntaxKind::Let
+        | SyntaxKind::Set
+        | SyntaxKind::Show
+        | SyntaxKind::If
+        | SyntaxKind::While
+        | SyntaxKind::For
+        | SyntaxKind::Import
+        | SyntaxKind::Include
+        | SyntaxKind::Break
+        | SyntaxKind::Continue
+        | SyntaxKind::Return => embedded_expr(p),
+
+        // Code and content block.
+        SyntaxKind::LeftBrace => code_block(p),
+        SyntaxKind::LeftBracket => content_block(p),
+
         _ => p.unexpected(),
     }
 }
 
-fn math_group(p: &mut Parser, group: Group, l: char, r: char) {
+fn math_group(p: &mut Parser, group: Group) {
     p.perform(SyntaxKind::Math, |p| {
-        let marker = p.marker();
         p.start_group(group);
-        marker.convert(p, SyntaxKind::Atom(l.into()));
         while !p.eof() {
             math_node(p);
         }
-        let marker = p.marker();
         p.end_group();
-        marker.convert(p, SyntaxKind::Atom(r.into()));
     })
 }
 
@@ -582,6 +609,7 @@ fn primary(p: &mut Parser, atomic: bool) -> ParseResult {
         Some(SyntaxKind::LeftParen) => parenthesized(p, atomic),
         Some(SyntaxKind::LeftBrace) => Ok(code_block(p)),
         Some(SyntaxKind::LeftBracket) => Ok(content_block(p)),
+        Some(SyntaxKind::Dollar) => Ok(math(p)),
 
         // Keywords.
         Some(SyntaxKind::Let) => let_binding(p),
@@ -902,6 +930,28 @@ fn args(p: &mut Parser) -> ParseResult {
     Ok(())
 }
 
+fn math_args(p: &mut Parser) {
+    p.start_group(Group::MathRow('(', ')'));
+    p.perform(SyntaxKind::Args, |p| {
+        let mut marker = p.marker();
+        while !p.eof() {
+            if matches!(p.peek(), Some(SyntaxKind::Atom(s)) if s == ",") {
+                marker.end(p, SyntaxKind::Math);
+                let comma = p.marker();
+                p.eat();
+                comma.convert(p, SyntaxKind::Comma);
+                marker = p.marker();
+            } else {
+                math_node(p);
+            }
+        }
+        if marker != p.marker() {
+            marker.end(p, SyntaxKind::Math);
+        }
+    });
+    p.end_group();
+}
+
 fn let_binding(p: &mut Parser) -> ParseResult {
     p.perform(SyntaxKind::LetBinding, |p| {
         p.assert(SyntaxKind::Let);
diff --git a/src/syntax/tests.rs b/src/syntax/tests.rs
new file mode 100644
index 00000000..7b5dd870
--- /dev/null
+++ b/src/syntax/tests.rs
@@ -0,0 +1,483 @@
+#![allow(non_snake_case)]
+
+use std::num::NonZeroUsize;
+use std::sync::Arc;
+
+use super::*;
+use crate::geom::{AbsUnit, AngleUnit};
+
+use ErrorPos::*;
+use Option::None;
+use SyntaxKind::*;
+use TokenMode::{Code, Markup};
+
+use std::fmt::Debug;
+
+#[track_caller]
+pub fn check<T>(text: &str, found: T, expected: T)
+where
+    T: Debug + PartialEq,
+{
+    if found != expected {
+        println!("source:   {text:?}");
+        println!("expected: {expected:#?}");
+        println!("found:    {found:#?}");
+        panic!("test failed");
+    }
+}
+
+fn Space(newlines: usize) -> SyntaxKind {
+    SyntaxKind::Space { newlines }
+}
+
+fn Raw(text: &str, lang: Option<&str>, block: bool) -> SyntaxKind {
+    SyntaxKind::Raw(Arc::new(RawFields {
+        text: text.into(),
+        lang: lang.map(Into::into),
+        block,
+    }))
+}
+
+fn Str(string: &str) -> SyntaxKind {
+    SyntaxKind::Str(string.into())
+}
+
+fn Text(string: &str) -> SyntaxKind {
+    SyntaxKind::Text(string.into())
+}
+
+fn Ident(ident: &str) -> SyntaxKind {
+    SyntaxKind::Ident(ident.into())
+}
+
+fn Error(pos: ErrorPos, message: &str) -> SyntaxKind {
+    SyntaxKind::Error(pos, message.into())
+}
+
+/// Building blocks for suffix testing.
+///
+/// We extend each test case with a collection of different suffixes to make
+/// sure tokens end at the correct position. These suffixes are split into
+/// blocks, which can be disabled/enabled per test case. For example, when
+/// testing identifiers we disable letter suffixes because these would
+/// mingle with the identifiers.
+///
+/// Suffix blocks:
+/// - ' ': spacing
+/// - 'a': letters
+/// - '1': numbers
+/// - '/': symbols
+const BLOCKS: &str = " a1/";
+
+// Suffixes described by four-tuples of:
+//
+// - block the suffix is part of
+// - mode in which the suffix is applicable
+// - the suffix string
+// - the resulting suffix NodeKind
+fn suffixes() -> impl Iterator<Item = (char, Option<TokenMode>, &'static str, SyntaxKind)>
+{
+    [
+        // Whitespace suffixes.
+        (' ', None, " ", Space(0)),
+        (' ', None, "\n", Space(1)),
+        (' ', None, "\r", Space(1)),
+        (' ', None, "\r\n", Space(1)),
+        // Letter suffixes.
+        ('a', Some(Markup), "hello", Text("hello")),
+        ('a', Some(Markup), "💚", Text("💚")),
+        ('a', Some(Code), "val", Ident("val")),
+        ('a', Some(Code), "α", Ident("α")),
+        ('a', Some(Code), "_", Ident("_")),
+        // Number suffixes.
+        ('1', Some(Code), "2", Int(2)),
+        ('1', Some(Code), ".2", Float(0.2)),
+        // Symbol suffixes.
+        ('/', None, "[", LeftBracket),
+        ('/', None, "//", LineComment),
+        ('/', None, "/**/", BlockComment),
+        ('/', Some(Markup), "*", Star),
+        ('/', Some(Markup), r"\\", Escape('\\')),
+        ('/', Some(Markup), "#let", Let),
+        ('/', Some(Code), "(", LeftParen),
+        ('/', Some(Code), ":", Colon),
+        ('/', Some(Code), "+=", PlusEq),
+    ]
+    .into_iter()
+}
+
+macro_rules! t {
+    (Both $($tts:tt)*) => {
+        t!(Markup $($tts)*);
+        t!(Code $($tts)*);
+    };
+    ($mode:ident $([$blocks:literal])?: $text:expr => $($token:expr),*) => {{
+        // Test without suffix.
+        t!(@$mode: $text => $($token),*);
+
+        // Test with each applicable suffix.
+        for (block, mode, suffix, ref token) in suffixes() {
+            let text = $text;
+            #[allow(unused_variables)]
+            let blocks = BLOCKS;
+            $(let blocks = $blocks;)?
+            assert!(!blocks.contains(|c| !BLOCKS.contains(c)));
+            if (mode.is_none() || mode == Some($mode)) && blocks.contains(block) {
+                t!(@$mode: format!("{}{}", text, suffix) => $($token,)* token);
+            }
+        }
+    }};
+    (@$mode:ident: $text:expr => $($token:expr),*) => {{
+        let text = $text;
+        let found = Tokens::new(&text, $mode).collect::<Vec<_>>();
+        let expected = vec![$($token.clone()),*];
+        check(&text, found, expected);
+    }};
+}
+
+#[test]
+fn test_tokenize_brackets() {
+    // Test in markup.
+    t!(Markup: "{"       => LeftBrace);
+    t!(Markup: "}"       => RightBrace);
+    t!(Markup: "["       => LeftBracket);
+    t!(Markup: "]"       => RightBracket);
+    t!(Markup[" /"]: "(" => Text("("));
+    t!(Markup[" /"]: ")" => Text(")"));
+
+    // Test in code.
+    t!(Code: "{" => LeftBrace);
+    t!(Code: "}" => RightBrace);
+    t!(Code: "[" => LeftBracket);
+    t!(Code: "]" => RightBracket);
+    t!(Code: "(" => LeftParen);
+    t!(Code: ")" => RightParen);
+}
+
+#[test]
+fn test_tokenize_whitespace() {
+    // Test basic whitespace.
+    t!(Both["a1/"]: ""         => );
+    t!(Both["a1/"]: " "        => Space(0));
+    t!(Both["a1/"]: "    "     => Space(0));
+    t!(Both["a1/"]: "\t"       => Space(0));
+    t!(Both["a1/"]: "  \t"     => Space(0));
+    t!(Both["a1/"]: "\u{202F}" => Space(0));
+
+    // Test newline counting.
+    t!(Both["a1/"]: "\n"           => Space(1));
+    t!(Both["a1/"]: "\n "          => Space(1));
+    t!(Both["a1/"]: "  \n"         => Space(1));
+    t!(Both["a1/"]: "  \n   "      => Space(1));
+    t!(Both["a1/"]: "\r\n"         => Space(1));
+    t!(Both["a1/"]: "\r\n\r"       => Space(2));
+    t!(Both["a1/"]: "  \n\t \n  "  => Space(2));
+    t!(Both["a1/"]: "\n\r"         => Space(2));
+    t!(Both["a1/"]: " \r\r\n \x0D" => Space(3));
+}
+
+#[test]
+fn test_tokenize_text() {
+    // Test basic text.
+    t!(Markup[" /"]: "hello"      => Text("hello"));
+    t!(Markup[" /"]: "reha-world" => Text("reha-world"));
+
+    // Test code symbols in text.
+    t!(Markup[" /"]: "a():\"b" => Text("a()"), Colon, SmartQuote { double: true }, Text("b"));
+    t!(Markup[" /"]: ";,|/+"  => Text(";,|/+"));
+    t!(Markup[" /"]: "=-a"     => Eq, Minus, Text("a"));
+    t!(Markup[" "]: "#123"     => Text("#123"));
+
+    // Test text ends.
+    t!(Markup[""]: "hello " => Text("hello"), Space(0));
+    t!(Markup[""]: "hello~" => Text("hello"), Shorthand('\u{00A0}'));
+}
+
+#[test]
+fn test_tokenize_escape_sequences() {
+    // Test escapable symbols.
+    t!(Markup: r"\\" => Escape('\\'));
+    t!(Markup: r"\/" => Escape('/'));
+    t!(Markup: r"\[" => Escape('['));
+    t!(Markup: r"\]" => Escape(']'));
+    t!(Markup: r"\{" => Escape('{'));
+    t!(Markup: r"\}" => Escape('}'));
+    t!(Markup: r"\*" => Escape('*'));
+    t!(Markup: r"\_" => Escape('_'));
+    t!(Markup: r"\=" => Escape('='));
+    t!(Markup: r"\~" => Escape('~'));
+    t!(Markup: r"\'" => Escape('\''));
+    t!(Markup: r#"\""# => Escape('"'));
+    t!(Markup: r"\`" => Escape('`'));
+    t!(Markup: r"\$" => Escape('$'));
+    t!(Markup: r"\#" => Escape('#'));
+    t!(Markup: r"\a"   => Escape('a'));
+    t!(Markup: r"\u"   => Escape('u'));
+    t!(Markup: r"\1"   => Escape('1'));
+
+    // Test basic unicode escapes.
+    t!(Markup: r"\u{}"     => Error(Full, "invalid unicode escape sequence"));
+    t!(Markup: r"\u{2603}" => Escape('☃'));
+    t!(Markup: r"\u{P}"    => Error(Full, "invalid unicode escape sequence"));
+
+    // Test unclosed unicode escapes.
+    t!(Markup[" /"]: r"\u{"     => Error(End, "expected closing brace"));
+    t!(Markup[" /"]: r"\u{1"    => Error(End, "expected closing brace"));
+    t!(Markup[" /"]: r"\u{26A4" => Error(End, "expected closing brace"));
+    t!(Markup[" /"]: r"\u{1Q3P" => Error(End, "expected closing brace"));
+    t!(Markup: r"\u{1🏕}"       => Error(End, "expected closing brace"), Text("🏕"), RightBrace);
+}
+
+#[test]
+fn test_tokenize_markup_symbols() {
+    // Test markup tokens.
+    t!(Markup[" a1"]: "*"   => Star);
+    t!(Markup: "_"          => Underscore);
+    t!(Markup[""]: "==="    => Eq, Eq, Eq);
+    t!(Markup["a1/"]: "= "  => Eq, Space(0));
+    t!(Markup[" "]: r"\"    => Linebreak);
+    t!(Markup: "~"          => Shorthand('\u{00A0}'));
+    t!(Markup["a1/"]: "-?"  => Shorthand('\u{00AD}'));
+    t!(Markup["a "]: r"a--" => Text("a"), Shorthand('\u{2013}'));
+    t!(Markup["a1/"]: "- "  => Minus, Space(0));
+    t!(Markup[" "]: "+"     => Plus);
+    t!(Markup[" "]: "1."    => EnumNumbering(NonZeroUsize::new(1).unwrap()));
+    t!(Markup[" "]: "1.a"   => EnumNumbering(NonZeroUsize::new(1).unwrap()), Text("a"));
+    t!(Markup[" /"]: "a1."  => Text("a1."));
+}
+
+#[test]
+fn test_tokenize_code_symbols() {
+    // Test all symbols.
+    t!(Code: ","        => Comma);
+    t!(Code: ";"        => Semicolon);
+    t!(Code: ":"        => Colon);
+    t!(Code: "+"        => Plus);
+    t!(Code: "-"        => Minus);
+    t!(Code[" a1"]: "*" => Star);
+    t!(Code[" a1"]: "/" => Slash);
+    t!(Code[" a/"]: "." => Dot);
+    t!(Code: "="        => Eq);
+    t!(Code: "=="       => EqEq);
+    t!(Code: "!="       => ExclEq);
+    t!(Code[" /"]: "<"  => Lt);
+    t!(Code: "<="       => LtEq);
+    t!(Code: ">"        => Gt);
+    t!(Code: ">="       => GtEq);
+    t!(Code: "+="       => PlusEq);
+    t!(Code: "-="       => HyphEq);
+    t!(Code: "*="       => StarEq);
+    t!(Code: "/="       => SlashEq);
+    t!(Code: ".."       => Dots);
+    t!(Code: "=>"       => Arrow);
+
+    // Test combinations.
+    t!(Code: "<=>"        => LtEq, Gt);
+    t!(Code[" a/"]: "..." => Dots, Dot);
+
+    // Test hyphen as symbol vs part of identifier.
+    t!(Code[" /"]: "-1"   => Minus, Int(1));
+    t!(Code[" /"]: "-a"   => Minus, Ident("a"));
+    t!(Code[" /"]: "--1"  => Minus, Minus, Int(1));
+    t!(Code[" /"]: "--_a" => Minus, Minus, Ident("_a"));
+    t!(Code[" /"]: "a-b"  => Ident("a-b"));
+
+    // Test invalid.
+    t!(Code: r"\" => Error(Full, "not valid here"));
+}
+
+#[test]
+fn test_tokenize_keywords() {
+    // A list of a few (not all) keywords.
+    let list = [
+        ("not", Not),
+        ("let", Let),
+        ("if", If),
+        ("else", Else),
+        ("for", For),
+        ("in", In),
+        ("import", Import),
+    ];
+
+    for (s, t) in list.clone() {
+        t!(Markup[" "]: format!("#{}", s) => t);
+        t!(Markup[" "]: format!("#{0}#{0}", s) => t, t);
+        t!(Markup[" /"]: format!("# {}", s) => Text(&format!("# {s}")));
+    }
+
+    for (s, t) in list {
+        t!(Code[" "]: s => t);
+        t!(Markup[" /"]: s => Text(s));
+    }
+
+    // Test simple identifier.
+    t!(Markup[" "]: "#letter" => Ident("letter"));
+    t!(Code[" /"]: "falser"   => Ident("falser"));
+    t!(Code[" /"]: "None"     => Ident("None"));
+    t!(Code[" /"]: "True"     => Ident("True"));
+}
+
+#[test]
+fn test_tokenize_raw_blocks() {
+    // Test basic raw block.
+    t!(Markup: "``"     => Raw("", None, false));
+    t!(Markup: "`raw`"  => Raw("raw", None, false));
+    t!(Markup[""]: "`]" => Error(End, "expected 1 backtick"));
+
+    // Test special symbols in raw block.
+    t!(Markup: "`[brackets]`" => Raw("[brackets]", None, false));
+    t!(Markup[""]: r"`\`` "   => Raw(r"\", None, false), Error(End, "expected 1 backtick"));
+
+    // Test separated closing backticks.
+    t!(Markup: "```not `y`e`t```" => Raw("`y`e`t", Some("not"), false));
+
+    // Test more backticks.
+    t!(Markup: "``nope``"             => Raw("", None, false), Text("nope"), Raw("", None, false));
+    t!(Markup: "````🚀````"           => Raw("", None, false));
+    t!(Markup[""]: "`````👩‍🚀````noend" => Error(End, "expected 5 backticks"));
+    t!(Markup[""]: "````raw``````"    => Raw("", Some("raw"), false), Raw("", None, false));
+}
+
+#[test]
+fn test_tokenize_idents() {
+    // Test valid identifiers.
+    t!(Code[" /"]: "x"           => Ident("x"));
+    t!(Code[" /"]: "value"       => Ident("value"));
+    t!(Code[" /"]: "__main__"    => Ident("__main__"));
+    t!(Code[" /"]: "_snake_case" => Ident("_snake_case"));
+
+    // Test non-ascii.
+    t!(Code[" /"]: "α"    => Ident("α"));
+    t!(Code[" /"]: "ម្តាយ" => Ident("ម្តាយ"));
+
+    // Test hyphen parsed as identifier.
+    t!(Code[" /"]: "kebab-case" => Ident("kebab-case"));
+    t!(Code[" /"]: "one-10"     => Ident("one-10"));
+}
+
+#[test]
+fn test_tokenize_numeric() {
+    let ints = [("7", 7), ("012", 12)];
+    let floats = [
+        (".3", 0.3),
+        ("0.3", 0.3),
+        ("3.", 3.0),
+        ("3.0", 3.0),
+        ("14.3", 14.3),
+        ("10e2", 1000.0),
+        ("10e+0", 10.0),
+        ("10e+1", 100.0),
+        ("10e-2", 0.1),
+        ("10.e1", 100.0),
+        ("10.e-1", 1.0),
+        (".1e1", 1.0),
+        ("10E2", 1000.0),
+    ];
+
+    // Test integers.
+    for &(s, v) in &ints {
+        t!(Code[" /"]: s => Int(v));
+    }
+
+    // Test floats.
+    for &(s, v) in &floats {
+        t!(Code[" /"]: s => Float(v));
+    }
+
+    // Test attached numbers.
+    t!(Code[" /"]: ".2.3"  => Float(0.2), Float(0.3));
+    t!(Code[" /"]: "1.2.3"  => Float(1.2), Float(0.3));
+    t!(Code[" /"]: "1e-2+3" => Float(0.01), Plus, Int(3));
+
+    // Test float from too large integer.
+    let large = i64::MAX as f64 + 1.0;
+    t!(Code[" /"]: large.to_string() => Float(large));
+
+    // Combined integers and floats.
+    let nums = ints.iter().map(|&(k, v)| (k, v as f64)).chain(floats);
+
+    let suffixes: &[(&str, fn(f64) -> SyntaxKind)] = &[
+        ("mm", |x| Numeric(x, Unit::Length(AbsUnit::Mm))),
+        ("pt", |x| Numeric(x, Unit::Length(AbsUnit::Pt))),
+        ("cm", |x| Numeric(x, Unit::Length(AbsUnit::Cm))),
+        ("in", |x| Numeric(x, Unit::Length(AbsUnit::In))),
+        ("rad", |x| Numeric(x, Unit::Angle(AngleUnit::Rad))),
+        ("deg", |x| Numeric(x, Unit::Angle(AngleUnit::Deg))),
+        ("em", |x| Numeric(x, Unit::Em)),
+        ("fr", |x| Numeric(x, Unit::Fr)),
+        ("%", |x| Numeric(x, Unit::Percent)),
+    ];
+
+    // Numeric types.
+    for &(suffix, build) in suffixes {
+        for (s, v) in nums.clone() {
+            t!(Code[" /"]: format!("{}{}", s, suffix) => build(v));
+        }
+    }
+
+    // Multiple dots close the number.
+    t!(Code[" /"]: "1..2"   => Int(1), Dots, Int(2));
+    t!(Code[" /"]: "1..2.3" => Int(1), Dots, Float(2.3));
+    t!(Code[" /"]: "1.2..3" => Float(1.2), Dots, Int(3));
+
+    // Test invalid.
+    t!(Code[" /"]: "1foo" => Error(Full, "invalid number suffix"));
+}
+
+#[test]
+fn test_tokenize_strings() {
+    // Test basic strings.
+    t!(Code: "\"hi\""        => Str("hi"));
+    t!(Code: "\"hi\nthere\"" => Str("hi\nthere"));
+    t!(Code: "\"🌎\""        => Str("🌎"));
+
+    // Test unterminated.
+    t!(Code[""]: "\"hi" => Error(End, "expected quote"));
+
+    // Test escaped quote.
+    t!(Code: r#""a\"bc""# => Str("a\"bc"));
+    t!(Code[""]: r#""\""# => Error(End, "expected quote"));
+}
+
+#[test]
+fn test_tokenize_line_comments() {
+    // Test line comment with no trailing newline.
+    t!(Both[""]: "//" => LineComment);
+
+    // Test line comment ends at newline.
+    t!(Both["a1/"]: "//bc\n"   => LineComment, Space(1));
+    t!(Both["a1/"]: "// bc \n" => LineComment, Space(1));
+    t!(Both["a1/"]: "//bc\r\n" => LineComment, Space(1));
+
+    // Test nested line comments.
+    t!(Both["a1/"]: "//a//b\n" => LineComment, Space(1));
+}
+
+#[test]
+fn test_tokenize_block_comments() {
+    // Test basic block comments.
+    t!(Both[""]: "/*" => BlockComment);
+    t!(Both: "/**/"   => BlockComment);
+    t!(Both: "/*🏞*/" => BlockComment);
+    t!(Both: "/*\n*/" => BlockComment);
+
+    // Test depth 1 and 2 nested block comments.
+    t!(Both: "/* /* */ */"  => BlockComment);
+    t!(Both: "/*/*/**/*/*/" => BlockComment);
+
+    // Test two nested, one unclosed block comments.
+    t!(Both[""]: "/*/*/**/*/" => BlockComment);
+
+    // Test all combinations of up to two following slashes and stars.
+    t!(Both[""]: "/*"   => BlockComment);
+    t!(Both[""]: "/*/"  => BlockComment);
+    t!(Both[""]: "/**"  => BlockComment);
+    t!(Both[""]: "/*//" => BlockComment);
+    t!(Both[""]: "/*/*" => BlockComment);
+    t!(Both[""]: "/**/" => BlockComment);
+    t!(Both[""]: "/***" => BlockComment);
+
+    // Test unexpected terminator.
+    t!(Both: "/*Hi*/*/" => BlockComment,
+        Error(Full, "unexpected end of block comment"));
+}
diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs
index 130ad668..57188096 100644
--- a/src/syntax/tokens.rs
+++ b/src/syntax/tokens.rs
@@ -35,14 +35,12 @@ pub enum TokenMode {
 
 impl<'s> Tokens<'s> {
     /// Create a new token iterator with the given mode.
-    #[inline]
     pub fn new(text: &'s str, mode: TokenMode) -> Self {
         Self::with_prefix("", text, mode)
     }
 
     /// Create a new token iterator with the given mode and a prefix to offset
     /// column calculations.
-    #[inline]
     pub fn with_prefix(prefix: &str, text: &'s str, mode: TokenMode) -> Self {
         Self {
             s: Scanner::new(text),
@@ -53,54 +51,46 @@ impl<'s> Tokens<'s> {
     }
 
     /// Get the current token mode.
-    #[inline]
     pub fn mode(&self) -> TokenMode {
         self.mode
     }
 
     /// Change the token mode.
-    #[inline]
     pub fn set_mode(&mut self, mode: TokenMode) {
         self.mode = mode;
     }
 
     /// The index in the string at which the last token ends and next token
     /// will start.
-    #[inline]
     pub fn cursor(&self) -> usize {
         self.s.cursor()
     }
 
     /// Jump to the given index in the string.
-    #[inline]
     pub fn jump(&mut self, index: usize) {
         self.s.jump(index);
     }
 
     /// The underlying scanner.
-    #[inline]
     pub fn scanner(&self) -> Scanner<'s> {
         self.s
     }
 
     /// Whether the last token was terminated.
-    #[inline]
     pub fn terminated(&self) -> bool {
         self.terminated
     }
 
     /// The column index of a given index in the source string.
-    #[inline]
     pub fn column(&self, index: usize) -> usize {
         column(self.s.string(), index, self.column_offset)
     }
 }
 
-impl<'s> Iterator for Tokens<'s> {
+impl Iterator for Tokens<'_> {
     type Item = SyntaxKind;
 
     /// Parse the next token in the source code.
-    #[inline]
     fn next(&mut self) -> Option<Self::Item> {
         let start = self.s.cursor();
         let c = self.s.eat()?;
@@ -124,7 +114,8 @@ impl<'s> Iterator for Tokens<'s> {
     }
 }
 
-impl<'s> Tokens<'s> {
+/// Shared.
+impl Tokens<'_> {
     fn line_comment(&mut self) -> SyntaxKind {
         self.s.eat_until(is_newline);
         if self.s.peek().is_none() {
@@ -189,8 +180,9 @@ impl<'s> Tokens<'s> {
 
         SyntaxKind::Space { newlines }
     }
+}
 
-    #[inline]
+impl Tokens<'_> {
     fn markup(&mut self, start: usize, c: char) -> SyntaxKind {
         match c {
             // Blocks.
@@ -231,7 +223,6 @@ impl<'s> Tokens<'s> {
         }
     }
 
-    #[inline]
     fn text(&mut self, start: usize) -> SyntaxKind {
         macro_rules! table {
             ($(|$c:literal)*) => {{
@@ -303,7 +294,11 @@ impl<'s> Tokens<'s> {
     }
 
     fn hash(&mut self, start: usize) -> SyntaxKind {
-        if self.s.at(is_id_start) {
+        if self.s.eat_if('{') {
+            SyntaxKind::LeftBrace
+        } else if self.s.eat_if('[') {
+            SyntaxKind::LeftBracket
+        } else if self.s.at(is_id_start) {
             let read = self.s.eat_while(is_id_continue);
             match keyword(read) {
                 Some(keyword) => keyword,
@@ -342,8 +337,10 @@ impl<'s> Tokens<'s> {
         if start < end {
             self.s.expect(':');
             SyntaxKind::Symbol(self.s.get(start..end).into())
-        } else {
+        } else if self.mode == TokenMode::Markup {
             SyntaxKind::Colon
+        } else {
+            SyntaxKind::Atom(":".into())
         }
     }
 
@@ -426,26 +423,25 @@ impl<'s> Tokens<'s> {
         self.text(start)
     }
 
-    fn label(&mut self) -> SyntaxKind {
-        let label = self.s.eat_while(is_id_continue);
-        if self.s.eat_if('>') {
-            if !label.is_empty() {
-                SyntaxKind::Label(label.into())
-            } else {
-                SyntaxKind::Error(ErrorPos::Full, "label cannot be empty".into())
-            }
-        } else {
-            self.terminated = false;
-            SyntaxKind::Error(ErrorPos::End, "expected closing angle bracket".into())
-        }
-    }
-
     fn reference(&mut self) -> SyntaxKind {
         SyntaxKind::Ref(self.s.eat_while(is_id_continue).into())
     }
 
+    fn in_word(&self) -> bool {
+        let alphanumeric = |c: Option<char>| c.map_or(false, |c| c.is_alphanumeric());
+        let prev = self.s.scout(-2);
+        let next = self.s.peek();
+        alphanumeric(prev) && alphanumeric(next)
+    }
+}
+
+/// Math.
+impl Tokens<'_> {
     fn math(&mut self, start: usize, c: char) -> SyntaxKind {
         match c {
+            // Multi-char things.
+            '#' => self.hash(start),
+
             // Escape sequences.
             '\\' => self.backslash(),
 
@@ -456,18 +452,32 @@ impl<'s> Tokens<'s> {
             '&' => SyntaxKind::Amp,
             '$' => SyntaxKind::Dollar,
 
-            // Brackets.
-            '{' => SyntaxKind::LeftBrace,
-            '}' => SyntaxKind::RightBrace,
-            '[' => SyntaxKind::LeftBracket,
-            ']' => SyntaxKind::RightBracket,
-            '(' => SyntaxKind::LeftParen,
-            ')' => SyntaxKind::RightParen,
+            // Symbol notation.
+            ':' => self.colon(),
 
-            // Identifiers.
+            // Strings.
+            '"' => self.string(),
+
+            // Identifiers and symbol notation.
             c if is_math_id_start(c) && self.s.at(is_math_id_continue) => {
                 self.s.eat_while(is_math_id_continue);
-                SyntaxKind::Ident(self.s.from(start).into())
+
+                let mut symbol = false;
+                while self.s.eat_if(':')
+                    && !self.s.eat_while(char::is_alphanumeric).is_empty()
+                {
+                    symbol = true;
+                }
+
+                if symbol {
+                    SyntaxKind::Symbol(self.s.from(start).into())
+                } else {
+                    if self.s.scout(-1) == Some(':') {
+                        self.s.uneat();
+                    }
+
+                    SyntaxKind::Ident(self.s.from(start).into())
+                }
             }
 
             // Numbers.
@@ -480,7 +490,10 @@ impl<'s> Tokens<'s> {
             c => SyntaxKind::Atom(c.into()),
         }
     }
+}
 
+/// Code.
+impl Tokens<'_> {
     fn code(&mut self, start: usize, c: char) -> SyntaxKind {
         match c {
             // Blocks.
@@ -493,6 +506,9 @@ impl<'s> Tokens<'s> {
             '(' => SyntaxKind::LeftParen,
             ')' => SyntaxKind::RightParen,
 
+            // Math.
+            '$' => SyntaxKind::Dollar,
+
             // Labels.
             '<' if self.s.at(is_id_continue) => self.label(),
 
@@ -619,14 +635,22 @@ impl<'s> Tokens<'s> {
         }
     }
 
-    fn in_word(&self) -> bool {
-        let alphanumeric = |c: Option<char>| c.map_or(false, |c| c.is_alphanumeric());
-        let prev = self.s.scout(-2);
-        let next = self.s.peek();
-        alphanumeric(prev) && alphanumeric(next)
+    fn label(&mut self) -> SyntaxKind {
+        let label = self.s.eat_while(is_id_continue);
+        if self.s.eat_if('>') {
+            if !label.is_empty() {
+                SyntaxKind::Label(label.into())
+            } else {
+                SyntaxKind::Error(ErrorPos::Full, "label cannot be empty".into())
+            }
+        } else {
+            self.terminated = false;
+            SyntaxKind::Error(ErrorPos::End, "expected closing angle bracket".into())
+        }
     }
 }
 
+/// Try to parse an identifier into a keyword.
 fn keyword(ident: &str) -> Option<SyntaxKind> {
     Some(match ident {
         "not" => SyntaxKind::Not,
@@ -652,7 +676,6 @@ fn keyword(ident: &str) -> Option<SyntaxKind> {
 
 /// The column index of a given index in the source string, given a column
 /// offset for the first line.
-#[inline]
 fn column(string: &str, index: usize, offset: usize) -> usize {
     let mut apply_offset = false;
     let res = string[..index]
@@ -729,471 +752,3 @@ fn is_math_id_start(c: char) -> bool {
 fn is_math_id_continue(c: char) -> bool {
     c.is_xid_continue() && c != '_'
 }
-
-#[cfg(test)]
-#[allow(non_snake_case)]
-mod tests {
-    use super::super::tests::check;
-    use super::*;
-
-    use ErrorPos::*;
-    use Option::None;
-    use SyntaxKind::*;
-    use TokenMode::{Code, Markup};
-
-    fn Space(newlines: usize) -> SyntaxKind {
-        SyntaxKind::Space { newlines }
-    }
-
-    fn Raw(text: &str, lang: Option<&str>, block: bool) -> SyntaxKind {
-        SyntaxKind::Raw(Arc::new(RawFields {
-            text: text.into(),
-            lang: lang.map(Into::into),
-            block,
-        }))
-    }
-
-    fn Str(string: &str) -> SyntaxKind {
-        SyntaxKind::Str(string.into())
-    }
-
-    fn Text(string: &str) -> SyntaxKind {
-        SyntaxKind::Text(string.into())
-    }
-
-    fn Ident(ident: &str) -> SyntaxKind {
-        SyntaxKind::Ident(ident.into())
-    }
-
-    fn Error(pos: ErrorPos, message: &str) -> SyntaxKind {
-        SyntaxKind::Error(pos, message.into())
-    }
-
-    /// Building blocks for suffix testing.
-    ///
-    /// We extend each test case with a collection of different suffixes to make
-    /// sure tokens end at the correct position. These suffixes are split into
-    /// blocks, which can be disabled/enabled per test case. For example, when
-    /// testing identifiers we disable letter suffixes because these would
-    /// mingle with the identifiers.
-    ///
-    /// Suffix blocks:
-    /// - ' ': spacing
-    /// - 'a': letters
-    /// - '1': numbers
-    /// - '/': symbols
-    const BLOCKS: &str = " a1/";
-
-    // Suffixes described by four-tuples of:
-    //
-    // - block the suffix is part of
-    // - mode in which the suffix is applicable
-    // - the suffix string
-    // - the resulting suffix NodeKind
-    fn suffixes(
-    ) -> impl Iterator<Item = (char, Option<TokenMode>, &'static str, SyntaxKind)> {
-        [
-            // Whitespace suffixes.
-            (' ', None, " ", Space(0)),
-            (' ', None, "\n", Space(1)),
-            (' ', None, "\r", Space(1)),
-            (' ', None, "\r\n", Space(1)),
-            // Letter suffixes.
-            ('a', Some(Markup), "hello", Text("hello")),
-            ('a', Some(Markup), "💚", Text("💚")),
-            ('a', Some(Code), "val", Ident("val")),
-            ('a', Some(Code), "α", Ident("α")),
-            ('a', Some(Code), "_", Ident("_")),
-            // Number suffixes.
-            ('1', Some(Code), "2", Int(2)),
-            ('1', Some(Code), ".2", Float(0.2)),
-            // Symbol suffixes.
-            ('/', None, "[", LeftBracket),
-            ('/', None, "//", LineComment),
-            ('/', None, "/**/", BlockComment),
-            ('/', Some(Markup), "*", Star),
-            ('/', Some(Markup), r"\\", Escape('\\')),
-            ('/', Some(Markup), "#let", Let),
-            ('/', Some(Code), "(", LeftParen),
-            ('/', Some(Code), ":", Colon),
-            ('/', Some(Code), "+=", PlusEq),
-        ]
-        .into_iter()
-    }
-
-    macro_rules! t {
-        (Both $($tts:tt)*) => {
-            t!(Markup $($tts)*);
-            t!(Code $($tts)*);
-        };
-        ($mode:ident $([$blocks:literal])?: $text:expr => $($token:expr),*) => {{
-            // Test without suffix.
-            t!(@$mode: $text => $($token),*);
-
-            // Test with each applicable suffix.
-            for (block, mode, suffix, ref token) in suffixes() {
-                let text = $text;
-                #[allow(unused_variables)]
-                let blocks = BLOCKS;
-                $(let blocks = $blocks;)?
-                assert!(!blocks.contains(|c| !BLOCKS.contains(c)));
-                if (mode.is_none() || mode == Some($mode)) && blocks.contains(block) {
-                    t!(@$mode: format!("{}{}", text, suffix) => $($token,)* token);
-                }
-            }
-        }};
-        (@$mode:ident: $text:expr => $($token:expr),*) => {{
-            let text = $text;
-            let found = Tokens::new(&text, $mode).collect::<Vec<_>>();
-            let expected = vec![$($token.clone()),*];
-            check(&text, found, expected);
-        }};
-    }
-
-    #[test]
-    fn test_tokenize_brackets() {
-        // Test in markup.
-        t!(Markup: "{"       => LeftBrace);
-        t!(Markup: "}"       => RightBrace);
-        t!(Markup: "["       => LeftBracket);
-        t!(Markup: "]"       => RightBracket);
-        t!(Markup[" /"]: "(" => Text("("));
-        t!(Markup[" /"]: ")" => Text(")"));
-
-        // Test in code.
-        t!(Code: "{" => LeftBrace);
-        t!(Code: "}" => RightBrace);
-        t!(Code: "[" => LeftBracket);
-        t!(Code: "]" => RightBracket);
-        t!(Code: "(" => LeftParen);
-        t!(Code: ")" => RightParen);
-    }
-
-    #[test]
-    fn test_tokenize_whitespace() {
-        // Test basic whitespace.
-        t!(Both["a1/"]: ""         => );
-        t!(Both["a1/"]: " "        => Space(0));
-        t!(Both["a1/"]: "    "     => Space(0));
-        t!(Both["a1/"]: "\t"       => Space(0));
-        t!(Both["a1/"]: "  \t"     => Space(0));
-        t!(Both["a1/"]: "\u{202F}" => Space(0));
-
-        // Test newline counting.
-        t!(Both["a1/"]: "\n"           => Space(1));
-        t!(Both["a1/"]: "\n "          => Space(1));
-        t!(Both["a1/"]: "  \n"         => Space(1));
-        t!(Both["a1/"]: "  \n   "      => Space(1));
-        t!(Both["a1/"]: "\r\n"         => Space(1));
-        t!(Both["a1/"]: "\r\n\r"       => Space(2));
-        t!(Both["a1/"]: "  \n\t \n  "  => Space(2));
-        t!(Both["a1/"]: "\n\r"         => Space(2));
-        t!(Both["a1/"]: " \r\r\n \x0D" => Space(3));
-    }
-
-    #[test]
-    fn test_tokenize_text() {
-        // Test basic text.
-        t!(Markup[" /"]: "hello"      => Text("hello"));
-        t!(Markup[" /"]: "reha-world" => Text("reha-world"));
-
-        // Test code symbols in text.
-        t!(Markup[" /"]: "a():\"b" => Text("a()"), Colon, SmartQuote { double: true }, Text("b"));
-        t!(Markup[" /"]: ";,|/+"  => Text(";,|/+"));
-        t!(Markup[" /"]: "=-a"     => Eq, Minus, Text("a"));
-        t!(Markup[" "]: "#123"     => Text("#123"));
-
-        // Test text ends.
-        t!(Markup[""]: "hello " => Text("hello"), Space(0));
-        t!(Markup[""]: "hello~" => Text("hello"), Shorthand('\u{00A0}'));
-    }
-
-    #[test]
-    fn test_tokenize_escape_sequences() {
-        // Test escapable symbols.
-        t!(Markup: r"\\" => Escape('\\'));
-        t!(Markup: r"\/" => Escape('/'));
-        t!(Markup: r"\[" => Escape('['));
-        t!(Markup: r"\]" => Escape(']'));
-        t!(Markup: r"\{" => Escape('{'));
-        t!(Markup: r"\}" => Escape('}'));
-        t!(Markup: r"\*" => Escape('*'));
-        t!(Markup: r"\_" => Escape('_'));
-        t!(Markup: r"\=" => Escape('='));
-        t!(Markup: r"\~" => Escape('~'));
-        t!(Markup: r"\'" => Escape('\''));
-        t!(Markup: r#"\""# => Escape('"'));
-        t!(Markup: r"\`" => Escape('`'));
-        t!(Markup: r"\$" => Escape('$'));
-        t!(Markup: r"\#" => Escape('#'));
-        t!(Markup: r"\a"   => Escape('a'));
-        t!(Markup: r"\u"   => Escape('u'));
-        t!(Markup: r"\1"   => Escape('1'));
-
-        // Test basic unicode escapes.
-        t!(Markup: r"\u{}"     => Error(Full, "invalid unicode escape sequence"));
-        t!(Markup: r"\u{2603}" => Escape('☃'));
-        t!(Markup: r"\u{P}"    => Error(Full, "invalid unicode escape sequence"));
-
-        // Test unclosed unicode escapes.
-        t!(Markup[" /"]: r"\u{"     => Error(End, "expected closing brace"));
-        t!(Markup[" /"]: r"\u{1"    => Error(End, "expected closing brace"));
-        t!(Markup[" /"]: r"\u{26A4" => Error(End, "expected closing brace"));
-        t!(Markup[" /"]: r"\u{1Q3P" => Error(End, "expected closing brace"));
-        t!(Markup: r"\u{1🏕}"       => Error(End, "expected closing brace"), Text("🏕"), RightBrace);
-    }
-
-    #[test]
-    fn test_tokenize_markup_symbols() {
-        // Test markup tokens.
-        t!(Markup[" a1"]: "*"   => Star);
-        t!(Markup: "_"          => Underscore);
-        t!(Markup[""]: "==="    => Eq, Eq, Eq);
-        t!(Markup["a1/"]: "= "  => Eq, Space(0));
-        t!(Markup[" "]: r"\"    => Linebreak);
-        t!(Markup: "~"          => Shorthand('\u{00A0}'));
-        t!(Markup["a1/"]: "-?"  => Shorthand('\u{00AD}'));
-        t!(Markup["a "]: r"a--" => Text("a"), Shorthand('\u{2013}'));
-        t!(Markup["a1/"]: "- "  => Minus, Space(0));
-        t!(Markup[" "]: "+"     => Plus);
-        t!(Markup[" "]: "1."    => EnumNumbering(NonZeroUsize::new(1).unwrap()));
-        t!(Markup[" "]: "1.a"   => EnumNumbering(NonZeroUsize::new(1).unwrap()), Text("a"));
-        t!(Markup[" /"]: "a1."  => Text("a1."));
-    }
-
-    #[test]
-    fn test_tokenize_code_symbols() {
-        // Test all symbols.
-        t!(Code: ","        => Comma);
-        t!(Code: ";"        => Semicolon);
-        t!(Code: ":"        => Colon);
-        t!(Code: "+"        => Plus);
-        t!(Code: "-"        => Minus);
-        t!(Code[" a1"]: "*" => Star);
-        t!(Code[" a1"]: "/" => Slash);
-        t!(Code[" a/"]: "." => Dot);
-        t!(Code: "="        => Eq);
-        t!(Code: "=="       => EqEq);
-        t!(Code: "!="       => ExclEq);
-        t!(Code[" /"]: "<"  => Lt);
-        t!(Code: "<="       => LtEq);
-        t!(Code: ">"        => Gt);
-        t!(Code: ">="       => GtEq);
-        t!(Code: "+="       => PlusEq);
-        t!(Code: "-="       => HyphEq);
-        t!(Code: "*="       => StarEq);
-        t!(Code: "/="       => SlashEq);
-        t!(Code: ".."       => Dots);
-        t!(Code: "=>"       => Arrow);
-
-        // Test combinations.
-        t!(Code: "<=>"        => LtEq, Gt);
-        t!(Code[" a/"]: "..." => Dots, Dot);
-
-        // Test hyphen as symbol vs part of identifier.
-        t!(Code[" /"]: "-1"   => Minus, Int(1));
-        t!(Code[" /"]: "-a"   => Minus, Ident("a"));
-        t!(Code[" /"]: "--1"  => Minus, Minus, Int(1));
-        t!(Code[" /"]: "--_a" => Minus, Minus, Ident("_a"));
-        t!(Code[" /"]: "a-b"  => Ident("a-b"));
-
-        // Test invalid.
-        t!(Code: r"\" => Error(Full, "not valid here"));
-    }
-
-    #[test]
-    fn test_tokenize_keywords() {
-        // A list of a few (not all) keywords.
-        let list = [
-            ("not", Not),
-            ("let", Let),
-            ("if", If),
-            ("else", Else),
-            ("for", For),
-            ("in", In),
-            ("import", Import),
-        ];
-
-        for (s, t) in list.clone() {
-            t!(Markup[" "]: format!("#{}", s) => t);
-            t!(Markup[" "]: format!("#{0}#{0}", s) => t, t);
-            t!(Markup[" /"]: format!("# {}", s) => Text(&format!("# {s}")));
-        }
-
-        for (s, t) in list {
-            t!(Code[" "]: s => t);
-            t!(Markup[" /"]: s => Text(s));
-        }
-
-        // Test simple identifier.
-        t!(Markup[" "]: "#letter" => Ident("letter"));
-        t!(Code[" /"]: "falser"   => Ident("falser"));
-        t!(Code[" /"]: "None"     => Ident("None"));
-        t!(Code[" /"]: "True"     => Ident("True"));
-    }
-
-    #[test]
-    fn test_tokenize_raw_blocks() {
-        // Test basic raw block.
-        t!(Markup: "``"     => Raw("", None, false));
-        t!(Markup: "`raw`"  => Raw("raw", None, false));
-        t!(Markup[""]: "`]" => Error(End, "expected 1 backtick"));
-
-        // Test special symbols in raw block.
-        t!(Markup: "`[brackets]`" => Raw("[brackets]", None, false));
-        t!(Markup[""]: r"`\`` "   => Raw(r"\", None, false), Error(End, "expected 1 backtick"));
-
-        // Test separated closing backticks.
-        t!(Markup: "```not `y`e`t```" => Raw("`y`e`t", Some("not"), false));
-
-        // Test more backticks.
-        t!(Markup: "``nope``"             => Raw("", None, false), Text("nope"), Raw("", None, false));
-        t!(Markup: "````🚀````"           => Raw("", None, false));
-        t!(Markup[""]: "`````👩‍🚀````noend" => Error(End, "expected 5 backticks"));
-        t!(Markup[""]: "````raw``````"    => Raw("", Some("raw"), false), Raw("", None, false));
-    }
-
-    #[test]
-    fn test_tokenize_idents() {
-        // Test valid identifiers.
-        t!(Code[" /"]: "x"           => Ident("x"));
-        t!(Code[" /"]: "value"       => Ident("value"));
-        t!(Code[" /"]: "__main__"    => Ident("__main__"));
-        t!(Code[" /"]: "_snake_case" => Ident("_snake_case"));
-
-        // Test non-ascii.
-        t!(Code[" /"]: "α"    => Ident("α"));
-        t!(Code[" /"]: "ម្តាយ" => Ident("ម្តាយ"));
-
-        // Test hyphen parsed as identifier.
-        t!(Code[" /"]: "kebab-case" => Ident("kebab-case"));
-        t!(Code[" /"]: "one-10"     => Ident("one-10"));
-    }
-
-    #[test]
-    fn test_tokenize_numeric() {
-        let ints = [("7", 7), ("012", 12)];
-        let floats = [
-            (".3", 0.3),
-            ("0.3", 0.3),
-            ("3.", 3.0),
-            ("3.0", 3.0),
-            ("14.3", 14.3),
-            ("10e2", 1000.0),
-            ("10e+0", 10.0),
-            ("10e+1", 100.0),
-            ("10e-2", 0.1),
-            ("10.e1", 100.0),
-            ("10.e-1", 1.0),
-            (".1e1", 1.0),
-            ("10E2", 1000.0),
-        ];
-
-        // Test integers.
-        for &(s, v) in &ints {
-            t!(Code[" /"]: s => Int(v));
-        }
-
-        // Test floats.
-        for &(s, v) in &floats {
-            t!(Code[" /"]: s => Float(v));
-        }
-
-        // Test attached numbers.
-        t!(Code[" /"]: ".2.3"  => Float(0.2), Float(0.3));
-        t!(Code[" /"]: "1.2.3"  => Float(1.2), Float(0.3));
-        t!(Code[" /"]: "1e-2+3" => Float(0.01), Plus, Int(3));
-
-        // Test float from too large integer.
-        let large = i64::MAX as f64 + 1.0;
-        t!(Code[" /"]: large.to_string() => Float(large));
-
-        // Combined integers and floats.
-        let nums = ints.iter().map(|&(k, v)| (k, v as f64)).chain(floats);
-
-        let suffixes: &[(&str, fn(f64) -> SyntaxKind)] = &[
-            ("mm", |x| Numeric(x, Unit::Length(AbsUnit::Mm))),
-            ("pt", |x| Numeric(x, Unit::Length(AbsUnit::Pt))),
-            ("cm", |x| Numeric(x, Unit::Length(AbsUnit::Cm))),
-            ("in", |x| Numeric(x, Unit::Length(AbsUnit::In))),
-            ("rad", |x| Numeric(x, Unit::Angle(AngleUnit::Rad))),
-            ("deg", |x| Numeric(x, Unit::Angle(AngleUnit::Deg))),
-            ("em", |x| Numeric(x, Unit::Em)),
-            ("fr", |x| Numeric(x, Unit::Fr)),
-            ("%", |x| Numeric(x, Unit::Percent)),
-        ];
-
-        // Numeric types.
-        for &(suffix, build) in suffixes {
-            for (s, v) in nums.clone() {
-                t!(Code[" /"]: format!("{}{}", s, suffix) => build(v));
-            }
-        }
-
-        // Multiple dots close the number.
-        t!(Code[" /"]: "1..2"   => Int(1), Dots, Int(2));
-        t!(Code[" /"]: "1..2.3" => Int(1), Dots, Float(2.3));
-        t!(Code[" /"]: "1.2..3" => Float(1.2), Dots, Int(3));
-
-        // Test invalid.
-        t!(Code[" /"]: "1foo" => Error(Full, "invalid number suffix"));
-    }
-
-    #[test]
-    fn test_tokenize_strings() {
-        // Test basic strings.
-        t!(Code: "\"hi\""        => Str("hi"));
-        t!(Code: "\"hi\nthere\"" => Str("hi\nthere"));
-        t!(Code: "\"🌎\""        => Str("🌎"));
-
-        // Test unterminated.
-        t!(Code[""]: "\"hi" => Error(End, "expected quote"));
-
-        // Test escaped quote.
-        t!(Code: r#""a\"bc""# => Str("a\"bc"));
-        t!(Code[""]: r#""\""# => Error(End, "expected quote"));
-    }
-
-    #[test]
-    fn test_tokenize_line_comments() {
-        // Test line comment with no trailing newline.
-        t!(Both[""]: "//" => LineComment);
-
-        // Test line comment ends at newline.
-        t!(Both["a1/"]: "//bc\n"   => LineComment, Space(1));
-        t!(Both["a1/"]: "// bc \n" => LineComment, Space(1));
-        t!(Both["a1/"]: "//bc\r\n" => LineComment, Space(1));
-
-        // Test nested line comments.
-        t!(Both["a1/"]: "//a//b\n" => LineComment, Space(1));
-    }
-
-    #[test]
-    fn test_tokenize_block_comments() {
-        // Test basic block comments.
-        t!(Both[""]: "/*" => BlockComment);
-        t!(Both: "/**/"   => BlockComment);
-        t!(Both: "/*🏞*/" => BlockComment);
-        t!(Both: "/*\n*/" => BlockComment);
-
-        // Test depth 1 and 2 nested block comments.
-        t!(Both: "/* /* */ */"  => BlockComment);
-        t!(Both: "/*/*/**/*/*/" => BlockComment);
-
-        // Test two nested, one unclosed block comments.
-        t!(Both[""]: "/*/*/**/*/" => BlockComment);
-
-        // Test all combinations of up to two following slashes and stars.
-        t!(Both[""]: "/*"   => BlockComment);
-        t!(Both[""]: "/*/"  => BlockComment);
-        t!(Both[""]: "/**"  => BlockComment);
-        t!(Both[""]: "/*//" => BlockComment);
-        t!(Both[""]: "/*/*" => BlockComment);
-        t!(Both[""]: "/**/" => BlockComment);
-        t!(Both[""]: "/***" => BlockComment);
-
-        // Test unexpected terminator.
-        t!(Both: "/*Hi*/*/" => BlockComment,
-           Error(Full, "unexpected end of block comment"));
-    }
-}