More spans in AST

5 files changed, 136 insertions, 139 deletions

diff --git a/src/parse/incremental.rs b/src/parse/incremental.rs
index e0be9b6d..4651a784 100644
--- a/src/parse/incremental.rs
+++ b/src/parse/incremental.rs
@@ -389,16 +389,12 @@ fn is_bounded(kind: &NodeKind) -> bool {
     match kind {
         NodeKind::CodeBlock
         | NodeKind::ContentBlock
-        | NodeKind::Backslash
-        | NodeKind::Tilde
-        | NodeKind::HyphQuest
-        | NodeKind::Hyph2
-        | NodeKind::Hyph3
-        | NodeKind::Dot3
-        | NodeKind::Quote { .. }
+        | NodeKind::Linebreak
+        | NodeKind::SmartQuote { .. }
         | NodeKind::BlockComment
         | NodeKind::Space { .. }
-        | NodeKind::Escape(_) => true,
+        | NodeKind::Escape(_)
+        | NodeKind::Shorthand(_) => true,
         _ => false,
     }
 }
diff --git a/src/parse/mod.rs b/src/parse/mod.rs
index 4f42442f..ac8ec6eb 100644
--- a/src/parse/mod.rs
+++ b/src/parse/mod.rs
@@ -11,9 +11,8 @@ pub use tokens::*;
 
 use std::collections::HashSet;
 
-use crate::diag::ErrorPos;
 use crate::syntax::ast::{Assoc, BinOp, UnOp};
-use crate::syntax::{NodeKind, SyntaxNode};
+use crate::syntax::{ErrorPos, NodeKind, SyntaxNode};
 use crate::util::EcoString;
 
 /// Parse a source file.
@@ -240,14 +239,10 @@ fn markup_node(p: &mut Parser, at_start: &mut bool) {
 
         // Text and markup.
         NodeKind::Text(_)
-        | NodeKind::Backslash
-        | NodeKind::Tilde
-        | NodeKind::HyphQuest
-        | NodeKind::Hyph2
-        | NodeKind::Hyph3
-        | NodeKind::Dot3
-        | NodeKind::Quote { .. }
+        | NodeKind::Linebreak
+        | NodeKind::SmartQuote { .. }
         | NodeKind::Escape(_)
+        | NodeKind::Shorthand(_)
         | NodeKind::Link(_)
         | NodeKind::Raw(_)
         | NodeKind::Label(_)
@@ -475,15 +470,15 @@ fn math_primary(p: &mut Parser) {
     match token {
         // Spaces, atoms and expressions.
         NodeKind::Space { .. }
-        | NodeKind::Backslash
+        | NodeKind::Linebreak
         | NodeKind::Escape(_)
         | NodeKind::Atom(_)
         | NodeKind::Ident(_) => p.eat(),
 
         // Groups.
-        NodeKind::LeftParen => group(p, Group::Paren),
-        NodeKind::LeftBracket => group(p, Group::Bracket),
-        NodeKind::LeftBrace => group(p, Group::Brace),
+        NodeKind::LeftParen => group(p, Group::Paren, '(', ')'),
+        NodeKind::LeftBracket => group(p, Group::Bracket, '[', ']'),
+        NodeKind::LeftBrace => group(p, Group::Brace, '{', '}'),
 
         // Alignment indactor.
         NodeKind::Amp => align(p),
@@ -493,13 +488,17 @@ fn math_primary(p: &mut Parser) {
 }
 
 /// Parse grouped math.
-fn group(p: &mut Parser, group: Group) {
+fn group(p: &mut Parser, group: Group, l: char, r: char) {
     p.perform(NodeKind::Math, |p| {
+        let marker = p.marker();
         p.start_group(group);
+        marker.convert(p, NodeKind::Atom(l.into()));
         while !p.eof() {
             math_node(p);
         }
+        let marker = p.marker();
         p.end_group();
+        marker.convert(p, NodeKind::Atom(r.into()));
     })
 }
 
@@ -532,7 +531,7 @@ fn expr_prec(p: &mut Parser, atomic: bool, min_prec: usize) -> ParseResult {
             p.eat();
             let prec = op.precedence();
             expr_prec(p, atomic, prec)?;
-            marker.end(p, NodeKind::UnaryExpr);
+            marker.end(p, NodeKind::Unary);
         }
         _ => primary(p, atomic)?,
     };
@@ -585,7 +584,7 @@ fn expr_prec(p: &mut Parser, atomic: bool, min_prec: usize) -> ParseResult {
             Assoc::Right => {}
         }
 
-        marker.perform(p, NodeKind::BinaryExpr, |p| expr_prec(p, atomic, prec))?;
+        marker.perform(p, NodeKind::Binary, |p| expr_prec(p, atomic, prec))?;
     }
 
     Ok(())
@@ -605,9 +604,9 @@ fn primary(p: &mut Parser, atomic: bool) -> ParseResult {
 
             // Arrow means this is a closure's lone parameter.
             if !atomic && p.at(NodeKind::Arrow) {
-                marker.end(p, NodeKind::ClosureParams);
+                marker.end(p, NodeKind::Params);
                 p.assert(NodeKind::Arrow);
-                marker.perform(p, NodeKind::ClosureExpr, expr)
+                marker.perform(p, NodeKind::Closure, expr)
             } else {
                 Ok(())
             }
@@ -703,12 +702,12 @@ fn parenthesized(p: &mut Parser, atomic: bool) -> ParseResult {
     if !atomic && p.at(NodeKind::Arrow) {
         params(p, marker);
         p.assert(NodeKind::Arrow);
-        return marker.perform(p, NodeKind::ClosureExpr, expr);
+        return marker.perform(p, NodeKind::Closure, expr);
     }
 
     // Transform into the identified collection.
     match kind {
-        CollectionKind::Group => marker.end(p, NodeKind::GroupExpr),
+        CollectionKind::Group => marker.end(p, NodeKind::Parenthesized),
         CollectionKind::Positional => array(p, marker),
         CollectionKind::Named => dict(p, marker),
     }
@@ -833,7 +832,7 @@ fn array(p: &mut Parser, marker: Marker) {
         NodeKind::Named | NodeKind::Keyed => Err("expected expression"),
         _ => Ok(()),
     });
-    marker.end(p, NodeKind::ArrayExpr);
+    marker.end(p, NodeKind::Array);
 }
 
 /// Convert a collection into a dictionary, producing errors for anything other
@@ -855,7 +854,7 @@ fn dict(p: &mut Parser, marker: Marker) {
         NodeKind::Spread | NodeKind::Comma | NodeKind::Colon => Ok(()),
         _ => Err("expected named or keyed pair"),
     });
-    marker.end(p, NodeKind::DictExpr);
+    marker.end(p, NodeKind::Dict);
 }
 
 /// Convert a collection into a list of parameters, producing errors for
@@ -874,7 +873,7 @@ fn params(p: &mut Parser, marker: Marker) {
         }
         _ => Err("expected identifier, named pair or argument sink"),
     });
-    marker.end(p, NodeKind::ClosureParams);
+    marker.end(p, NodeKind::Params);
 }
 
 /// Parse a code block: `{...}`.
@@ -920,7 +919,7 @@ fn args(p: &mut Parser) -> ParseResult {
         }
     }
 
-    p.perform(NodeKind::CallArgs, |p| {
+    p.perform(NodeKind::Args, |p| {
         if p.at(NodeKind::LeftParen) {
             let marker = p.marker();
             p.start_group(Group::Paren);
@@ -953,7 +952,7 @@ fn args(p: &mut Parser) -> ParseResult {
 
 /// Parse a let expression.
 fn let_expr(p: &mut Parser) -> ParseResult {
-    p.perform(NodeKind::LetExpr, |p| {
+    p.perform(NodeKind::LetBinding, |p| {
         p.assert(NodeKind::Let);
 
         let marker = p.marker();
@@ -978,7 +977,7 @@ fn let_expr(p: &mut Parser) -> ParseResult {
 
         // Rewrite into a closure expression if it's a function definition.
         if has_params {
-            marker.end(p, NodeKind::ClosureExpr);
+            marker.end(p, NodeKind::Closure);
         }
 
         Ok(())
@@ -987,7 +986,7 @@ fn let_expr(p: &mut Parser) -> ParseResult {
 
 /// Parse a set expression.
 fn set_expr(p: &mut Parser) -> ParseResult {
-    p.perform(NodeKind::SetExpr, |p| {
+    p.perform(NodeKind::SetRule, |p| {
         p.assert(NodeKind::Set);
         ident(p)?;
         args(p)
@@ -996,7 +995,7 @@ fn set_expr(p: &mut Parser) -> ParseResult {
 
 /// Parse a show expression.
 fn show_expr(p: &mut Parser) -> ParseResult {
-    p.perform(NodeKind::ShowExpr, |p| {
+    p.perform(NodeKind::ShowRule, |p| {
         p.assert(NodeKind::Show);
         let marker = p.marker();
         expr(p)?;
@@ -1014,7 +1013,7 @@ fn show_expr(p: &mut Parser) -> ParseResult {
 
 /// Parse a wrap expression.
 fn wrap_expr(p: &mut Parser) -> ParseResult {
-    p.perform(NodeKind::WrapExpr, |p| {
+    p.perform(NodeKind::WrapRule, |p| {
         p.assert(NodeKind::Wrap);
         ident(p)?;
         p.expect(NodeKind::In)?;
@@ -1024,7 +1023,7 @@ fn wrap_expr(p: &mut Parser) -> ParseResult {
 
 /// Parse an if-else expresion.
 fn if_expr(p: &mut Parser) -> ParseResult {
-    p.perform(NodeKind::IfExpr, |p| {
+    p.perform(NodeKind::Conditional, |p| {
         p.assert(NodeKind::If);
 
         expr(p)?;
@@ -1044,7 +1043,7 @@ fn if_expr(p: &mut Parser) -> ParseResult {
 
 /// Parse a while expresion.
 fn while_expr(p: &mut Parser) -> ParseResult {
-    p.perform(NodeKind::WhileExpr, |p| {
+    p.perform(NodeKind::WhileLoop, |p| {
         p.assert(NodeKind::While);
         expr(p)?;
         body(p)
@@ -1053,7 +1052,7 @@ fn while_expr(p: &mut Parser) -> ParseResult {
 
 /// Parse a for-in expression.
 fn for_expr(p: &mut Parser) -> ParseResult {
-    p.perform(NodeKind::ForExpr, |p| {
+    p.perform(NodeKind::ForLoop, |p| {
         p.assert(NodeKind::For);
         for_pattern(p)?;
         p.expect(NodeKind::In)?;
@@ -1075,7 +1074,7 @@ fn for_pattern(p: &mut Parser) -> ParseResult {
 
 /// Parse an import expression.
 fn import_expr(p: &mut Parser) -> ParseResult {
-    p.perform(NodeKind::ImportExpr, |p| {
+    p.perform(NodeKind::ModuleImport, |p| {
         p.assert(NodeKind::Import);
 
         if !p.eat_if(NodeKind::Star) {
@@ -1103,7 +1102,7 @@ fn import_expr(p: &mut Parser) -> ParseResult {
 
 /// Parse an include expression.
 fn include_expr(p: &mut Parser) -> ParseResult {
-    p.perform(NodeKind::IncludeExpr, |p| {
+    p.perform(NodeKind::ModuleInclude, |p| {
         p.assert(NodeKind::Include);
         expr(p)
     })
@@ -1111,7 +1110,7 @@ fn include_expr(p: &mut Parser) -> ParseResult {
 
 /// Parse a break expression.
 fn break_expr(p: &mut Parser) -> ParseResult {
-    p.perform(NodeKind::BreakExpr, |p| {
+    p.perform(NodeKind::BreakStmt, |p| {
         p.assert(NodeKind::Break);
         Ok(())
     })
@@ -1119,7 +1118,7 @@ fn break_expr(p: &mut Parser) -> ParseResult {
 
 /// Parse a continue expression.
 fn continue_expr(p: &mut Parser) -> ParseResult {
-    p.perform(NodeKind::ContinueExpr, |p| {
+    p.perform(NodeKind::ContinueStmt, |p| {
         p.assert(NodeKind::Continue);
         Ok(())
     })
@@ -1127,7 +1126,7 @@ fn continue_expr(p: &mut Parser) -> ParseResult {
 
 /// Parse a return expression.
 fn return_expr(p: &mut Parser) -> ParseResult {
-    p.perform(NodeKind::ReturnExpr, |p| {
+    p.perform(NodeKind::ReturnStmt, |p| {
         p.assert(NodeKind::Return);
         if !p.at(NodeKind::Comma) && !p.eof() {
             expr(p)?;
diff --git a/src/parse/parser.rs b/src/parse/parser.rs
index fe04f29e..3dbb7d50 100644
--- a/src/parse/parser.rs
+++ b/src/parse/parser.rs
@@ -3,8 +3,7 @@ use std::mem;
 use std::ops::Range;
 
 use super::{TokenMode, Tokens};
-use crate::diag::ErrorPos;
-use crate::syntax::{InnerNode, NodeData, NodeKind, SyntaxNode};
+use crate::syntax::{ErrorPos, InnerNode, NodeData, NodeKind, SyntaxNode};
 use crate::util::EcoString;
 
 /// A convenient token-based parser.
diff --git a/src/parse/resolve.rs b/src/parse/resolve.rs
index d68282c0..9fde0cf4 100644
--- a/src/parse/resolve.rs
+++ b/src/parse/resolve.rs
@@ -1,7 +1,7 @@
 use unscanny::Scanner;
 
 use super::{is_ident, is_newline};
-use crate::syntax::ast::RawNode;
+use crate::syntax::RawKind;
 use crate::util::EcoString;
 
 /// Resolve all escape sequences in a string.
@@ -46,17 +46,17 @@ pub fn resolve_hex(sequence: &str) -> Option<char> {
 }
 
 /// Resolve the language tag and trim the raw text.
-pub fn resolve_raw(column: usize, backticks: usize, text: &str) -> RawNode {
+pub fn resolve_raw(column: usize, backticks: usize, text: &str) -> RawKind {
     if backticks > 1 {
         let (tag, inner) = split_at_lang_tag(text);
         let (text, block) = trim_and_split_raw(column, inner);
-        RawNode {
+        RawKind {
             lang: is_ident(tag).then(|| tag.into()),
             text: text.into(),
             block,
         }
     } else {
-        RawNode {
+        RawKind {
             lang: None,
             text: split_lines(text).join("\n").into(),
             block: false,
diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs
index 7cba1823..73c64d1e 100644
--- a/src/parse/tokens.rs
+++ b/src/parse/tokens.rs
@@ -4,10 +4,8 @@ use unicode_xid::UnicodeXID;
 use unscanny::Scanner;
 
 use super::resolve::{resolve_hex, resolve_raw, resolve_string};
-use crate::diag::ErrorPos;
 use crate::geom::{AngleUnit, LengthUnit};
-use crate::syntax::ast::{RawNode, Unit};
-use crate::syntax::NodeKind;
+use crate::syntax::{ErrorPos, NodeKind, RawKind, Unit};
 use crate::util::EcoString;
 
 /// An iterator over the tokens of a string of source code.
@@ -199,14 +197,25 @@ impl<'s> Tokens<'s> {
             '[' => NodeKind::LeftBracket,
             ']' => NodeKind::RightBracket,
 
+            // Multi-char things.
+            '#' => self.hash(start),
+            '.' if self.s.eat_if("..") => NodeKind::Shorthand('\u{2026}'),
+            '-' => self.hyph(),
+            'h' if self.s.eat_if("ttp://") || self.s.eat_if("ttps://") => {
+                self.link(start)
+            }
+            '`' => self.raw(),
+            c if c.is_ascii_digit() => self.numbering(start),
+            '<' => self.label(),
+            '@' => self.reference(start),
+
             // Escape sequences.
             '\\' => self.backslash(),
 
             // Single-char things.
-            '~' => NodeKind::Tilde,
-            '.' if self.s.eat_if("..") => NodeKind::Dot3,
-            '\'' => NodeKind::Quote { double: false },
-            '"' => NodeKind::Quote { double: true },
+            '~' => NodeKind::Shorthand('\u{00A0}'),
+            '\'' => NodeKind::SmartQuote { double: false },
+            '"' => NodeKind::SmartQuote { double: true },
             '*' if !self.in_word() => NodeKind::Star,
             '_' if !self.in_word() => NodeKind::Underscore,
             '$' => NodeKind::Dollar,
@@ -215,17 +224,6 @@ impl<'s> Tokens<'s> {
             '/' => NodeKind::Slash,
             ':' => NodeKind::Colon,
 
-            // Multi-char things.
-            '#' => self.hash(start),
-            '-' => self.hyph(),
-            'h' if self.s.eat_if("ttp://") || self.s.eat_if("ttps://") => {
-                self.link(start)
-            }
-            '`' => self.raw(),
-            c if c.is_ascii_digit() => self.numbering(start),
-            '<' => self.label(),
-            '@' => self.reference(start),
-
             // Plain text.
             _ => self.text(start),
         }
@@ -291,8 +289,8 @@ impl<'s> Tokens<'s> {
             }
 
             // Linebreaks.
-            Some(c) if c.is_whitespace() => NodeKind::Backslash,
-            None => NodeKind::Backslash,
+            Some(c) if c.is_whitespace() => NodeKind::Linebreak,
+            None => NodeKind::Linebreak,
 
             // Escapes.
             Some(c) => {
@@ -317,24 +315,17 @@ impl<'s> Tokens<'s> {
     fn hyph(&mut self) -> NodeKind {
         if self.s.eat_if('-') {
             if self.s.eat_if('-') {
-                NodeKind::Hyph3
+                NodeKind::Shorthand('\u{2014}')
             } else {
-                NodeKind::Hyph2
+                NodeKind::Shorthand('\u{2013}')
             }
         } else if self.s.eat_if('?') {
-            NodeKind::HyphQuest
+            NodeKind::Shorthand('\u{00AD}')
         } else {
             NodeKind::Minus
         }
     }
 
-    fn in_word(&self) -> bool {
-        let alphanumeric = |c: Option<char>| c.map_or(false, |c| c.is_alphanumeric());
-        let prev = self.s.scout(-2);
-        let next = self.s.peek();
-        alphanumeric(prev) && alphanumeric(next)
-    }
-
     fn link(&mut self, start: usize) -> NodeKind {
         #[rustfmt::skip]
         self.s.eat_while(|c: char| matches!(c,
@@ -360,7 +351,7 @@ impl<'s> Tokens<'s> {
 
         // Special case for empty inline block.
         if backticks == 2 {
-            return NodeKind::Raw(Arc::new(RawNode {
+            return NodeKind::Raw(Arc::new(RawKind {
                 text: EcoString::new(),
                 lang: None,
                 block: false,
@@ -567,22 +558,23 @@ impl<'s> Tokens<'s> {
             }
         }
 
-        if let Ok(f) = number.parse::<f64>() {
-            match suffix {
-                "" => NodeKind::Float(f),
-                "pt" => NodeKind::Numeric(f, Unit::Length(LengthUnit::Pt)),
-                "mm" => NodeKind::Numeric(f, Unit::Length(LengthUnit::Mm)),
-                "cm" => NodeKind::Numeric(f, Unit::Length(LengthUnit::Cm)),
-                "in" => NodeKind::Numeric(f, Unit::Length(LengthUnit::In)),
-                "deg" => NodeKind::Numeric(f, Unit::Angle(AngleUnit::Deg)),
-                "rad" => NodeKind::Numeric(f, Unit::Angle(AngleUnit::Rad)),
-                "em" => NodeKind::Numeric(f, Unit::Em),
-                "fr" => NodeKind::Numeric(f, Unit::Fr),
-                "%" => NodeKind::Numeric(f, Unit::Percent),
-                _ => NodeKind::Error(ErrorPos::Full, "invalid number suffix".into()),
-            }
-        } else {
-            NodeKind::Error(ErrorPos::Full, "invalid number".into())
+        let v = match number.parse::<f64>() {
+            Ok(v) => v,
+            Err(_) => return NodeKind::Error(ErrorPos::Full, "invalid number".into()),
+        };
+
+        match suffix {
+            "" => NodeKind::Float(v),
+            "pt" => NodeKind::Numeric(v, Unit::Length(LengthUnit::Pt)),
+            "mm" => NodeKind::Numeric(v, Unit::Length(LengthUnit::Mm)),
+            "cm" => NodeKind::Numeric(v, Unit::Length(LengthUnit::Cm)),
+            "in" => NodeKind::Numeric(v, Unit::Length(LengthUnit::In)),
+            "deg" => NodeKind::Numeric(v, Unit::Angle(AngleUnit::Deg)),
+            "rad" => NodeKind::Numeric(v, Unit::Angle(AngleUnit::Rad)),
+            "em" => NodeKind::Numeric(v, Unit::Em),
+            "fr" => NodeKind::Numeric(v, Unit::Fr),
+            "%" => NodeKind::Numeric(v, Unit::Percent),
+            _ => NodeKind::Error(ErrorPos::Full, "invalid number suffix".into()),
         }
     }
 
@@ -605,6 +597,13 @@ impl<'s> Tokens<'s> {
             NodeKind::Error(ErrorPos::End, "expected quote".into())
         }
     }
+
+    fn in_word(&self) -> bool {
+        let alphanumeric = |c: Option<char>| c.map_or(false, |c| c.is_alphanumeric());
+        let prev = self.s.scout(-2);
+        let next = self.s.peek();
+        alphanumeric(prev) && alphanumeric(next)
+    }
 }
 
 fn keyword(ident: &str) -> Option<NodeKind> {
@@ -724,7 +723,7 @@ mod tests {
     }
 
     fn Raw(text: &str, lang: Option<&str>, block: bool) -> NodeKind {
-        NodeKind::Raw(Arc::new(RawNode {
+        NodeKind::Raw(Arc::new(RawKind {
             text: text.into(),
             lang: lang.map(Into::into),
             block,
@@ -762,6 +761,43 @@ mod tests {
     /// - '/': symbols
     const BLOCKS: &str = " a1/";
 
+    // Suffixes described by four-tuples of:
+    //
+    // - block the suffix is part of
+    // - mode in which the suffix is applicable
+    // - the suffix string
+    // - the resulting suffix NodeKind
+    fn suffixes()
+    -> impl Iterator<Item = (char, Option<TokenMode>, &'static str, NodeKind)> {
+        [
+            // Whitespace suffixes.
+            (' ', None, " ", Space(0)),
+            (' ', None, "\n", Space(1)),
+            (' ', None, "\r", Space(1)),
+            (' ', None, "\r\n", Space(1)),
+            // Letter suffixes.
+            ('a', Some(Markup), "hello", Text("hello")),
+            ('a', Some(Markup), "💚", Text("💚")),
+            ('a', Some(Code), "val", Ident("val")),
+            ('a', Some(Code), "α", Ident("α")),
+            ('a', Some(Code), "_", Ident("_")),
+            // Number suffixes.
+            ('1', Some(Code), "2", Int(2)),
+            ('1', Some(Code), ".2", Float(0.2)),
+            // Symbol suffixes.
+            ('/', None, "[", LeftBracket),
+            ('/', None, "//", LineComment),
+            ('/', None, "/**/", BlockComment),
+            ('/', Some(Markup), "*", Star),
+            ('/', Some(Markup), r"\\", Escape('\\')),
+            ('/', Some(Markup), "#let", Let),
+            ('/', Some(Code), "(", LeftParen),
+            ('/', Some(Code), ":", Colon),
+            ('/', Some(Code), "+=", PlusEq),
+        ]
+        .into_iter()
+    }
+
     macro_rules! t {
         (Both $($tts:tt)*) => {
             t!(Markup $($tts)*);
@@ -771,41 +807,8 @@ mod tests {
             // Test without suffix.
             t!(@$mode: $text => $($token),*);
 
-            // Suffixes described by four-tuples of:
-            //
-            // - block the suffix is part of
-            // - mode in which the suffix is applicable
-            // - the suffix string
-            // - the resulting suffix NodeKind
-            let suffixes: &[(char, Option<TokenMode>, &str, NodeKind)] = &[
-                // Whitespace suffixes.
-                (' ', None, " ", Space(0)),
-                (' ', None, "\n", Space(1)),
-                (' ', None, "\r", Space(1)),
-                (' ', None, "\r\n", Space(1)),
-                // Letter suffixes.
-                ('a', Some(Markup), "hello", Text("hello")),
-                ('a', Some(Markup), "💚", Text("💚")),
-                ('a', Some(Code), "val", Ident("val")),
-                ('a', Some(Code), "α", Ident("α")),
-                ('a', Some(Code), "_", Ident("_")),
-                // Number suffixes.
-                ('1', Some(Code), "2", Int(2)),
-                ('1', Some(Code), ".2", Float(0.2)),
-                // Symbol suffixes.
-                ('/', None, "[", LeftBracket),
-                ('/', None, "//", LineComment),
-                ('/', None, "/**/", BlockComment),
-                ('/', Some(Markup), "*", Star),
-                ('/', Some(Markup), r"\\", Escape('\\')),
-                ('/', Some(Markup), "#let", Let),
-                ('/', Some(Code), "(", LeftParen),
-                ('/', Some(Code), ":", Colon),
-                ('/', Some(Code), "+=", PlusEq),
-            ];
-
             // Test with each applicable suffix.
-            for &(block, mode, suffix, ref token) in suffixes {
+            for (block, mode, suffix, ref token) in suffixes() {
                 let text = $text;
                 #[allow(unused_variables)]
                 let blocks = BLOCKS;
@@ -872,14 +875,14 @@ mod tests {
         t!(Markup[" /"]: "reha-world" => Text("reha-world"));
 
         // Test code symbols in text.
-        t!(Markup[" /"]: "a():\"b" => Text("a()"), Colon, Quote { double: true }, Text("b"));
+        t!(Markup[" /"]: "a():\"b" => Text("a()"), Colon, SmartQuote { double: true }, Text("b"));
         t!(Markup[" /"]: ";,|/+"  => Text(";,|/+"));
         t!(Markup[" /"]: "=-a"     => Eq, Minus, Text("a"));
         t!(Markup[" "]: "#123"     => Text("#123"));
 
         // Test text ends.
         t!(Markup[""]: "hello " => Text("hello"), Space(0));
-        t!(Markup[""]: "hello~" => Text("hello"), Tilde);
+        t!(Markup[""]: "hello~" => Text("hello"), Shorthand('\u{00A0}'));
     }
 
     #[test]
@@ -924,10 +927,10 @@ mod tests {
         t!(Markup: "_"          => Underscore);
         t!(Markup[""]: "==="    => Eq, Eq, Eq);
         t!(Markup["a1/"]: "= "  => Eq, Space(0));
-        t!(Markup[" "]: r"\"    => Backslash);
-        t!(Markup: "~"          => Tilde);
-        t!(Markup["a1/"]: "-?"  => HyphQuest);
-        t!(Markup["a "]: r"a--" => Text("a"), Hyph2);
+        t!(Markup[" "]: r"\"    => Linebreak);
+        t!(Markup: "~"          => Shorthand('\u{00A0}'));
+        t!(Markup["a1/"]: "-?"  => Shorthand('\u{00AD}'));
+        t!(Markup["a "]: r"a--" => Text("a"), Shorthand('\u{2013}'));
         t!(Markup["a1/"]: "- "  => Minus, Space(0));
         t!(Markup[" "]: "+"     => Plus);
         t!(Markup[" "]: "1."    => EnumNumbering(1));