Rewrite parser

1 files changed, 149 insertions, 119 deletions

diff --git a/src/syntax/kind.rs b/src/syntax/kind.rs
index 26e92b93..5928fa0a 100644
--- a/src/syntax/kind.rs
+++ b/src/syntax/kind.rs
@@ -1,17 +1,72 @@
-/// All syntactical building blocks that can be part of a Typst document.
+/// A syntactical building block of a Typst file.
 ///
 /// Can be created by the lexer or by the parser.
 #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
+#[repr(u8)]
 pub enum SyntaxKind {
-    /// A line comment: `// ...`.
-    LineComment,
-    /// A block comment: `/* ... */`.
-    BlockComment,
-    /// One or more whitespace characters. Single spaces are collapsed into text
-    /// nodes if they would otherwise be surrounded by text nodes.
+    /// Markup of which all lines must have a minimal indentation.
     ///
-    /// Also stores how many newlines are contained.
-    Space { newlines: usize },
+    /// Notably, the number does not determine in which column the markup
+    /// started, but to the right of which column all markup elements must be,
+    /// so it is zero except inside indent-aware constructs like lists.
+    Markup,
+    /// Plain text without markup.
+    Text,
+    /// Whitespace. Contains at most one newline in markup, as more indicate a
+    /// paragraph break.
+    Space,
+    /// A forced line break: `\`.
+    Linebreak,
+    /// A paragraph break, indicated by one or multiple blank lines.
+    Parbreak,
+    /// An escape sequence: `\#`, `\u{1F5FA}`.
+    Escape,
+    /// A shorthand for a unicode codepoint. For example, `~` for non-breaking
+    /// space or `-?` for a soft hyphen.
+    Shorthand,
+    /// Symbol notation: `:arrow:l:`. The string only contains the inner part
+    /// without leading and trailing dot.
+    Symbol,
+    /// A smart quote: `'` or `"`.
+    SmartQuote,
+    /// Strong content: `*Strong*`.
+    Strong,
+    /// Emphasized content: `_Emphasized_`.
+    Emph,
+    /// Raw text with optional syntax highlighting: `` `...` ``.
+    Raw,
+    /// A hyperlink: `https://typst.org`.
+    Link,
+    /// A label: `<intro>`.
+    Label,
+    /// A reference: `@target`.
+    Ref,
+    /// A section heading: `= Introduction`.
+    Heading,
+    /// Introduces a section heading: `=`, `==`, ...
+    HeadingMarker,
+    /// An item in a bullet list: `- ...`.
+    ListItem,
+    /// Introduces a list item: `-`.
+    ListMarker,
+    /// An item in an enumeration (numbered list): `+ ...` or `1. ...`.
+    EnumItem,
+    /// Introduces an enumeration item: `+`, `1.`.
+    EnumMarker,
+    /// An item in a term list: `/ Term: Details`.
+    TermItem,
+    /// Introduces a term item: `/`.
+    TermMarker,
+    /// A mathematical formula: `$x$`, `$ x^2 $`.
+    Math,
+    /// An atom in math: `x`, `+`, `12`.
+    Atom,
+    /// A base with optional sub- and superscripts in math: `a_1^2`.
+    Script,
+    /// A fraction in math: `x/2`.
+    Frac,
+    /// An alignment point in math: `&`.
+    AlignPoint,
 
     /// A left curly brace, starting a code block: `{`.
     LeftBrace,
@@ -37,19 +92,17 @@ pub enum SyntaxKind {
     /// The strong text toggle, multiplication operator, and wildcard import
     /// symbol: `*`.
     Star,
-    /// Toggles emphasized text and indicates a subscript in a formula: `_`.
+    /// Toggles emphasized text and indicates a subscript in math: `_`.
     Underscore,
     /// Starts and ends a math formula: `$`.
     Dollar,
-    /// The unary plus, binary addition operator, and start of enum items: `+`.
+    /// The unary plus and binary addition operator: `+`.
     Plus,
-    /// The unary negation, binary subtraction operator, and start of list
-    /// items: `-`.
+    /// The unary negation and binary subtraction operator: `-`.
     Minus,
-    /// The division operator, start of term list items, and fraction operator
-    /// in a formula: `/`.
+    /// The division operator and fraction operator in math: `/`.
     Slash,
-    /// The superscript operator in a formula: `^`.
+    /// The superscript operator in math: `^`.
     Hat,
     /// The field access and method call operator: `.`.
     Dot,
@@ -119,59 +172,6 @@ pub enum SyntaxKind {
     /// The `as` keyword.
     As,
 
-    /// Markup of which all lines must have a minimal indentation.
-    ///
-    /// Notably, the number does not determine in which column the markup
-    /// started, but to the right of which column all markup elements must be,
-    /// so it is zero except inside indent-aware constructs like lists.
-    Markup { min_indent: usize },
-    /// Plain text without markup.
-    Text,
-    /// A forced line break: `\`.
-    Linebreak,
-    /// An escape sequence: `\#`, `\u{1F5FA}`.
-    Escape,
-    /// A shorthand for a unicode codepoint. For example, `~` for non-breaking
-    /// space or `-?` for a soft hyphen.
-    Shorthand,
-    /// Symbol notation: `:arrow:l:`. The string only contains the inner part
-    /// without leading and trailing dot.
-    Symbol,
-    /// A smart quote: `'` or `"`.
-    SmartQuote,
-    /// Strong content: `*Strong*`.
-    Strong,
-    /// Emphasized content: `_Emphasized_`.
-    Emph,
-    /// Raw text with optional syntax highlighting: `` `...` ``.
-    Raw { column: usize },
-    /// A hyperlink: `https://typst.org`.
-    Link,
-    /// A label: `<intro>`.
-    Label,
-    /// A reference: `@target`.
-    Ref,
-    /// A section heading: `= Introduction`.
-    Heading,
-    /// An item in a bullet list: `- ...`.
-    ListItem,
-    /// An item in an enumeration (numbered list): `+ ...` or `1. ...`.
-    EnumItem,
-    /// An explicit enumeration numbering: `23.`.
-    EnumNumbering,
-    /// An item in a term list: `/ Term: Details`.
-    TermItem,
-    /// A mathematical formula: `$x$`, `$ x^2 $`.
-    Math,
-    /// An atom in a formula: `x`, `+`, `12`.
-    Atom,
-    /// A base with optional sub- and superscripts in a formula: `a_1^2`.
-    Script,
-    /// A fraction in a formula: `x/2`.
-    Frac,
-    /// An alignment point in a formula: `&`.
-    AlignPoint,
-
     /// An identifier: `it`.
     Ident,
     /// A boolean: `true`, `false`.
@@ -243,54 +243,103 @@ pub enum SyntaxKind {
     /// A return from a function: `return`, `return x + 1`.
     FuncReturn,
 
+    /// A line comment: `// ...`.
+    LineComment,
+    /// A block comment: `/* ... */`.
+    BlockComment,
     /// An invalid sequence of characters.
     Error,
+    /// The end of the file.
+    Eof,
 }
 
 impl SyntaxKind {
-    /// Whether this is trivia.
-    pub fn is_trivia(self) -> bool {
-        self.is_space() || self.is_comment() || self.is_error()
-    }
-
-    /// Whether this is a space.
-    pub fn is_space(self) -> bool {
-        matches!(self, Self::Space { .. })
-    }
-
-    /// Whether this is a comment.
-    pub fn is_comment(self) -> bool {
-        matches!(self, Self::LineComment | Self::BlockComment)
+    /// Is this a bracket, brace, or parenthesis?
+    pub fn is_grouping(self) -> bool {
+        matches!(
+            self,
+            Self::LeftBracket
+                | Self::LeftBrace
+                | Self::LeftParen
+                | Self::RightBracket
+                | Self::RightBrace
+                | Self::RightParen
+        )
     }
 
-    /// Whether this is an error.
-    pub fn is_error(self) -> bool {
-        matches!(self, SyntaxKind::Error)
+    /// Does this node terminate a preceding expression?
+    pub fn is_terminator(self) -> bool {
+        matches!(
+            self,
+            Self::Eof
+                | Self::Semicolon
+                | Self::RightBrace
+                | Self::RightParen
+                | Self::RightBracket
+        )
     }
 
-    /// Whether this is a left or right parenthesis.
-    pub fn is_paren(self) -> bool {
-        matches!(self, Self::LeftParen | Self::RightParen)
+    /// Is this a code or content block.
+    pub fn is_block(self) -> bool {
+        matches!(self, Self::CodeBlock | Self::ContentBlock)
     }
 
     /// Does this node need termination through a semicolon or linebreak?
     pub fn is_stmt(self) -> bool {
         matches!(
             self,
-            SyntaxKind::LetBinding
-                | SyntaxKind::SetRule
-                | SyntaxKind::ShowRule
-                | SyntaxKind::ModuleImport
-                | SyntaxKind::ModuleInclude
+            Self::LetBinding
+                | Self::SetRule
+                | Self::ShowRule
+                | Self::ModuleImport
+                | Self::ModuleInclude
         )
     }
 
+    /// Whether this kind of node is automatically skipped by the parser in
+    /// code and math mode.
+    pub fn is_trivia(self) -> bool {
+        matches!(
+            self,
+            Self::Space | Self::Parbreak | Self::LineComment | Self::BlockComment
+        )
+    }
+
+    /// Whether this is an error.
+    pub fn is_error(self) -> bool {
+        self == Self::Error
+    }
+
     /// A human-readable name for the kind.
     pub fn name(self) -> &'static str {
         match self {
-            Self::LineComment => "line comment",
-            Self::BlockComment => "block comment",
-            Self::Space { .. } => "space",
+            Self::Markup => "markup",
+            Self::Text => "text",
+            Self::Space => "space",
+            Self::Linebreak => "line break",
+            Self::Parbreak => "paragraph break",
+            Self::Escape => "escape sequence",
+            Self::Shorthand => "shorthand",
+            Self::Symbol => "symbol notation",
+            Self::Strong => "strong content",
+            Self::Emph => "emphasized content",
+            Self::Raw => "raw block",
+            Self::Link => "link",
+            Self::Label => "label",
+            Self::Ref => "reference",
+            Self::Heading => "heading",
+            Self::HeadingMarker => "heading marker",
+            Self::ListItem => "list item",
+            Self::ListMarker => "list marker",
+            Self::EnumItem => "enum item",
+            Self::EnumMarker => "enum marker",
+            Self::TermItem => "term list item",
+            Self::TermMarker => "term marker",
+            Self::Math => "math formula",
+            Self::Atom => "math atom",
+            Self::Script => "script",
+            Self::Frac => "fraction",
+            Self::AlignPoint => "alignment point",
             Self::LeftBrace => "opening brace",
             Self::RightBrace => "closing brace",
             Self::LeftBracket => "opening bracket",
@@ -309,7 +358,7 @@ impl SyntaxKind {
             Self::Slash => "slash",
             Self::Hat => "hat",
             Self::Dot => "dot",
-            Self::Eq => "assignment operator",
+            Self::Eq => "equals sign",
             Self::EqEq => "equality operator",
             Self::ExclEq => "inequality operator",
             Self::Lt => "less-than operator",
@@ -341,28 +390,6 @@ impl SyntaxKind {
             Self::Import => "keyword `import`",
             Self::Include => "keyword `include`",
             Self::As => "keyword `as`",
-            Self::Markup { .. } => "markup",
-            Self::Text => "text",
-            Self::Linebreak => "linebreak",
-            Self::Escape => "escape sequence",
-            Self::Shorthand => "shorthand",
-            Self::Symbol => "symbol notation",
-            Self::Strong => "strong content",
-            Self::Emph => "emphasized content",
-            Self::Raw { .. } => "raw block",
-            Self::Link => "link",
-            Self::Label => "label",
-            Self::Ref => "reference",
-            Self::Heading => "heading",
-            Self::ListItem => "list item",
-            Self::EnumItem => "enumeration item",
-            Self::EnumNumbering => "enumeration item numbering",
-            Self::TermItem => "term list item",
-            Self::Math => "math formula",
-            Self::Atom => "math atom",
-            Self::Script => "script",
-            Self::Frac => "fraction",
-            Self::AlignPoint => "alignment point",
             Self::Ident => "identifier",
             Self::Bool => "boolean",
             Self::Int => "integer",
@@ -398,7 +425,10 @@ impl SyntaxKind {
             Self::LoopBreak => "`break` expression",
             Self::LoopContinue => "`continue` expression",
             Self::FuncReturn => "`return` expression",
+            Self::LineComment => "line comment",
+            Self::BlockComment => "block comment",
             Self::Error => "syntax error",
+            Self::Eof => "end of file",
         }
     }
 }