diff options
| -rw-r--r-- | benches/oneshot.rs | 2 | ||||
| -rw-r--r-- | src/eval/mod.rs | 28 | ||||
| -rw-r--r-- | src/library/text/raw.rs | 33 | ||||
| -rw-r--r-- | src/parse/incremental.rs | 37 | ||||
| -rw-r--r-- | src/parse/mod.rs | 23 | ||||
| -rw-r--r-- | src/parse/parser.rs | 10 | ||||
| -rw-r--r-- | src/parse/tokens.rs | 49 | ||||
| -rw-r--r-- | src/source.rs | 4 | ||||
| -rw-r--r-- | src/syntax/ast.rs | 108 | ||||
| -rw-r--r-- | src/syntax/highlight.rs | 283 | ||||
| -rw-r--r-- | src/syntax/kind.rs | 548 | ||||
| -rw-r--r-- | src/syntax/mod.rs | 614 | ||||
| -rw-r--r-- | tests/typ/code/array.typ | 4 | ||||
| -rw-r--r-- | tests/typ/code/block.typ | 2 | ||||
| -rw-r--r-- | tests/typ/code/call.typ | 2 | ||||
| -rw-r--r-- | tests/typ/text/link.typ | 4 |
16 files changed, 848 insertions, 903 deletions
diff --git a/benches/oneshot.rs b/benches/oneshot.rs index 23f829b3..d4727512 100644 --- a/benches/oneshot.rs +++ b/benches/oneshot.rs @@ -66,7 +66,7 @@ fn bench_edit(iai: &mut Iai) { fn bench_highlight(iai: &mut Iai) { let source = Source::detached(TEXT); iai.run(|| { - typst::syntax::highlight_node( + typst::syntax::highlight::highlight_categories( source.root(), 0 .. source.len_bytes(), &mut |_, _| {}, diff --git a/src/eval/mod.rs b/src/eval/mod.rs index 5bbac77e..cc9d3422 100644 --- a/src/eval/mod.rs +++ b/src/eval/mod.rs @@ -133,25 +133,25 @@ pub trait Eval { fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output>; } -impl Eval for Markup { +impl Eval for MarkupNode { type Output = Content; fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> { - eval_markup(vm, &mut self.nodes()) + eval_markup(vm, &mut self.items()) } } /// Evaluate a stream of markup nodes. fn eval_markup( vm: &mut Vm, - nodes: &mut impl Iterator<Item = MarkupNode>, + nodes: &mut impl Iterator<Item = MarkupItem>, ) -> SourceResult<Content> { let flow = vm.flow.take(); let mut seq = Vec::with_capacity(nodes.size_hint().1.unwrap_or_default()); while let Some(node) = nodes.next() { seq.push(match node { - MarkupNode::Expr(Expr::Set(set)) => { + MarkupItem::Expr(Expr::Set(set)) => { let styles = set.eval(vm)?; if vm.flow.is_some() { break; @@ -159,7 +159,7 @@ fn eval_markup( eval_markup(vm, nodes)?.styled_with_map(styles) } - MarkupNode::Expr(Expr::Show(show)) => { + MarkupItem::Expr(Expr::Show(show)) => { let recipe = show.eval(vm)?; if vm.flow.is_some() { break; @@ -168,7 +168,7 @@ fn eval_markup( eval_markup(vm, nodes)? .styled_with_entry(StyleEntry::Recipe(recipe).into()) } - MarkupNode::Expr(Expr::Wrap(wrap)) => { + MarkupItem::Expr(Expr::Wrap(wrap)) => { let tail = eval_markup(vm, nodes)?; vm.scopes.top.define(wrap.binding().take(), tail); wrap.body().eval(vm)?.display() @@ -189,7 +189,7 @@ fn eval_markup( Ok(Content::sequence(seq)) } -impl Eval for MarkupNode { +impl Eval for MarkupItem { type Output = Content; fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> { @@ -252,12 +252,12 @@ impl Eval for RawNode { } } -impl Eval for Math { +impl Eval for MathNode { type Output = Content; fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> { let nodes = - self.nodes().map(|node| node.eval(vm)).collect::<SourceResult<_>>()?; + self.items().map(|node| node.eval(vm)).collect::<SourceResult<_>>()?; Ok(Content::show(library::math::MathNode::Row( Arc::new(nodes), self.span(), @@ -265,7 +265,7 @@ impl Eval for Math { } } -impl Eval for MathNode { +impl Eval for MathItem { type Output = library::math::MathNode; fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> { @@ -278,7 +278,7 @@ impl Eval for MathNode { Self::Align(node) => node.eval(vm)?, Self::Group(node) => library::math::MathNode::Row( Arc::new( - node.nodes() + node.items() .map(|node| node.eval(vm)) .collect::<SourceResult<_>>()?, ), @@ -346,7 +346,7 @@ impl Eval for HeadingNode { } } -impl Eval for ListNode { +impl Eval for ListItem { type Output = Content; fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> { @@ -355,7 +355,7 @@ impl Eval for ListNode { } } -impl Eval for EnumNode { +impl Eval for EnumItem { type Output = Content; fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> { @@ -367,7 +367,7 @@ impl Eval for EnumNode { } } -impl Eval for DescNode { +impl Eval for DescItem { type Output = Content; fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> { diff --git a/src/library/text/raw.rs b/src/library/text/raw.rs index a64b1a92..8b0874f8 100644 --- a/src/library/text/raw.rs +++ b/src/library/text/raw.rs @@ -8,8 +8,6 @@ use syntect::parsing::SyntaxSet; use super::{FontFamily, Hyphenate, TextNode}; use crate::library::layout::BlockSpacing; use crate::library::prelude::*; -use crate::parse::TokenMode; -use crate::syntax; /// Monospaced text with optional syntax highlighting. #[derive(Debug, Hash)] @@ -73,14 +71,14 @@ impl Show for RawNode { .into(); let mut realized = if matches!(lang.as_deref(), Some("typ" | "typst" | "typc")) { - let mode = match lang.as_deref() { - Some("typc") => TokenMode::Code, - _ => TokenMode::Markup, + let root = match lang.as_deref() { + Some("typc") => crate::parse::parse_code(&self.text), + _ => crate::parse::parse(&self.text), }; let mut seq = vec![]; - syntax::highlight_themed(&self.text, mode, &THEME, |piece, style| { - seq.push(styled(piece, foreground, style)); + crate::syntax::highlight::highlight_themed(&root, &THEME, |range, style| { + seq.push(styled(&self.text[range], foreground, style)); }); Content::sequence(seq) @@ -167,24 +165,29 @@ pub static THEME: Lazy<Theme> = Lazy::new(|| Theme { author: Some("The Typst Project Developers".into()), settings: ThemeSettings::default(), scopes: vec![ + item("comment", Some("#8a8a8a"), None), + item("constant.character.escape", Some("#1d6c76"), None), + item("constant.character.shortcut", Some("#1d6c76"), None), item("markup.bold", None, Some(FontStyle::BOLD)), item("markup.italic", None, Some(FontStyle::ITALIC)), + item("markup.underline", None, Some(FontStyle::UNDERLINE)), + item("markup.raw", Some("#818181"), None), + item("string.other.math.typst", None, None), + item("punctuation.definition.math", Some("#298e0d"), None), + item("keyword.operator.math", Some("#1d6c76"), None), item("markup.heading, entity.name.section", None, Some(FontStyle::BOLD)), item("markup.heading.typst", None, Some(FontStyle::BOLD | FontStyle::UNDERLINE)), - item("markup.raw", Some("#818181"), None), - item("markup.list", Some("#8b41b1"), None), - item("comment", Some("#8a8a8a"), None), - item("punctuation.shortcut", Some("#1d6c76"), None), - item("constant.character.escape", Some("#1d6c76"), None), + item("punctuation.definition.list", Some("#8b41b1"), None), + item("markup.list.term", None, Some(FontStyle::BOLD)), item("entity.name.label, markup.other.reference", Some("#1d6c76"), None), item("keyword, constant.language, variable.language", Some("#d73a49"), None), item("storage.type, storage.modifier", Some("#d73a49"), None), - item("entity.other", Some("#8b41b1"), None), + item("constant", Some("#b60157"), None), + item("string", Some("#298e0d"), None), item("entity.name, variable.function, support", Some("#4b69c6"), None), item("support.macro", Some("#16718d"), None), item("meta.annotation", Some("#301414"), None), - item("constant", Some("#b60157"), None), - item("string", Some("#298e0d"), None), + item("entity.other, meta.interpolation", Some("#8b41b1"), None), item("invalid", Some("#ff0000"), None), ], }); diff --git a/src/parse/incremental.rs b/src/parse/incremental.rs index 06096a75..e0be9b6d 100644 --- a/src/parse/incremental.rs +++ b/src/parse/incremental.rs @@ -96,11 +96,10 @@ fn try_reparse( && (ahead.is_none() || change.replaced.start > child_span.end) && !ahead.map_or(false, Ahead::is_compulsory) { - ahead = - Some(Ahead::new(pos, at_start, child.kind().is_bounded())); + ahead = Some(Ahead::new(pos, at_start, is_bounded(child.kind()))); } - at_start = child.kind().is_at_start(at_start); + at_start = next_at_start(child.kind(), at_start); } } SearchState::Inside(start) => { @@ -137,7 +136,7 @@ fn try_reparse( if let SearchState::Contained(pos) = search { // Do not allow replacement of elements inside of constructs whose // opening and closing brackets look the same. - let safe_inside = node.kind().is_bounded(); + let safe_inside = is_bounded(node.kind()); let child = &mut node.children_mut()[pos.idx]; let prev_len = child.len(); let prev_descendants = child.descendants(); @@ -384,6 +383,36 @@ enum ReparseMode { MarkupElements { at_start: bool, min_indent: usize }, } +/// Whether changes _inside_ this node are safely encapsulated, so that only +/// this node must be reparsed. +fn is_bounded(kind: &NodeKind) -> bool { + match kind { + NodeKind::CodeBlock + | NodeKind::ContentBlock + | NodeKind::Backslash + | NodeKind::Tilde + | NodeKind::HyphQuest + | NodeKind::Hyph2 + | NodeKind::Hyph3 + | NodeKind::Dot3 + | NodeKind::Quote { .. } + | NodeKind::BlockComment + | NodeKind::Space { .. } + | NodeKind::Escape(_) => true, + _ => false, + } +} + +/// Whether `at_start` would still be true after this node given the +/// previous value of the property. +fn next_at_start(kind: &NodeKind, prev: bool) -> bool { + match kind { + NodeKind::Space { newlines: (1 ..) } => true, + NodeKind::Space { .. } | NodeKind::LineComment | NodeKind::BlockComment => prev, + _ => false, + } +} + #[cfg(test)] #[rustfmt::skip] mod tests { diff --git a/src/parse/mod.rs b/src/parse/mod.rs index 7eb7343b..832c297e 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -22,17 +22,6 @@ pub fn parse(text: &str) -> SyntaxNode { p.finish().into_iter().next().unwrap() } -/// Parse math directly, only used for syntax highlighting. -pub fn parse_math(text: &str) -> SyntaxNode { - let mut p = Parser::new(text, TokenMode::Math); - p.perform(NodeKind::Math, |p| { - while !p.eof() { - math_node(p); - } - }); - p.finish().into_iter().next().unwrap() -} - /// Parse code directly, only used for syntax highlighting. pub fn parse_code(text: &str) -> SyntaxNode { let mut p = Parser::new(text, TokenMode::Code); @@ -250,7 +239,7 @@ fn markup_node(p: &mut Parser, at_start: &mut bool) { // Text and markup. NodeKind::Text(_) - | NodeKind::Linebreak { .. } + | NodeKind::Backslash | NodeKind::Tilde | NodeKind::HyphQuest | NodeKind::Hyph2 @@ -353,7 +342,7 @@ fn list_node(p: &mut Parser, at_start: bool) { let min_indent = p.column(p.prev_end()); if at_start && p.eat_if(NodeKind::Space { newlines: 0 }) && !p.eof() { markup_indented(p, min_indent); - marker.end(p, NodeKind::List); + marker.end(p, NodeKind::ListItem); } else { marker.convert(p, NodeKind::Text(text)); } @@ -368,7 +357,7 @@ fn enum_node(p: &mut Parser, at_start: bool) { let min_indent = p.column(p.prev_end()); if at_start && p.eat_if(NodeKind::Space { newlines: 0 }) && !p.eof() { markup_indented(p, min_indent); - marker.end(p, NodeKind::Enum); + marker.end(p, NodeKind::EnumItem); } else { marker.convert(p, NodeKind::Text(text)); } @@ -385,7 +374,7 @@ fn desc_node(p: &mut Parser, at_start: bool) -> ParseResult { markup_line(p, |node| matches!(node, NodeKind::Colon)); p.expect(NodeKind::Colon)?; markup_indented(p, min_indent); - marker.end(p, NodeKind::Desc); + marker.end(p, NodeKind::DescItem); } else { marker.convert(p, NodeKind::Text(text)); } @@ -485,7 +474,7 @@ fn math_primary(p: &mut Parser) { match token { // Spaces, atoms and expressions. NodeKind::Space { .. } - | NodeKind::Linebreak + | NodeKind::Backslash | NodeKind::Escape(_) | NodeKind::Atom(_) | NodeKind::Ident(_) => p.eat(), @@ -820,7 +809,7 @@ fn item(p: &mut Parser, keyed: bool) -> ParseResult<NodeKind> { } if let Some(kind) = kind { msg.push_str(", found "); - msg.push_str(kind.as_str()); + msg.push_str(kind.name()); } let error = NodeKind::Error(SpanPos::Full, msg); marker.end(p, error); diff --git a/src/parse/parser.rs b/src/parse/parser.rs index 12dd324b..4b73c2b9 100644 --- a/src/parse/parser.rs +++ b/src/parse/parser.rs @@ -159,7 +159,7 @@ impl<'s> Parser<'s> { self.eat(); Ok(()) } else { - self.expected(kind.as_str()); + self.expected(kind.name()); Err(ParseError) } } @@ -293,7 +293,7 @@ impl<'s> Parser<'s> { self.stray_terminator = s; rescan = false; } else if required { - self.expected(end.as_str()); + self.expected(end.name()); self.unterminated_group = true; } } @@ -397,7 +397,7 @@ impl Parser<'_> { /// Eat the current token and add an error that it is unexpected. pub fn unexpected(&mut self) { if let Some(found) = self.peek() { - let msg = format_eco!("unexpected {}", found); + let msg = format_eco!("unexpected {}", found.name()); let error = NodeKind::Error(SpanPos::Full, msg); self.perform(error, Self::eat); } @@ -421,7 +421,7 @@ impl Parser<'_> { pub fn expected_found(&mut self, thing: &str) { match self.peek() { Some(found) => { - let msg = format_eco!("expected {}, found {}", thing, found); + let msg = format_eco!("expected {}, found {}", thing, found.name()); let error = NodeKind::Error(SpanPos::Full, msg); self.perform(error, Self::eat); } @@ -492,7 +492,7 @@ impl Marker { let mut msg = EcoString::from(msg); if msg.starts_with("expected") { msg.push_str(", found "); - msg.push_str(child.kind().as_str()); + msg.push_str(child.kind().name()); } let error = NodeKind::Error(SpanPos::Full, msg); let inner = mem::take(child); diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs index d495afa0..d3c497f3 100644 --- a/src/parse/tokens.rs +++ b/src/parse/tokens.rs @@ -108,7 +108,9 @@ impl<'s> Iterator for Tokens<'s> { // Trivia. '/' if self.s.eat_if('/') => self.line_comment(), '/' if self.s.eat_if('*') => self.block_comment(), - '*' if self.s.eat_if('/') => NodeKind::Unknown("*/".into()), + '*' if self.s.eat_if('/') => { + NodeKind::Error(SpanPos::Full, "unexpected end of block comment".into()) + } c if c.is_whitespace() => self.whitespace(c), // Other things. @@ -288,8 +290,8 @@ impl<'s> Tokens<'s> { } // Linebreaks. - Some(c) if c.is_whitespace() => NodeKind::Linebreak, - None => NodeKind::Linebreak, + Some(c) if c.is_whitespace() => NodeKind::Backslash, + None => NodeKind::Backslash, // Escapes. Some(c) => { @@ -517,7 +519,7 @@ impl<'s> Tokens<'s> { '"' => self.string(), // Invalid token. - _ => NodeKind::Unknown(self.s.from(start).into()), + _ => NodeKind::Error(SpanPos::Full, "not valid here".into()), } } @@ -556,7 +558,6 @@ impl<'s> Tokens<'s> { let number = self.s.get(start .. suffix_start); let suffix = self.s.from(suffix_start); - let all = self.s.from(start); // Find out whether it is a simple number. if suffix.is_empty() { @@ -577,10 +578,10 @@ impl<'s> Tokens<'s> { "em" => NodeKind::Numeric(f, Unit::Em), "fr" => NodeKind::Numeric(f, Unit::Fr), "%" => NodeKind::Numeric(f, Unit::Percent), - _ => NodeKind::Unknown(all.into()), + _ => NodeKind::Error(SpanPos::Full, "invalid number suffix".into()), } } else { - NodeKind::Unknown(all.into()) + NodeKind::Error(SpanPos::Full, "invalid number".into()) } } @@ -745,10 +746,6 @@ mod tests { NodeKind::Error(pos, message.into()) } - fn Invalid(invalid: &str) -> NodeKind { - NodeKind::Unknown(invalid.into()) - } - /// Building blocks for suffix testing. /// /// We extend each test case with a collection of different suffixes to make @@ -926,7 +923,7 @@ mod tests { t!(Markup: "_" => Underscore); t!(Markup[""]: "===" => Eq, Eq, Eq); t!(Markup["a1/"]: "= " => Eq, Space(0)); - t!(Markup[" "]: r"\" => Linebreak); + t!(Markup[" "]: r"\" => Backslash); t!(Markup: "~" => Tilde); t!(Markup["a1/"]: "-?" => HyphQuest); t!(Markup["a "]: r"a--" => Text("a"), Hyph2); @@ -972,6 +969,9 @@ mod tests { t!(Code[" /"]: "--1" => Minus, Minus, Int(1)); t!(Code[" /"]: "--_a" => Minus, Minus, Ident("_a")); t!(Code[" /"]: "a-b" => Ident("a-b")); + + // Test invalid. + t!(Code: r"\" => Error(Full, "not valid here")); } #[test] @@ -1107,6 +1107,9 @@ mod tests { t!(Code[" /"]: "1..2" => Int(1), Dots, Int(2)); t!(Code[" /"]: "1..2.3" => Int(1), Dots, Float(2.3)); t!(Code[" /"]: "1.2..3" => Float(1.2), Dots, Int(3)); + + // Test invalid. + t!(Code[" /"]: "1foo" => Error(Full, "invalid number suffix")); } #[test] @@ -1161,25 +1164,9 @@ mod tests { t!(Both[""]: "/*/*" => BlockComment); t!(Both[""]: "/**/" => BlockComment); t!(Both[""]: "/***" => BlockComment); - } - #[test] - fn test_tokenize_invalid() { - // Test invalidly closed block comments. - t!(Both: "*/" => Invalid("*/")); - t!(Both: "/**/*/" => BlockComment, Invalid("*/")); - - // Test invalid expressions. - t!(Code: r"\" => Invalid(r"\")); - t!(Code: "π" => Invalid("π")); - t!(Code: r"\:" => Invalid(r"\"), Colon); - t!(Code: "mealβ" => Ident("meal"), Invalid("β")); - t!(Code[" /"]: r"\a" => Invalid(r"\"), Ident("a")); - t!(Code[" /"]: "#" => Invalid("#")); - - // Test invalid number suffixes. - t!(Code[" /"]: "1foo" => Invalid("1foo")); - t!(Code: "1p%" => Invalid("1p"), Invalid("%")); - t!(Code: "1%%" => Numeric(1.0, Unit::Percent), Invalid("%")); + // Test unexpected terminator. + t!(Both: "/*Hi*/*/" => BlockComment, + Error(Full, "unexpected end of block comment")); } } diff --git a/src/source.rs b/src/source.rs index 0ada1b04..978b9986 100644 --- a/src/source.rs +++ b/src/source.rs @@ -10,7 +10,7 @@ use unscanny::Scanner; use crate::diag::SourceResult; use crate::parse::{is_newline, parse, reparse}; -use crate::syntax::ast::Markup; +use crate::syntax::ast::MarkupNode; use crate::syntax::{Span, SyntaxNode}; use crate::util::{PathExt, StrExt}; @@ -64,7 +64,7 @@ impl Source { } /// The root node of the file's typed abstract syntax tree. - pub fn ast(&self) -> SourceResult<Markup> { + pub fn ast(&self) -> SourceResult<MarkupNode> { let errors = self.root.errors(); if errors.is_empty() { Ok(self.root.cast().expect("root node must be markup")) diff --git a/src/syntax/ast.rs b/src/syntax/ast.rs index 6a016e79..aa590da2 100644 --- a/src/syntax/ast.rs +++ b/src/syntax/ast.rs @@ -1,6 +1,6 @@ //! A typed layer over the untyped syntax tree. //! -//! The AST is rooted in the [`Markup`] node. +//! The AST is rooted in the [`MarkupNode`]. use std::num::NonZeroUsize; use std::ops::Deref; @@ -54,19 +54,19 @@ macro_rules! node { node! { /// The syntactical root capable of representing a full parsed document. - Markup: NodeKind::Markup { .. } + MarkupNode: NodeKind::Markup { .. } } -impl Markup { - /// The markup nodes. - pub fn nodes(&self) -> impl Iterator<Item = MarkupNode> + '_ { +impl MarkupNode { + /// The children. + pub fn items(&self) -> impl Iterator<Item = MarkupItem> + '_ { self.0.children().filter_map(SyntaxNode::cast) } } /// A single piece of markup. #[derive(Debug, Clone, PartialEq)] -pub enum MarkupNode { +pub enum MarkupItem { /// Whitespace containing less than two newlines. Space, /// A forced line break. @@ -81,34 +81,34 @@ pub enum MarkupNode { Strong(StrongNode), /// Emphasized content: `_Emphasized_`. Emph(EmphNode), - /// A hyperlink. + /// A hyperlink: `https://typst.org`. Link(EcoString), /// A raw block with optional syntax highlighting: `` `...` ``. Raw(RawNode), - /// A math formula: `$a^2 = b^2 + c^2$`. - Math(Math), + /// A math formula: `$x$`, `$ x^2 $`. + Math(MathNode), /// A section heading: `= Introduction`. Heading(HeadingNode), /// An item in an unordered list: `- ...`. - List(ListNode), + List(ListItem), /// An item in an enumeration (ordered list): `+ ...` or `1. ...`. - Enum(EnumNode), - /// An item in a description list: `/ Term: Details. - Desc(DescNode), - /// A label. + Enum(EnumItem), + /// An item in a description list: `/ Term: Details`. + Desc(DescItem), + /// A label: `<label>`. Label(EcoString), - /// A reference. + /// A reference: `@label`. Ref(EcoString), /// An expression. Expr(Expr), } -impl TypedNode for MarkupNode { +impl TypedNode for MarkupItem { fn from_untyped(node: &SyntaxNode) -> Option<Self> { match node.kind() { NodeKind::Space { newlines: (2 ..) } => Some(Self::Parbreak), NodeKind::Space { .. } => Some(Self::Space), - NodeKind::Linebreak => Some(Self::Linebreak), + NodeKind::Backslash => Some(Self::Linebreak), NodeKind::Text(s) => Some(Self::Text(s.clone())), NodeKind::Escape(c) => Some(Self::Text((*c).into())), NodeKind::Tilde => Some(Self::Text('\u{00A0}'.into())), @@ -123,9 +123,9 @@ impl TypedNode for MarkupNode { NodeKind::Raw(raw) => Some(Self::Raw(raw.as_ref().clone())), NodeKind::Math => node.cast().map(Self::Math), NodeKind::Heading => node.cast().map(Self::Heading), - NodeKind::List => node.cast().map(Self::List), - NodeKind::Enum => node.cast().map(Self::Enum), - NodeKind::Desc => node.cast().map(Self::Desc), + NodeKind::ListItem => node.cast().map(Self::List), + NodeKind::EnumItem => node.cast().map(Self::Enum), + NodeKind::DescItem => node.cast().map(Self::Desc), NodeKind::Label(v) => Some(Self::Label(v.clone())), NodeKind::Ref(v) => Some(Self::Ref(v.clone())), _ => node.cast().map(Self::Expr), @@ -144,7 +144,7 @@ node! { impl StrongNode { /// The contents of the strong node. - pub fn body(&self) -> Markup { + pub fn body(&self) -> MarkupNode { self.0.cast_first_child().expect("strong node is missing markup body") } } @@ -156,7 +156,7 @@ node! { impl EmphNode { /// The contents of the emphasis node. - pub fn body(&self) -> Markup { + pub fn body(&self) -> MarkupNode { self.0 .cast_first_child() .expect("emphasis node is missing markup body") @@ -178,19 +178,19 @@ pub struct RawNode { node! { /// A math formula: `$x$`, `$ x^2 $`. - Math: NodeKind::Math { .. } + MathNode: NodeKind::Math { .. } } -impl Math { - /// The math nodes. - pub fn nodes(&self) -> impl Iterator<Item = MathNode> + '_ { +impl MathNode { + /// The children. + pub fn items(&self) -> impl Iterator<Item = MathItem> + '_ { self.0.children().filter_map(SyntaxNode::cast) } } /// A single piece of a math formula. #[derive(Debug, Clone, PartialEq, Hash)] -pub enum MathNode { +pub enum MathItem { /// Whitespace. Space, /// A forced line break. @@ -201,15 +201,15 @@ pub enum MathNode { Script(ScriptNode), /// A fraction: `x/2`. Frac(FracNode), - /// A math alignment indicator: `&`, `&&`. + /// An alignment indicator: `&`, `&&`. Align(AlignNode), /// Grouped mathematical material. - Group(Math), + Group(MathNode), /// An expression. Expr(Expr), } -impl TypedNode for MathNode { +impl TypedNode for MathItem { fn from_untyped(node: &SyntaxNode) -> Option<Self> { match node.kind() { NodeKind::Space { .. } => Some(Self::Space), @@ -219,7 +219,7 @@ impl TypedNode for MathNode { NodeKind::RightBracket => Some(Self::Atom(']'.into())), NodeKind::LeftParen => Some(Self::Atom('('.into())), NodeKind::RightParen => Some(Self::Atom(')'.into())), - NodeKind::Linebreak => Some(Self::Linebreak), + NodeKind::Backslash => Some(Self::Linebreak), NodeKind::Escape(c) => Some(Self::Atom((*c).into())), NodeKind::Atom(atom) => Some(Self::Atom(atom.clone())), NodeKind::Script => node.cast().map(Self::Script), @@ -242,12 +242,12 @@ node! { impl ScriptNode { /// The base of the script. - pub fn base(&self) -> MathNode { + pub fn base(&self) -> MathItem { self.0.cast_first_child().expect("subscript is missing base") } /// The subscript. - pub fn sub(&self) -> Option<MathNode> { + pub fn sub(&self) -> Option<MathItem> { self.0 .children() .skip_while(|node| !matches!(node.kind(), NodeKind::Underscore)) @@ -256,7 +256,7 @@ impl ScriptNode { } /// The superscript. - pub fn sup(&self) -> Option<MathNode> { + pub fn sup(&self) -> Option<MathItem> { self.0 .children() .skip_while(|node| !matches!(node.kind(), NodeKind::Hat)) @@ -272,12 +272,12 @@ node! { impl FracNode { /// The numerator. - pub fn num(&self) -> MathNode { + pub fn num(&self) -> MathItem { self.0.cast_first_child().expect("fraction is missing numerator") } /// The denominator. - pub fn denom(&self) -> MathNode { + pub fn denom(&self) -> MathItem { self.0.cast_last_child().expect("fraction is missing denominator") } } @@ -301,7 +301,7 @@ node! { impl HeadingNode { /// The contents of the heading. - pub fn body(&self) -> Markup { + pub fn body(&self) -> MarkupNode { self.0.cast_first_child().expect("heading is missing markup body") } @@ -318,27 +318,22 @@ impl HeadingNode { node! { /// An item in an unordered list: `- ...`. - ListNode: List + ListItem: ListItem } -impl ListNode { +impl ListItem { /// The contents of the list item. - pub fn body(&self) -> Markup { + pub fn body(&self) -> MarkupNode { self.0.cast_first_child().expect("list item is missing body") } } node! { /// An item in an enumeration (ordered list): `1. ...`. - EnumNode: Enum + EnumItem: EnumItem } -impl EnumNode { - /// The contents of the list item. - pub fn body(&self) -> Markup { - self.0.cast_first_child().expect("enum item is missing body") - } - +impl EnumItem { /// The number, if any. pub fn number(&self) -> Option<usize> { self.0.children().find_map(|node| match node.kind() { @@ -346,23 +341,28 @@ impl EnumNode { _ => None, }) } + + /// The contents of the list item. + pub fn body(&self) -> MarkupNode { + self.0.cast_first_child().expect("enum item is missing body") + } } node! { - /// An item in a description list: `/ Term: Details. - DescNode: Desc + /// An item in a description list: `/ Term: Details`. + DescItem: DescItem } -impl DescNode { - /// The term described by the list item. - pub fn term(&self) -> Markup { +impl DescItem { + /// The term described by the item. + pub fn term(&self) -> MarkupNode { self.0 .cast_first_child() .expect("description list item is missing term") } /// The description of the term. - pub fn body(&self) -> Markup { + pub fn body(&self) -> MarkupNode { self.0 .cast_last_child() .expect("description list item is missing body") @@ -586,7 +586,7 @@ node! { impl ContentBlock { /// The contained markup. - pub fn body(&self) -> Markup { + pub fn body(&self) -> MarkupNode { self.0.cast_first_child().expect("content is missing body") } } diff --git a/src/syntax/highlight.rs b/src/syntax/highlight.rs index e3640562..60d349d1 100644 --- a/src/syntax/highlight.rs +++ b/src/syntax/highlight.rs @@ -1,3 +1,5 @@ +//! Syntax highlighting for Typst source code. + use std::fmt::Write; use std::ops::Range; @@ -5,85 +7,9 @@ use syntect::highlighting::{Color, FontStyle, Highlighter, Style, Theme}; use syntect::parsing::Scope; use super::{NodeKind, SyntaxNode}; -use crate::parse::TokenMode; - -/// Provide highlighting categories for the descendants of a node that fall into -/// a range. -pub fn highlight_node<F>(root: &SyntaxNode, range: Range<usize>, mut f: F) -where - F: FnMut(Range<usize>, Category), -{ - highlight_node_impl(0, root, range, &mut f) -} - -/// Provide highlighting categories for the descendants of a node that fall into -/// a range. -pub fn highlight_node_impl<F>( - mut offset: usize, - node: &SyntaxNode, - range: Range<usize>, - f: &mut F, -) where - F: FnMut(Range<usize>, Category), -{ - for (i, child) in node.children().enumerate() { - let span = offset .. offset + child.len(); - if range.start <= span.end && range.end >= span.start { - if let Some(category) = Category::determine(child, node, i) { - f(span, category); - } - highlight_node_impl(offset, child, range.clone(), f); - } - offset += child.len(); - } -} - -/// Highlight source text in a theme by calling `f` with each consecutive piece -/// and its style. -pub fn highlight_themed<F>(text: &str, mode: TokenMode, theme: &Theme, mut f: F) -where - F: FnMut(&str, Style), -{ - let root = match mode { - TokenMode::Markup => crate::parse::parse(text), - TokenMode::Math => crate::parse::parse_math(text), - TokenMode::Code => crate::parse::parse_code(text), - }; - - let highlighter = Highlighter::new(&theme); - highlight_themed_impl(text, 0, &root, vec![], &highlighter, &mut f); -} - -/// Recursive implementation for highlighting with a syntect theme. -fn highlight_themed_impl<F>( - text: &str, - mut offset: usize, - node: &SyntaxNode, - scopes: Vec<Scope>, - highlighter: &Highlighter, - f: &mut F, -) where - F: FnMut(&str, Style), -{ - if node.children().len() == 0 { - let piece = &text[offset .. offset + node.len()]; - let style = highlighter.style_for_stack(&scopes); - f(piece, style); - return; - } - - for (i, child) in node.children().enumerate() { - let mut scopes = scopes.clone(); - if let Some(category) = Category::determine(child, node, i) { - scopes.push(Scope::new(category.tm_scope()).unwrap()) - } - highlight_themed_impl(text, offset, child, scopes, highlighter, f); - offset += child.len(); - } -} /// Highlight source text into a standalone HTML document. -pub fn highlight_html(text: &str, mode: TokenMode, theme: &Theme) -> String { +pub fn highlight_html(text: &str, theme: &Theme) -> String { let mut buf = String::new(); buf.push_str("<!DOCTYPE html>\n"); buf.push_str("<html>\n"); @@ -91,18 +17,19 @@ pub fn highlight_html(text: &str, mode: TokenMode, theme: &Theme) -> String { buf.push_str(" <meta charset=\"utf-8\">\n"); buf.push_str("</head>\n"); buf.push_str("<body>\n"); - buf.push_str(&highlight_pre(text, mode, theme)); + buf.push_str(&highlight_pre(text, theme)); buf.push_str("\n</body>\n"); buf.push_str("</html>\n"); buf } /// Highlight source text into an HTML pre element. -pub fn highlight_pre(text: &str, mode: TokenMode, theme: &Theme) -> String { +pub fn highlight_pre(text: &str, theme: &Theme) -> String { let mut buf = String::new(); buf.push_str("<pre>\n"); - highlight_themed(text, mode, theme, |piece, style| { + let root = crate::parse::parse(text); + highlight_themed(&root, theme, |range, style| { let styled = style != Style::default(); if styled { buf.push_str("<span style=\""); @@ -127,7 +54,7 @@ pub fn highlight_pre(text: &str, mode: TokenMode, theme: &Theme) -> String { buf.push_str("\">"); } - buf.push_str(piece); + buf.push_str(&text[range]); if styled { buf.push_str("</span>"); @@ -138,19 +65,82 @@ pub fn highlight_pre(text: &str, mode: TokenMode, theme: &Theme) -> String { buf } +/// Highlight a syntax node in a theme by calling `f` with ranges and their +/// styles. +pub fn highlight_themed<F>(root: &SyntaxNode, theme: &Theme, mut f: F) +where + F: FnMut(Range<usize>, Style), +{ + fn process<F>( + mut offset: usize, + node: &SyntaxNode, + scopes: Vec<Scope>, + highlighter: &Highlighter, + f: &mut F, + ) where + F: FnMut(Range<usize>, Style), + { + if node.children().len() == 0 { + let range = offset .. offset + node.len(); + let style = highlighter.style_for_stack(&scopes); + f(range, style); + return; + } + + for (i, child) in node.children().enumerate() { + let mut scopes = scopes.clone(); + if let Some(category) = Category::determine(child, node, i) { + scopes.push(Scope::new(category.tm_scope()).unwrap()) + } + process(offset, child, scopes, highlighter, f); + offset += child.len(); + } + } + + let highlighter = Highlighter::new(&theme); + process(0, root, vec![], &highlighter, &mut f); +} + +/// Highlight a syntax node by calling `f` with ranges overlapping `within` and +/// their categories. +pub fn highlight_categories<F>(root: &SyntaxNode, within: Range<usize>, mut f: F) +where + F: FnMut(Range<usize>, Category), +{ + fn process<F>(mut offset: usize, node: &SyntaxNode, range: Range<usize>, f: &mut F) + where + F: FnMut(Range<usize>, Category), + { + for (i, child) in node.children().enumerate() { + let span = offset .. offset + child.len(); + if range.start <= span.end && range.end >= span.start { + if let Some(category) = Category::determine(child, node, i) { + f(span, category); + } + process(offset, child, range.clone(), f); + } + offset += child.len(); + } + } + + process(0, root, within, &mut f) +} + /// The syntax highlighting category of a node. #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] pub enum Category { /// A line or block comment. Comment, - /// Any kind of bracket, parenthesis or brace. + /// A square bracket, parenthesis or brace. Bracket, /// Punctuation in code. Punctuation, - /// An easily typable shortcut to a unicode codepoint. - Shortcut, /// An escape sequence. Escape, + /// An easily typable shortcut to a unicode codepoint. + Shortcut, + /// A smart quote. + Quote, /// Strong text. Strong, /// Emphasized text. @@ -159,38 +149,40 @@ pub enum Category { Link, /// Raw text or code. Raw, - /// A math formula. + /// A full math formula. Math, + /// The delimiters of a math formula. + MathDelimiter, + /// A symbol with special meaning in a math formula. + MathSymbol, /// A section heading. Heading, + /// A full item of a list, enumeration or description list. + ListItem, /// A marker of a list, enumeration, or description list. ListMarker, /// A term in a description list. - Term, + ListTerm, /// A label. Label, /// A reference. Ref, /// A keyword. Keyword, + /// A literal defined by a keyword like `none`, `auto` or a boolean. + KeywordLiteral, /// An operator symbol. Operator, - /// The none literal. - None, - /// The auto literal. - Auto, - /// A boolean literal. - Bool, /// A numeric literal. Number, /// A string literal. String, - /// A function. + /// A function or method name. Function, - /// An interpolated variable in markup. + /// An interpolated variable in markup or math. Interpolated, - /// An invalid node. - Invalid, + /// A syntax error. + Error, } impl Category { @@ -214,40 +206,38 @@ impl Category { NodeKind::RightParen => Some(Category::Bracket), NodeKind::Comma => Some(Category::Punctuation), NodeKind::Semicolon => Some(Category::Punctuation), - NodeKind::Colon => match parent.kind() { - NodeKind::Desc => Some(Category::Term), - _ => Some(Category::Punctuation), - }, + NodeKind::Colon => Some(Category::Punctuation), NodeKind::Star => match parent.kind() { NodeKind::Strong => None, _ => Some(Category::Operator), }, NodeKind::Underscore => match parent.kind() { - NodeKind::Script => Some(Category::Shortcut), + NodeKind::Script => Some(Category::MathSymbol), _ => None, }, - NodeKind::Dollar => Some(Category::Math), + NodeKind::Dollar => Some(Category::MathDelimiter), + NodeKind::Backslash => Some(Category::Shortcut), NodeKind::Tilde => Some(Category::Shortcut), NodeKind::HyphQuest => Some(Category::Shortcut), NodeKind::Hyph2 => Some(Category::Shortcut), NodeKind::Hyph3 => Some(Category::Shortcut), NodeKind::Dot3 => Some(Category::Shortcut), - NodeKind::Quote { .. } => None, - NodeKind::Plus => match parent.kind() { - NodeKind::Enum => Some(Category::ListMarker), - _ => Some(Category::Operator), - }, - NodeKind::Minus => match parent.kind() { - NodeKind::List => Some(Category::ListMarker), - _ => Some(Category::Operator), - }, - NodeKind::Slash => match parent.kind() { - NodeKind::Desc => Some(Category::ListMarker), - NodeKind::Frac => Some(Category::Shortcut), - _ => Some(Category::Operator), - }, - NodeKind::Hat => Some(Category::Shortcut), - NodeKind::Amp => Some(Category::Shortcut), + NodeKind::Quote { .. } => Some(Category::Quote), + NodeKind::Plus => Some(match parent.kind() { + NodeKind::EnumItem => Category::ListMarker, + _ => Category::Operator, + }), + NodeKind::Minus => Some(match parent.kind() { + NodeKind::ListItem => Category::ListMarker, + _ => Category::Operator, + }), + NodeKind::Slash => Some(match parent.kind() { + NodeKind::DescItem => Category::ListMarker, + NodeKind::Frac => Category::MathSymbol, + _ => Category::Operator, + }), + NodeKind::Hat => Some(Category::MathSymbol), + NodeKind::Amp => Some(Category::MathSymbol), NodeKind::Dot => Some(Category::Punctuation), NodeKind::Eq => match parent.kind() { NodeKind::Heading => None, @@ -269,8 +259,8 @@ impl Category { NodeKind::Not => Some(Category::Keyword), NodeKind::And => Some(Category::Keyword), NodeKind::Or => Some(Category::Keyword), - NodeKind::None => Some(Category::None), - NodeKind::Auto => Some(Category::Auto), + NodeKind::None => Some(Category::KeywordLiteral), + NodeKind::Auto => Some(Category::KeywordLiteral), NodeKind::Let => Some(Category::Keyword), NodeKind::Set => Some(Category::Keyword), NodeKind::Show => Some(Category::Keyword), @@ -289,37 +279,35 @@ impl Category { NodeKind::As => Some(Category::Keyword), NodeKind::Markup { .. } => match parent.kind() { - NodeKind::Desc + NodeKind::DescItem if parent .children() .take_while(|child| child.kind() != &NodeKind::Colon) .find(|c| matches!(c.kind(), NodeKind::Markup { .. })) .map_or(false, |ident| std::ptr::eq(ident, child)) => { - Some(Category::Term) + Some(Category::ListTerm) } _ => None, }, - NodeKind::Linebreak { .. } => Some(Category::Shortcut), NodeKind::Text(_) => None, NodeKind::Escape(_) => Some(Category::Escape), NodeKind::Strong => Some(Category::Strong), NodeKind::Emph => Some(Category::Emph), NodeKind::Link(_) => Some(Category::Link), NodeKind::Raw(_) => Some(Category::Raw), - NodeKind::Math => None, - NodeKind::Heading => Some(Category::Heading), - NodeKind::List => None, - NodeKind::Enum => None, - NodeKind::EnumNumbering(_) => Some(Category::ListMarker), - NodeKind::Desc => None, - NodeKind::Label(_) => Some(Category::Label), - NodeKind::Ref(_) => Some(Category::Ref), - + NodeKind::Math => Some(Category::Math), NodeKind::Atom(_) => None, NodeKind::Script => None, NodeKind::Frac => None, NodeKind::Align => None, + NodeKind::Heading => Some(Category::Heading), + NodeKind::ListItem => Some(Category::ListItem), + NodeKind::EnumItem => Some(Category::ListItem), + NodeKind::EnumNumbering(_) => Some(Category::ListMarker), + NodeKind::DescItem => Some(Category::ListItem), + NodeKind::Label(_) => Some(Category::Label), + NodeKind::Ref(_) => Some(Category::Ref), NodeKind::Ident(_) => match parent.kind() { NodeKind::Markup { .. } => Some(Category::Interpolated), @@ -341,7 +329,7 @@ impl Category { } _ => None, }, - NodeKind::Bool(_) => Some(Category::Bool), + NodeKind::Bool(_) => Some(Category::KeywordLiteral), NodeKind::Int(_) => Some(Category::Number), NodeKind::Float(_) => Some(Category::Number), NodeKind::Numeric(_, _) => Some(Category::Number), @@ -377,39 +365,40 @@ impl Category { NodeKind::ContinueExpr => None, NodeKind::ReturnExpr => None, - NodeKind::Error(_, _) => Some(Category::Invalid), - NodeKind::Unknown(_) => Some(Category::Invalid), + NodeKind::Error(_, _) => Some(Category::Error), } } /// Return the TextMate grammar scope for the given highlighting category. pub fn tm_scope(&self) -> &'static str { match self { - Self::Bracket => "punctuation.definition.typst", - Self::Punctuation => "punctuation.typst", Self::Comment => "comment.typst", - Self::Shortcut => "punctuation.shortcut.typst", - Self::Escape => "constant.character.escape.content.typst", + Self::Bracket => "punctuation.definition.bracket.typst", + Self::Punctuation => "punctuation.typst", + Self::Escape => "constant.character.escape.typst", + Self::Shortcut => "constant.character.shortcut.typst", + Self::Quote => "constant.character.quote.typst", Self::Strong => "markup.bold.typst", Self::Emph => "markup.italic.typst", Self::Link => "markup.underline.link.typst", Self::Raw => "markup.raw.typst", Self::Math => "string.other.math.typst", + Self::MathDelimiter => "punctuation.definition.math.typst", + Self::MathSymbol => "keyword.operator.math.typst", Self::Heading => "markup.heading.typst", - Self::ListMarker => "markup.list.typst", - Self::Term => "markup.list.term.typst", + Self::ListItem => "markup.list.typst", + Self::ListMarker => "punctuation.definition.list.typst", + Self::ListTerm => "markup.list.term.typst", Self::Label => "entity.name.label.typst", Self::Ref => "markup.other.reference.typst", Self::Keyword => "keyword.typst", Self::Operator => "keyword.operator.typst", - Self::None => "constant.language.none.typst", - Self::Auto => "constant.language.auto.typst", - Self::Bool => "constant.language.boolean.typst", + Self::KeywordLiteral => "constant.language.typst", Self::Number => "constant.numeric.typst", Self::String => "string.quoted.double.typst", Self::Function => "entity.name.function.typst", - Self::Interpolated => "entity.other.interpolated.typst", - Self::Invalid => "invalid.typst", + Self::Interpolated => "meta.interpolation.typst", + Self::Error => "invalid.typst", } } } @@ -428,7 +417,7 @@ mod tests { let mut vec = vec![]; let source = Source::detached(text); let full = 0 .. text.len(); - highlight_node(source.root(), full, &mut |range, category| { + highlight_categories(source.root(), full, &mut |range, category| { vec.push((range, category)); }); assert_eq!(vec, goal); diff --git a/src/syntax/kind.rs b/src/syntax/kind.rs new file mode 100644 index 00000000..e76c93aa --- /dev/null +++ b/src/syntax/kind.rs @@ -0,0 +1,548 @@ +use std::hash::{Hash, Hasher}; +use std::sync::Arc; + +use super::ast::{RawNode, Unit}; +use super::SpanPos; +use crate::util::EcoString; + +/// All syntactical building blocks that can be part of a Typst document. +/// +/// Can be emitted as a token by the tokenizer or as part of a syntax node by +/// the parser. +#[derive(Debug, Clone, PartialEq)] +pub enum NodeKind { + /// A line comment, two slashes followed by inner contents, terminated with + /// a newline: `//<str>\n`. + LineComment, + /// A block comment, a slash and a star followed by inner contents, + /// terminated with a star and a slash: `/*<str>*/`. + /// + /// The comment can contain nested block comments. + BlockComment, + /// One or more whitespace characters. Single spaces are collapsed into text + /// nodes if they would otherwise be surrounded by text nodes. + /// + /// Also stores how many newlines are contained. + Space { newlines: usize }, + + /// A left curly brace, starting a code block: `{`. + LeftBrace, + /// A right curly brace, terminating a code block: `}`. + RightBrace, + /// A left square bracket, starting a content block: `[`. + LeftBracket, + /// A right square bracket, terminating a content block: `]`. + RightBracket, + /// A left round parenthesis, starting a grouped expression, collection, + /// argument or parameter list: `(`. + LeftParen, + /// A right round parenthesis, terminating a grouped expression, collection, + /// argument or parameter list: `)`. + RightParen, + /// A comma separator in a sequence: `,`. + Comma, + /// A semicolon terminating an expression: `;`. + Semicolon, + /// A colon between name / key and value in a dictionary, argument or + /// parameter list, or between the term and body of a description list + /// term: `:`. + Colon, + /// The strong text toggle, multiplication operator, and wildcard import + /// symbol: `*`. + Star, + /// Toggles emphasized text and indicates a subscript in a formula: `_`. + Underscore, + /// Starts and ends a math formula. + Dollar, + /// A forced line break: `\`. + Backslash, + /// The non-breaking space: `~`. + Tilde, + /// The soft hyphen: `-?`. + HyphQuest, + /// The en-dash: `--`. + Hyph2, + /// The em-dash: `---`. + Hyph3, + /// The ellipsis: `...`. + Dot3, + /// A smart quote: `'` or `"`. + Quote { double: bool }, + /// The unary plus, binary addition operator, and start of enum items: `+`. + Plus, + /// The unary negation, binary subtraction operator, and start of list + /// items: `-`. + Minus, + /// The division operator, start of description list items, and fraction + /// operator in a formula: `/`. + Slash, + /// The superscript operator in a formula: `^`. + Hat, + /// The alignment operator in a formula: `&`. + Amp, + /// The field access and method call operator: `.`. + Dot, + /// The assignment operator: `=`. + Eq, + /// The equality operator: `==`. + EqEq, + /// The inequality operator: `!=`. + ExclEq, + /// The less-than operator: `<`. + Lt, + /// The less-than or equal operator: `<=`. + LtEq, + /// The greater-than operator: `>`. + Gt, + /// The greater-than or equal operator: `>=`. + GtEq, + /// The add-assign operator: `+=`. + PlusEq, + /// The subtract-assign operator: `-=`. + HyphEq, + /// The multiply-assign operator: `*=`. + StarEq, + /// The divide-assign operator: `/=`. + SlashEq, + /// The spread operator: `..`. + Dots, + /// An arrow between a closure's parameters and body: `=>`. + Arrow, + + /// The `not` operator. + Not, + /// The `and` operator. + And, + /// The `or` operator. + Or, + /// The `none` literal. + None, + /// The `auto` literal. + Auto, + /// The `let` keyword. + Let, + /// The `set` keyword. + Set, + /// The `show` keyword. + Show, + /// The `wrap` keyword. + Wrap, + /// The `if` keyword. + If, + /// The `else` keyword. + Else, + /// The `for` keyword. + For, + /// The `in` keyword. + In, + /// The `while` keyword. + While, + /// The `break` keyword. + Break, + /// The `continue` keyword. + Continue, + /// The `return` keyword. + Return, + /// The `import` keyword. + Import, + /// The `include` keyword. + Include, + /// The `from` keyword. + From, + /// The `as` keyword. + As, + + /// Markup of which all lines must have a minimal indentation. + /// + /// Notably, the number does not determine in which column the markup + /// started, but to the right of which column all markup elements must be, + /// so it is zero except inside indent-aware constructs like lists. + Markup { min_indent: usize }, + /// Consecutive text without markup. + Text(EcoString), + /// A unicode escape sequence, written as a slash and the letter "u" + /// followed by a hexadecimal unicode entity enclosed in curly braces: + /// `\u{1F5FA}`. + Escape(char), + /// Strong content: `*Strong*`. + Strong, + /// Emphasized content: `_Emphasized_`. + Emph, + /// A hyperlink: `https://typst.org`. + Link(EcoString), + /// A raw block with optional syntax highlighting: `` `...` ``. + Raw(Arc<RawNode>), + /// A math formula: `$x$`, `$ x^2 $`. + Math, + /// An atom in a math formula: `x`, `+`, `12`. + Atom(EcoString), + /// A base with optional sub- and superscript in a math formula: `a_1^2`. + Script, + /// A fraction in a math formula: `x/2`. + Frac, + /// An alignment indicator in a math formula: `&`, `&&`. + Align, + /// A section heading: `= Introduction`. + Heading, + /// An item in an unordered list: `- ...`. + ListItem, + /// An item in an enumeration (ordered list): `+ ...` or `1. ...`. + EnumItem, + /// An explicit enumeration numbering: `23.`. + EnumNumbering(usize), + /// An item in a description list: `/ Term: Details. + DescItem, + /// A label: `<label>`. + Label(EcoString), + /// A reference: `@label`. + Ref(EcoString), + + /// An identifier: `center`. + Ident(EcoString), + /// A boolean: `true`, `false`. + Bool(bool), + /// An integer: `120`. + Int(i64), + /// A floating-point number: `1.2`, `10e-4`. + Float(f64), + /// A numeric value with a unit: `12pt`, `3cm`, `2em`, `90deg`, `50%`. + Numeric(f64, Unit), + /// A quoted string: `"..."`. + Str(EcoString), + /// A code block: `{ let x = 1; x + 2 }`. + CodeBlock, + /// A content block: `[*Hi* there!]`. + ContentBlock, + /// A grouped expression: `(1 + 2)`. + GroupExpr, + /// An array expression: `(1, "hi", 12cm)`. + ArrayExpr, + /// A dictionary expression: `(thickness: 3pt, pattern: dashed)`. + DictExpr, + /// A named pair: `thickness: 3pt`. + Named, + /// A keyed pair: `"spacy key": true`. + Keyed, + /// A unary operation: `-x`. + UnaryExpr, + /// A binary operation: `a + b`. + BinaryExpr, + /// A field access: `properties.age`. + FieldAccess, + /// An invocation of a function: `f(x, y)`. + FuncCall, + /// An invocation of a method: `array.push(v)`. + MethodCall, + /// A function call's argument list: `(x, y)`. + CallArgs, + /// Spreaded arguments or a argument sink: `..x`. + Spread, + /// A closure expression: `(x, y) => z`. + ClosureExpr, + /// A closure's parameters: `(x, y)`. + ClosureParams, + /// A let expression: `let x = 1`. + LetExpr, + /// A set expression: `set text(...)`. + SetExpr, + /// A show expression: `show node: heading as [*{nody.body}*]`. + ShowExpr, + /// A wrap expression: `wrap body in columns(2, body)`. + WrapExpr, + /// An if-else expression: `if x { y } else { z }`. + IfExpr, + /// A while loop expression: `while x { ... }`. + WhileExpr, + /// A for loop expression: `for x in y { ... }`. + ForExpr, + /// A for loop's destructuring pattern: `x` or `x, y`. + ForPattern, + /// An import expression: `import a, b, c from "utils.typ"`. + ImportExpr, + /// Items to import: `a, b, c`. + ImportItems, + /// An include expression: `include "chapter1.typ"`. + IncludeExpr, + /// A break expression: `break`. + BreakExpr, + /// A continue expression: `continue`. + ContinueExpr, + /// A return expression: `return x + 1`. + ReturnExpr, + + /// An invalid sequence of characters. + Error(SpanPos, EcoString), +} + +impl NodeKind { + /// Whether this is a kind of parenthesis. + pub fn is_paren(&self) -> bool { + matches!(self, Self::LeftParen | Self::RightParen) + } + + /// Whether this is a space. + pub fn is_space(&self) -> bool { + matches!(self, Self::Space { .. }) + } + + /// Whether this is trivia. + pub fn is_trivia(&self) -> bool { + self.is_space() || matches!(self, Self::LineComment | Self::BlockComment) + } + + /// Whether this is a kind of error. + pub fn is_error(&self) -> bool { + matches!(self, NodeKind::Error(_, _)) + } + + /// A human-readable name for the kind. + pub fn name(&self) -> &'static str { + match self { + Self::LineComment => "line comment", + Self::BlockComment => "block comment", + Self::Space { .. } => "space", + Self::LeftBrace => "opening brace", + Self::RightBrace => "closing brace", + Self::LeftBracket => "opening bracket", + Self::RightBracket => "closing bracket", + Self::LeftParen => "opening paren", + Self::RightParen => "closing paren", + Self::Comma => "comma", + Self::Semicolon => "semicolon", + Self::Colon => "colon", + Self::Star => "star", + Self::Underscore => "underscore", + Self::Dollar => "dollar sign", + Self::Backslash => "linebreak", + Self::Tilde => "non-breaking space", + Self::HyphQuest => "soft hyphen", + Self::Hyph2 => "en dash", + Self::Hyph3 => "em dash", + Self::Dot3 => "ellipsis", + Self::Quote { double: false } => "single quote", + Self::Quote { double: true } => "double quote", + Self::Plus => "plus", + Self::Minus => "minus", + Self::Slash => "slash", + Self::Hat => "hat", + Self::Amp => "ampersand", + Self::Dot => "dot", + Self::Eq => "assignment operator", + Self::EqEq => "equality operator", + Self::ExclEq => "inequality operator", + Self::Lt => "less-than operator", + Self::LtEq => "less-than or equal operator", + Self::Gt => "greater-than operator", + Self::GtEq => "greater-than or equal operator", + Self::PlusEq => "add-assign operator", + Self::HyphEq => "subtract-assign operator", + Self::StarEq => "multiply-assign operator", + Self::SlashEq => "divide-assign operator", + Self::Dots => "dots", + Self::Arrow => "arrow", + Self::Not => "operator `not`", + Self::And => "operator `and`", + Self::Or => "operator `or`", + Self::None => "`none`", + Self::Auto => "`auto`", + Self::Let => "keyword `let`", + Self::Set => "keyword `set`", + Self::Show => "keyword `show`", + Self::Wrap => "keyword `wrap`", + Self::If => "keyword `if`", + Self::Else => "keyword `else`", + Self::For => "keyword `for`", + Self::In => "keyword `in`", + Self::While => "keyword `while`", + Self::Break => "keyword `break`", + Self::Continue => "keyword `continue`", + Self::Return => "keyword `return`", + Self::Import => "keyword `import`", + Self::Include => "keyword `include`", + Self::From => "keyword `from`", + Self::As => "keyword `as`", + Self::Markup { .. } => "markup", + Self::Text(_) => "text", + Self::Escape(_) => "escape sequence", + Self::Strong => "strong content", + Self::Emph => "emphasized content", + Self::Link(_) => "link", + Self::Raw(_) => "raw block", + Self::Math => "math formula", + Self::Atom(_) => "math atom", + Self::Script => "script", + Self::Frac => "fraction", + Self::Align => "alignment indicator", + Self::Heading => "heading", + Self::ListItem => "list item", + Self::EnumItem => "enumeration item", + Self::EnumNumbering(_) => "enumeration item numbering", + Self::DescItem => "description list item", + Self::Label(_) => "label", + Self::Ref(_) => "reference", + Self::Ident(_) => "identifier", + Self::Bool(_) => "boolean", + Self::Int(_) => "integer", + Self::Float(_) => "float", + Self::Numeric(_, _) => "numeric value", + Self::Str(_) => "string", + Self::CodeBlock => "code block", + Self::ContentBlock => "content block", + Self::GroupExpr => "group", + Self::ArrayExpr => "array", + Self::DictExpr => "dictionary", + Self::Named => "named pair", + Self::Keyed => "keyed pair", + Self::UnaryExpr => "unary expression", + Self::BinaryExpr => "binary expression", + Self::FieldAccess => "field access", + Self::FuncCall => "function call", + Self::MethodCall => "method call", + Self::CallArgs => "call arguments", + Self::Spread => "spread", + Self::ClosureExpr => "closure", + Self::ClosureParams => "closure parameters", + Self::LetExpr => "`let` expression", + Self::SetExpr => "`set` expression", + Self::ShowExpr => "`show` expression", + Self::WrapExpr => "`wrap` expression", + Self::IfExpr => "`if` expression", + Self::WhileExpr => "while-loop expression", + Self::ForExpr => "for-loop expression", + Self::ForPattern => "for-loop destructuring pattern", + Self::ImportExpr => "`import` expression", + Self::ImportItems => "import items", + Self::IncludeExpr => "`include` expression", + Self::BreakExpr => "`break` expression", + Self::ContinueExpr => "`continue` expression", + Self::ReturnExpr => "`return` expression", + Self::Error(_, _) => "syntax error", + } + } +} + +impl Hash for NodeKind { + fn hash<H: Hasher>(&self, state: &mut H) { + std::mem::discriminant(self).hash(state); + match self { + Self::LineComment => {} + Self::BlockComment => {} + Self::Space { newlines } => newlines.hash(state), + Self::LeftBrace => {} + Self::RightBrace => {} + Self::LeftBracket => {} + Self::RightBracket => {} + Self::LeftParen => {} + Self::RightParen => {} + Self::Comma => {} + Self::Semicolon => {} + Self::Colon => {} + Self::Star => {} + Self::Underscore => {} + Self::Dollar => {} + Self::Backslash => {} + Self::Tilde => {} + Self::HyphQuest => {} + Self::Hyph2 => {} + Self::Hyph3 => {} + Self::Dot3 => {} + Self::Quote { double } => double.hash(state), + Self::Plus => {} + Self::Minus => {} + Self::Slash => {} + Self::Hat => {} + Self::Amp => {} + Self::Dot => {} + Self::Eq => {} + Self::EqEq => {} + Self::ExclEq => {} + Self::Lt => {} + Self::LtEq => {} + Self::Gt => {} + Self::GtEq => {} + Self::PlusEq => {} + Self::HyphEq => {} + Self::StarEq => {} + Self::SlashEq => {} + Self::Dots => {} + Self::Arrow => {} + Self::Not => {} + Self::And => {} + Self::Or => {} + Self::None => {} + Self::Auto => {} + Self::Let => {} + Self::Set => {} + Self::Show => {} + Self::Wrap => {} + Self::If => {} + Self::Else => {} + Self::For => {} + Self::In => {} + Self::While => {} + Self::Break => {} + Self::Continue => {} + Self::Return => {} + Self::Import => {} + Self::Include => {} + Self::From => {} + Self::As => {} + Self::Markup { min_indent } => min_indent.hash(state), + Self::Text(s) => s.hash(state), + Self::Escape(c) => c.hash(state), + Self::Strong => {} + Self::Emph => {} + Self::Link(link) => link.hash(state), + Self::Raw(raw) => raw.hash(state), + Self::Math => {} + Self::Atom(c) => c.hash(state), + Self::Script => {} + Self::Frac => {} + Self::Align => {} + Self::Heading => {} + Self::ListItem => {} + Self::EnumItem => {} + Self::EnumNumbering(num) => num.hash(state), + Self::DescItem => {} + Self::Label(c) => c.hash(state), + Self::Ref(c) => c.hash(state), + Self::Ident(v) => v.hash(state), + Self::Bool(v) => v.hash(state), + Self::Int(v) => v.hash(state), + Self::Float(v) => v.to_bits().hash(state), + Self::Numeric(v, u) => (v.to_bits(), u).hash(state), + Self::Str(v) => v.hash(state), + Self::CodeBlock => {} + Self::ContentBlock => {} + Self::GroupExpr => {} + Self::ArrayExpr => {} + Self::DictExpr => {} + Self::Named => {} + Self::Keyed => {} + Self::UnaryExpr => {} + Self::BinaryExpr => {} + Self::FieldAccess => {} + Self::FuncCall => {} + Self::MethodCall => {} + Self::CallArgs => {} + Self::Spread => {} + Self::ClosureExpr => {} + Self::ClosureParams => {} + Self::LetExpr => {} + Self::SetExpr => {} + Self::ShowExpr => {} + Self::WrapExpr => {} + Self::IfExpr => {} + Self::WhileExpr => {} + Self::ForExpr => {} + Self::ForPattern => {} + Self::ImportExpr => {} + Self::ImportItems => {} + Self::IncludeExpr => {} + Self::BreakExpr => {} + Self::ContinueExpr => {} + Self::ReturnExpr => {} + Self::Error(pos, msg) => (pos, msg).hash(state), + } + } +} diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs index 367d0062..5ff99d03 100644 --- a/src/syntax/mod.rs +++ b/src/syntax/mod.rs @@ -1,21 +1,20 @@ //! Syntax types. pub mod ast; -mod highlight; +pub mod highlight; +mod kind; mod span; -use std::fmt::{self, Debug, Display, Formatter}; -use std::hash::{Hash, Hasher}; +use std::fmt::{self, Debug, Formatter}; use std::ops::Range; use std::sync::Arc; -pub use highlight::*; +pub use kind::*; pub use span::*; -use self::ast::{RawNode, TypedNode, Unit}; +use self::ast::TypedNode; use crate::diag::SourceError; use crate::source::SourceId; -use crate::util::EcoString; /// An inner or leaf node in the untyped syntax tree. #[derive(Clone, PartialEq, Hash)] @@ -73,8 +72,8 @@ impl SyntaxNode { } match self.kind() { - &NodeKind::Error(pos, ref message) => { - vec![SourceError::new(self.span().with_pos(pos), message)] + NodeKind::Error(pos, message) => { + vec![SourceError::new(self.span().with_pos(*pos), message)] } _ => self .children() @@ -564,602 +563,3 @@ impl PartialEq for NodeData { self.kind == other.kind && self.len == other.len } } - -/// All syntactical building blocks that can be part of a Typst document. -/// -/// Can be emitted as a token by the tokenizer or as part of a syntax node by -/// the parser. -#[derive(Debug, Clone, PartialEq)] -pub enum NodeKind { - /// A line comment, two slashes followed by inner contents, terminated with - /// a newline: `//<str>\n`. - LineComment, - /// A block comment, a slash and a star followed by inner contents, - /// terminated with a star and a slash: `/*<str>*/`. - /// - /// The comment can contain nested block comments. - BlockComment, - /// One or more whitespace characters. Single spaces are collapsed into text - /// nodes if they would otherwise be surrounded by text nodes. - /// - /// Also stores how many newlines are contained. - Space { newlines: usize }, - - /// A left curly brace, starting a code block: `{`. - LeftBrace, - /// A right curly brace, terminating a code block: `}`. - RightBrace, - /// A left square bracket, starting a content block: `[`. - LeftBracket, - /// A right square bracket, terminating a content block: `]`. - RightBracket, - /// A left round parenthesis, starting a grouped expression, collection, - /// argument or parameter list: `(`. - LeftParen, - /// A right round parenthesis, terminating a grouped expression, collection, - /// argument or parameter list: `)`. - RightParen, - /// A comma separator in a sequence: `,`. - Comma, - /// A semicolon terminating an expression: `;`. - Semicolon, - /// A colon between name / key and value in a dictionary, argument or - /// parameter list, or between the term and body of a description list - /// term: `:`. - Colon, - /// The strong text toggle, multiplication operator, and wildcard import - /// symbol: `*`. - Star, - /// Toggles emphasized text and indicates a subscript in a formula: `_`. - Underscore, - /// Starts and ends a math formula. - Dollar, - /// The non-breaking space: `~`. - Tilde, - /// The soft hyphen: `-?`. - HyphQuest, - /// The en-dash: `--`. - Hyph2, - /// The em-dash: `---`. - Hyph3, - /// The ellipsis: `...`. - Dot3, - /// A smart quote: `'` or `"`. - Quote { double: bool }, - /// The unary plus and addition operator, and start of enum items: `+`. - Plus, - /// The unary negation and subtraction operator, and start of list - /// items: `-`. - Minus, - /// The division operator, start of description list items, and fraction - /// operator in a formula: `/`. - Slash, - /// The superscript operator: `^`. - Hat, - /// The math alignment operator: `&`. - Amp, - /// The field access and method call operator: `.`. - Dot, - /// The assignment operator: `=`. - Eq, - /// The equality operator: `==`. - EqEq, - /// The inequality operator: `!=`. - ExclEq, - /// The less-than operator: `<`. - Lt, - /// The less-than or equal operator: `<=`. - LtEq, - /// The greater-than operator: `>`. - Gt, - /// The greater-than or equal operator: `>=`. - GtEq, - /// The add-assign operator: `+=`. - PlusEq, - /// The subtract-assign operator: `-=`. - HyphEq, - /// The multiply-assign operator: `*=`. - StarEq, - /// The divide-assign operator: `/=`. - SlashEq, - /// The spread operator: `..`. - Dots, - /// An arrow between a closure's parameters and body: `=>`. - Arrow, - - /// The `not` operator. - Not, - /// The `and` operator. - And, - /// The `or` operator. - Or, - /// The `none` literal. - None, - /// The `auto` literal. - Auto, - /// The `let` keyword. - Let, - /// The `set` keyword. - Set, - /// The `show` keyword. - Show, - /// The `wrap` keyword. - Wrap, - /// The `if` keyword. - If, - /// The `else` keyword. - Else, - /// The `for` keyword. - For, - /// The `in` keyword. - In, - /// The `while` keyword. - While, - /// The `break` keyword. - Break, - /// The `continue` keyword. - Continue, - /// The `return` keyword. - Return, - /// The `import` keyword. - Import, - /// The `include` keyword. - Include, - /// The `from` keyword. - From, - /// The `as` keyword. - As, - - /// Markup of which all lines must have a minimal indentation. - /// - /// Notably, the number does not determine in which column the markup - /// started, but to the right of which column all markup elements must be, - /// so it is zero except for headings and lists. - Markup { min_indent: usize }, - /// A forced line break in markup or math. - Linebreak, - /// Consecutive text without markup. While basic text with just single - /// spaces is collapsed into a single node, certain symbols that could - /// possibly be markup force text into multiple nodes. - Text(EcoString), - /// A slash and the letter "u" followed by a hexadecimal unicode entity - /// enclosed in curly braces: `\u{1F5FA}`. - Escape(char), - /// Strong content: `*Strong*`. - Strong, - /// Emphasized content: `_Emphasized_`. - Emph, - /// A hyperlink. - Link(EcoString), - /// A raw block with optional syntax highlighting: `` `...` ``. - Raw(Arc<RawNode>), - /// A math formula: `$x$`, `$ x^2 $`. - Math, - /// A section heading: `= Introduction`. - Heading, - /// An item in an unordered list: `- ...`. - List, - /// An item in an enumeration (ordered list): `+ ...` or `1. ...`. - Enum, - /// An explicit enumeration numbering: `23.`. - EnumNumbering(usize), - /// An item in a description list: `/ Term: Details. - Desc, - /// A label: `<label>`. - Label(EcoString), - /// A reference: `@label`. - Ref(EcoString), - - /// An atom in a math formula: `x`, `+`, `12`. - Atom(EcoString), - /// A base with an optional sub- and superscript in a formula: `a_1^2`. - Script, - /// A fraction: `x/2`. - Frac, - /// A math alignment indicator: `&`, `&&`. - Align, - - /// An identifier: `center`. - Ident(EcoString), - /// A boolean: `true`, `false`. - Bool(bool), - /// An integer: `120`. - Int(i64), - /// A floating-point number: `1.2`, `10e-4`. - Float(f64), - /// A numeric value with a unit: `12pt`, `3cm`, `2em`, `90deg`, `50%`. - Numeric(f64, Unit), - /// A quoted string: `"..."`. - Str(EcoString), - /// A code block: `{ let x = 1; x + 2 }`. - CodeBlock, - /// A content block: `[*Hi* there!]`. - ContentBlock, - /// A grouped expression: `(1 + 2)`. - GroupExpr, - /// An array expression: `(1, "hi", 12cm)`. - ArrayExpr, - /// A dictionary expression: `(thickness: 3pt, pattern: dashed)`. - DictExpr, - /// A named pair: `thickness: 3pt`. - Named, - /// A keyed pair: `"spacy key": true`. - Keyed, - /// A unary operation: `-x`. - UnaryExpr, - /// A binary operation: `a + b`. - BinaryExpr, - /// A field access: `properties.age`. - FieldAccess, - /// An invocation of a function: `f(x, y)`. - FuncCall, - /// An invocation of a method: `array.push(v)`. - MethodCall, - /// A function call's argument list: `(x, y)`. - CallArgs, - /// Spreaded arguments or a argument sink: `..x`. - Spread, - /// A closure expression: `(x, y) => z`. - ClosureExpr, - /// A closure's parameters: `(x, y)`. - ClosureParams, - /// A let expression: `let x = 1`. - LetExpr, - /// A set expression: `set text(...)`. - SetExpr, - /// A show expression: `show node: heading as [*{nody.body}*]`. - ShowExpr, - /// A wrap expression: `wrap body in columns(2, body)`. - WrapExpr, - /// An if-else expression: `if x { y } else { z }`. - IfExpr, - /// A while loop expression: `while x { ... }`. - WhileExpr, - /// A for loop expression: `for x in y { ... }`. - ForExpr, - /// A for loop's destructuring pattern: `x` or `x, y`. - ForPattern, - /// An import expression: `import a, b, c from "utils.typ"`. - ImportExpr, - /// Items to import: `a, b, c`. - ImportItems, - /// An include expression: `include "chapter1.typ"`. - IncludeExpr, - /// A break expression: `break`. - BreakExpr, - /// A continue expression: `continue`. - ContinueExpr, - /// A return expression: `return x + 1`. - ReturnExpr, - - /// Tokens that appear in the wrong place. - Error(SpanPos, EcoString), - /// Unknown character sequences. - Unknown(EcoString), -} - -impl NodeKind { - /// Whether this is a kind of parenthesis. - pub fn is_paren(&self) -> bool { - matches!(self, Self::LeftParen | Self::RightParen) - } - - /// Whether this is a space. - pub fn is_space(&self) -> bool { - matches!(self, Self::Space { .. }) - } - - /// Whether this is trivia. - pub fn is_trivia(&self) -> bool { - self.is_space() || matches!(self, Self::LineComment | Self::BlockComment) - } - - /// Whether this is a kind of error. - pub fn is_error(&self) -> bool { - matches!(self, NodeKind::Error(_, _) | NodeKind::Unknown(_)) - } - - /// Whether `at_start` would still be true after this node given the - /// previous value of the property. - pub fn is_at_start(&self, prev: bool) -> bool { - match self { - Self::Space { newlines: (1 ..) } => true, - Self::Space { .. } | Self::LineComment | Self::BlockComment => prev, - _ => false, - } - } - - /// Whether changes _inside_ this node are safely encapsulated, so that only - /// this node must be reparsed. - pub fn is_bounded(&self) -> bool { - match self { - Self::CodeBlock - | Self::ContentBlock - | Self::Linebreak { .. } - | Self::Tilde - | Self::HyphQuest - | Self::Hyph2 - | Self::Hyph3 - | Self::Dot3 - | Self::Quote { .. } - | Self::BlockComment - | Self::Space { .. } - | Self::Escape(_) => true, - _ => false, - } - } - - /// A human-readable name for the kind. - pub fn as_str(&self) -> &'static str { - match self { - Self::LineComment => "line comment", - Self::BlockComment => "block comment", - Self::Space { .. } => "space", - - Self::LeftBrace => "opening brace", - Self::RightBrace => "closing brace", - Self::LeftBracket => "opening bracket", - Self::RightBracket => "closing bracket", - Self::LeftParen => "opening paren", - Self::RightParen => "closing paren", - Self::Comma => "comma", - Self::Semicolon => "semicolon", - Self::Colon => "colon", - Self::Star => "star", - Self::Underscore => "underscore", - Self::Dollar => "dollar sign", - Self::Tilde => "non-breaking space", - Self::HyphQuest => "soft hyphen", - Self::Hyph2 => "en dash", - Self::Hyph3 => "em dash", - Self::Dot3 => "ellipsis", - Self::Quote { double: false } => "single quote", - Self::Quote { double: true } => "double quote", - Self::Plus => "plus", - Self::Minus => "minus", - Self::Slash => "slash", - Self::Hat => "hat", - Self::Amp => "ampersand", - Self::Dot => "dot", - Self::Eq => "assignment operator", - Self::EqEq => "equality operator", - Self::ExclEq => "inequality operator", - Self::Lt => "less-than operator", - Self::LtEq => "less-than or equal operator", - Self::Gt => "greater-than operator", - Self::GtEq => "greater-than or equal operator", - Self::PlusEq => "add-assign operator", - Self::HyphEq => "subtract-assign operator", - Self::StarEq => "multiply-assign operator", - Self::SlashEq => "divide-assign operator", - Self::Dots => "dots", - Self::Arrow => "arrow", - - Self::Not => "operator `not`", - Self::And => "operator `and`", - Self::Or => "operator `or`", - Self::None => "`none`", - Self::Auto => "`auto`", - Self::Let => "keyword `let`", - Self::Set => "keyword `set`", - Self::Show => "keyword `show`", - Self::Wrap => "keyword `wrap`", - Self::If => "keyword `if`", - Self::Else => "keyword `else`", - Self::For => "keyword `for`", - Self::In => "keyword `in`", - Self::While => "keyword `while`", - Self::Break => "keyword `break`", - Self::Continue => "keyword `continue`", - Self::Return => "keyword `return`", - Self::Import => "keyword `import`", - Self::Include => "keyword `include`", - Self::From => "keyword `from`", - Self::As => "keyword `as`", - - Self::Markup { .. } => "markup", - Self::Linebreak => "linebreak", - Self::Text(_) => "text", - Self::Escape(_) => "escape sequence", - Self::Strong => "strong content", - Self::Emph => "emphasized content", - Self::Link(_) => "link", - Self::Raw(_) => "raw block", - Self::Math => "math formula", - Self::Heading => "heading", - Self::List => "list item", - Self::Enum => "enumeration item", - Self::EnumNumbering(_) => "enumeration item numbering", - Self::Desc => "description list item", - Self::Label(_) => "label", - Self::Ref(_) => "reference", - - Self::Atom(_) => "math atom", - Self::Script => "script", - Self::Frac => "fraction", - Self::Align => "alignment indicator", - - Self::Ident(_) => "identifier", - Self::Bool(_) => "boolean", - Self::Int(_) => "integer", - Self::Float(_) => "float", - Self::Numeric(_, _) => "numeric value", - Self::Str(_) => "string", - Self::CodeBlock => "code block", - Self::ContentBlock => "content block", - Self::GroupExpr => "group", - Self::ArrayExpr => "array", - Self::DictExpr => "dictionary", - Self::Named => "named pair", - Self::Keyed => "keyed pair", - Self::UnaryExpr => "unary expression", - Self::BinaryExpr => "binary expression", - Self::FieldAccess => "field access", - Self::FuncCall => "function call", - Self::MethodCall => "method call", - Self::CallArgs => "call arguments", - Self::Spread => "spread", - Self::ClosureExpr => "closure", - Self::ClosureParams => "closure parameters", - Self::LetExpr => "`let` expression", - Self::SetExpr => "`set` expression", - Self::ShowExpr => "`show` expression", - Self::WrapExpr => "`wrap` expression", - Self::IfExpr => "`if` expression", - Self::WhileExpr => "while-loop expression", - Self::ForExpr => "for-loop expression", - Self::ForPattern => "for-loop destructuring pattern", - Self::ImportExpr => "`import` expression", - Self::ImportItems => "import items", - Self::IncludeExpr => "`include` expression", - Self::BreakExpr => "`break` expression", - Self::ContinueExpr => "`continue` expression", - Self::ReturnExpr => "`return` expression", - - Self::Error(_, _) => "parse error", - Self::Unknown(text) => match text.as_str() { - "*/" => "end of block comment", - _ => "invalid token", - }, - } - } -} - -impl Display for NodeKind { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.pad(self.as_str()) - } -} - -impl Hash for NodeKind { - fn hash<H: Hasher>(&self, state: &mut H) { - std::mem::discriminant(self).hash(state); - match self { - Self::LineComment => {} - Self::BlockComment => {} - Self::Space { newlines } => newlines.hash(state), - - Self::LeftBrace => {} - Self::RightBrace => {} - Self::LeftBracket => {} - Self::RightBracket => {} - Self::LeftParen => {} - Self::RightParen => {} - Self::Comma => {} - Self::Semicolon => {} - Self::Colon => {} - Self::Star => {} - Self::Underscore => {} - Self::Dollar => {} - Self::Tilde => {} - Self::HyphQuest => {} - Self::Hyph2 => {} - Self::Hyph3 => {} - Self::Dot3 => {} - Self::Quote { double } => double.hash(state), - Self::Plus => {} - Self::Minus => {} - Self::Slash => {} - Self::Hat => {} - Self::Amp => {} - Self::Dot => {} - Self::Eq => {} - Self::EqEq => {} - Self::ExclEq => {} - Self::Lt => {} - Self::LtEq => {} - Self::Gt => {} - Self::GtEq => {} - Self::PlusEq => {} - Self::HyphEq => {} - Self::StarEq => {} - Self::SlashEq => {} - Self::Dots => {} - Self::Arrow => {} - - Self::Not => {} - Self::And => {} - Self::Or => {} - Self::None => {} - Self::Auto => {} - Self::Let => {} - Self::Set => {} - Self::Show => {} - Self::Wrap => {} - Self::If => {} - Self::Else => {} - Self::For => {} - Self::In => {} - Self::While => {} - Self::Break => {} - Self::Continue => {} - Self::Return => {} - Self::Import => {} - Self::Include => {} - Self::From => {} - Self::As => {} - - Self::Markup { min_indent } => min_indent.hash(state), - Self::Linebreak => {} - Self::Text(s) => s.hash(state), - Self::Escape(c) => c.hash(state), - Self::Strong => {} - Self::Emph => {} - Self::Link(link) => link.hash(state), - Self::Raw(raw) => raw.hash(state), - Self::Math => {} - Self::Heading => {} - Self::List => {} - Self::Enum => {} - Self::EnumNumbering(num) => num.hash(state), - Self::Desc => {} - Self::Label(c) => c.hash(state), - Self::Ref(c) => c.hash(state), - - Self::Atom(c) => c.hash(state), - Self::Script => {} - Self::Frac => {} - Self::Align => {} - - Self::Ident(v) => v.hash(state), - Self::Bool(v) => v.hash(state), - Self::Int(v) => v.hash(state), - Self::Float(v) => v.to_bits().hash(state), - Self::Numeric(v, u) => (v.to_bits(), u).hash(state), - Self::Str(v) => v.hash(state), - Self::CodeBlock => {} - Self::ContentBlock => {} - Self::GroupExpr => {} - Self::ArrayExpr => {} - Self::DictExpr => {} - Self::Named => {} - Self::Keyed => {} - Self::UnaryExpr => {} - Self::BinaryExpr => {} - Self::FieldAccess => {} - Self::FuncCall => {} - Self::MethodCall => {} - Self::CallArgs => {} - Self::Spread => {} - Self::ClosureExpr => {} - Self::ClosureParams => {} - Self::LetExpr => {} - Self::SetExpr => {} - Self::ShowExpr => {} - Self::WrapExpr => {} - Self::IfExpr => {} - Self::WhileExpr => {} - Self::ForExpr => {} - Self::ForPattern => {} - Self::ImportExpr => {} - Self::ImportItems => {} - Self::IncludeExpr => {} - Self::BreakExpr => {} - Self::ContinueExpr => {} - Self::ReturnExpr => {} - - Self::Error(pos, msg) => (pos, msg).hash(state), - Self::Unknown(text) => text.hash(state), - } - } -} diff --git a/tests/typ/code/array.typ b/tests/typ/code/array.typ index 1fa4d13c..cb8433cb 100644 --- a/tests/typ/code/array.typ +++ b/tests/typ/code/array.typ @@ -76,10 +76,10 @@ {)} // Error: 4 expected comma -// Error: 4-6 expected expression, found end of block comment +// Error: 4-6 unexpected end of block comment {(1*/2)} -// Error: 6-8 expected expression, found invalid token +// Error: 6-8 invalid number suffix {(1, 1u 2)} // Error: 3-4 expected expression, found comma diff --git a/tests/typ/code/block.typ b/tests/typ/code/block.typ index 194cde7a..d82d497f 100644 --- a/tests/typ/code/block.typ +++ b/tests/typ/code/block.typ @@ -112,7 +112,7 @@ --- // Multiple unseparated expressions in one line. -// Error: 2-4 expected expression, found invalid token +// Error: 2-4 invalid number suffix {1u} // Should output `1`. diff --git a/tests/typ/code/call.typ b/tests/typ/code/call.typ index 5a3f45f9..dc582c9c 100644 --- a/tests/typ/code/call.typ +++ b/tests/typ/code/call.typ @@ -73,7 +73,7 @@ // Error: 7-8 expected expression, found colon #func(:) -// Error: 10-12 expected expression, found end of block comment +// Error: 10-12 unexpected end of block comment #func(a:1*/) // Error: 8 expected comma diff --git a/tests/typ/text/link.typ b/tests/typ/text/link.typ index 99b380f0..8e777956 100644 --- a/tests/typ/text/link.typ +++ b/tests/typ/text/link.typ @@ -5,7 +5,7 @@ https://example.com/ // Link with body. -#link("https://typst.app/")[Some text text text] +#link("https://typst.org/")[Some text text text] // With line break. This link appears #link("https://google.com/")[in the middle of] a paragraph. @@ -31,7 +31,7 @@ You could also make the // Transformed link. #set page(height: 60pt) #set link(underline: false) -#let mylink = link("https://typst.app/")[LINK] +#let mylink = link("https://typst.org/")[LINK] My cool #move(dx: 0.7cm, dy: 0.7cm, rotate(10deg, scale(200%, mylink))) --- |