diff options
| author | Laurenz <laurmaedje@gmail.com> | 2022-09-26 15:39:32 +0200 |
|---|---|---|
| committer | Laurenz <laurmaedje@gmail.com> | 2022-09-26 16:12:57 +0200 |
| commit | 704f2fbaf1b4483caa12f249a222c49e44f08961 (patch) | |
| tree | 146f7813fe63409df2c1bbaa487731e992d3ac71 /src | |
| parent | 2661f1a5066bd5e3f8a9c68e4a5c304c248efcb7 (diff) | |
Description lists, link syntax, and new enum syntax
Diffstat (limited to 'src')
| -rw-r--r-- | src/eval/mod.rs | 35 | ||||
| -rw-r--r-- | src/library/mod.rs | 1 | ||||
| -rw-r--r-- | src/library/structure/list.rs | 233 | ||||
| -rw-r--r-- | src/library/text/link.rs | 7 | ||||
| -rw-r--r-- | src/library/text/par.rs | 4 | ||||
| -rw-r--r-- | src/library/text/raw.rs | 2 | ||||
| -rw-r--r-- | src/model/content.rs | 25 | ||||
| -rw-r--r-- | src/model/property.rs | 7 | ||||
| -rw-r--r-- | src/model/recipe.rs | 7 | ||||
| -rw-r--r-- | src/parse/incremental.rs | 32 | ||||
| -rw-r--r-- | src/parse/mod.rs | 69 | ||||
| -rw-r--r-- | src/parse/resolve.rs | 2 | ||||
| -rw-r--r-- | src/parse/tokens.rs | 491 | ||||
| -rw-r--r-- | src/syntax/ast.rs | 54 | ||||
| -rw-r--r-- | src/syntax/highlight.rs | 130 | ||||
| -rw-r--r-- | src/syntax/mod.rs | 272 |
16 files changed, 789 insertions, 582 deletions
diff --git a/src/eval/mod.rs b/src/eval/mod.rs index fb65420d..7ef5bf76 100644 --- a/src/eval/mod.rs +++ b/src/eval/mod.rs @@ -195,16 +195,20 @@ impl Eval for MarkupNode { Ok(match self { Self::Space => Content::Space, Self::Parbreak => Content::Parbreak, - &Self::Linebreak { justified } => Content::Linebreak { justified }, + &Self::Linebreak => Content::Linebreak { justify: false }, Self::Text(text) => Content::Text(text.clone()), &Self::Quote { double } => Content::Quote { double }, Self::Strong(strong) => strong.eval(vm)?, Self::Emph(emph) => emph.eval(vm)?, + Self::Link(url) => { + Content::show(library::text::LinkNode::from_url(url.clone())) + } Self::Raw(raw) => raw.eval(vm)?, Self::Math(math) => math.eval(vm)?, Self::Heading(heading) => heading.eval(vm)?, Self::List(list) => list.eval(vm)?, Self::Enum(enum_) => enum_.eval(vm)?, + Self::Desc(desc) => desc.eval(vm)?, Self::Label(_) => Content::Empty, Self::Ref(label) => Content::show(library::structure::RefNode(label.clone())), Self::Expr(expr) => expr.eval(vm)?.display(), @@ -273,11 +277,8 @@ impl Eval for ListNode { type Output = Content; fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> { - Ok(Content::Item(library::structure::ListItem { - kind: library::structure::UNORDERED, - number: None, - body: Box::new(self.body().eval(vm)?), - })) + let body = Box::new(self.body().eval(vm)?); + Ok(Content::Item(library::structure::ListItem::List(body))) } } @@ -285,11 +286,23 @@ impl Eval for EnumNode { type Output = Content; fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> { - Ok(Content::Item(library::structure::ListItem { - kind: library::structure::ORDERED, - number: self.number(), - body: Box::new(self.body().eval(vm)?), - })) + let number = self.number(); + let body = Box::new(self.body().eval(vm)?); + Ok(Content::Item(library::structure::ListItem::Enum( + number, body, + ))) + } +} + +impl Eval for DescNode { + type Output = Content; + + fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> { + let term = self.term().eval(vm)?; + let body = self.body().eval(vm)?; + Ok(Content::Item(library::structure::ListItem::Desc(Box::new( + library::structure::DescItem { term, body }, + )))) } } diff --git a/src/library/mod.rs b/src/library/mod.rs index 91e4671c..e7617bc0 100644 --- a/src/library/mod.rs +++ b/src/library/mod.rs @@ -41,6 +41,7 @@ pub fn new() -> Scope { std.def_node::<structure::HeadingNode>("heading"); std.def_node::<structure::ListNode>("list"); std.def_node::<structure::EnumNode>("enum"); + std.def_node::<structure::DescNode>("desc"); std.def_node::<structure::TableNode>("table"); // Layout. diff --git a/src/library/structure/list.rs b/src/library/structure/list.rs index 9da14733..7a43e5db 100644 --- a/src/library/structure/list.rs +++ b/src/library/structure/list.rs @@ -1,5 +1,3 @@ -use std::fmt::Write; - use unscanny::Scanner; use crate::library::layout::{BlockSpacing, GridNode, TrackSizing}; @@ -9,9 +7,7 @@ use crate::library::utility::Numbering; /// An unordered (bulleted) or ordered (numbered) list. #[derive(Debug, Hash)] -pub struct ListNode<const L: ListKind = UNORDERED> { - /// Where the list starts. - pub start: usize, +pub struct ListNode<const L: ListKind = LIST> { /// If true, the items are separated by leading instead of list spacing. pub tight: bool, /// If true, the spacing above the list is leading instead of above spacing. @@ -20,19 +16,11 @@ pub struct ListNode<const L: ListKind = UNORDERED> { pub items: StyleVec<ListItem>, } -/// An item in a list. -#[derive(Clone, PartialEq, Hash)] -pub struct ListItem { - /// The kind of item. - pub kind: ListKind, - /// The number of the item. - pub number: Option<usize>, - /// The node that produces the item's body. - pub body: Box<Content>, -} - /// An ordered list. -pub type EnumNode = ListNode<ORDERED>; +pub type EnumNode = ListNode<ENUM>; + +/// A description list. +pub type DescNode = ListNode<DESC>; #[node(showable)] impl<const L: ListKind> ListNode<L> { @@ -44,7 +32,11 @@ impl<const L: ListKind> ListNode<L> { pub const INDENT: RawLength = RawLength::zero(); /// The space between the label and the body of each item. #[property(resolve)] - pub const BODY_INDENT: RawLength = Em::new(0.5).into(); + pub const BODY_INDENT: RawLength = Em::new(match L { + LIST | ENUM => 0.5, + DESC | _ => 1.0, + }) + .into(); /// The spacing above the list. #[property(resolve, shorthand(around))] @@ -57,19 +49,34 @@ impl<const L: ListKind> ListNode<L> { pub const SPACING: BlockSpacing = Ratio::one().into(); fn construct(_: &mut Vm, args: &mut Args) -> SourceResult<Content> { - Ok(Content::show(Self { - start: args.named("start")?.unwrap_or(1), - tight: args.named("tight")?.unwrap_or(true), - attached: args.named("attached")?.unwrap_or(false), - items: args + let items = match L { + LIST => args .all()? .into_iter() - .map(|body| ListItem { - kind: L, - number: None, - body: Box::new(body), - }) + .map(|body| ListItem::List(Box::new(body))) .collect(), + ENUM => { + let mut number: usize = args.named("start")?.unwrap_or(1); + args.all()? + .into_iter() + .map(|body| { + let item = ListItem::Enum(Some(number), Box::new(body)); + number += 1; + item + }) + .collect() + } + DESC | _ => args + .all()? + .into_iter() + .map(|item| ListItem::Desc(Box::new(item))) + .collect(), + }; + + Ok(Content::show(Self { + tight: args.named("tight")?.unwrap_or(true), + attached: args.named("attached")?.unwrap_or(false), + items, })) } } @@ -77,10 +84,7 @@ impl<const L: ListKind> ListNode<L> { impl<const L: ListKind> Show for ListNode<L> { fn unguard(&self, sel: Selector) -> ShowNode { Self { - items: self.items.map(|item| ListItem { - body: Box::new(item.body.unguard(sel).role(Role::ListItemBody)), - ..*item - }), + items: self.items.map(|item| item.unguard(sel)), ..*self } .pack() @@ -88,13 +92,12 @@ impl<const L: ListKind> Show for ListNode<L> { fn encode(&self, _: StyleChain) -> Dict { dict! { - "start" => Value::Int(self.start as i64), "tight" => Value::Bool(self.tight), "attached" => Value::Bool(self.attached), "items" => Value::Array( self.items .items() - .map(|item| Value::Content(item.body.as_ref().clone())) + .map(|item| item.encode()) .collect() ), } @@ -106,34 +109,54 @@ impl<const L: ListKind> Show for ListNode<L> { styles: StyleChain, ) -> SourceResult<Content> { let mut cells = vec![]; - let mut number = self.start; + let mut number = 1; let label = styles.get(Self::LABEL); + let indent = styles.get(Self::INDENT); + let body_indent = styles.get(Self::BODY_INDENT); + let gutter = if self.tight { + styles.get(ParNode::LEADING) + } else { + styles.get(Self::SPACING) + }; for (item, map) in self.items.iter() { - number = item.number.unwrap_or(number); + if let &ListItem::Enum(Some(n), _) = item { + number = n; + } cells.push(LayoutNode::default()); - cells.push( + + let label = if L == LIST || L == ENUM { label .resolve(world, L, number)? .styled_with_map(map.clone()) .role(Role::ListLabel) - .pack(), - ); + .pack() + } else { + LayoutNode::default() + }; + + cells.push(label); cells.push(LayoutNode::default()); - cells.push((*item.body).clone().styled_with_map(map.clone()).pack()); - number += 1; - } - let gutter = if self.tight { - styles.get(ParNode::LEADING) - } else { - styles.get(Self::SPACING) - }; + let body = match &item { + ListItem::List(body) => body.as_ref().clone(), + ListItem::Enum(_, body) => body.as_ref().clone(), + ListItem::Desc(item) => Content::sequence(vec![ + Content::Horizontal { + amount: (-body_indent).into(), + weak: false, + }, + (item.term.clone() + Content::Text(':'.into())).strong(), + Content::Space, + item.body.clone(), + ]), + }; - let indent = styles.get(Self::INDENT); - let body_indent = styles.get(Self::BODY_INDENT); + cells.push(body.styled_with_map(map.clone()).pack()); + number += 1; + } Ok(Content::block(GridNode { tracks: Spec::with_x(vec![ @@ -165,35 +188,110 @@ impl<const L: ListKind> Show for ListNode<L> { } } - Ok(realized - .role(Role::List { ordered: L == ORDERED }) - .spaced(above, below)) + Ok(realized.role(Role::List { ordered: L == ENUM }).spaced(above, below)) + } +} + +/// An item in a list. +#[derive(Clone, PartialEq, Hash)] +pub enum ListItem { + /// An item of an unordered list. + List(Box<Content>), + /// An item of an ordered list. + Enum(Option<usize>, Box<Content>), + /// An item of a description list. + Desc(Box<DescItem>), +} + +impl ListItem { + /// What kind of item this is. + pub fn kind(&self) -> ListKind { + match self { + Self::List(_) => LIST, + Self::Enum { .. } => ENUM, + Self::Desc { .. } => DESC, + } + } + + fn unguard(&self, sel: Selector) -> Self { + match self { + Self::List(body) => Self::List(Box::new(body.unguard(sel))), + Self::Enum(number, body) => Self::Enum(*number, Box::new(body.unguard(sel))), + Self::Desc(item) => Self::Desc(Box::new(DescItem { + term: item.term.unguard(sel), + body: item.body.unguard(sel), + })), + } + } + + /// Encode the item into a value. + fn encode(&self) -> Value { + match self { + Self::List(body) => Value::Content(body.as_ref().clone()), + Self::Enum(number, body) => Value::Dict(dict! { + "number" => match *number { + Some(n) => Value::Int(n as i64), + None => Value::None, + }, + "body" => Value::Content(body.as_ref().clone()), + }), + Self::Desc(item) => Value::Dict(dict! { + "term" => Value::Content(item.term.clone()), + "body" => Value::Content(item.body.clone()), + }), + } } } impl Debug for ListItem { fn fmt(&self, f: &mut Formatter) -> fmt::Result { - if self.kind == UNORDERED { - f.write_char('-')?; - } else { - if let Some(number) = self.number { - write!(f, "{}", number)?; - } - f.write_char('.')?; + match self { + Self::List(body) => write!(f, "- {body:?}"), + Self::Enum(number, body) => match number { + Some(n) => write!(f, "{n}. {body:?}"), + None => write!(f, "+ {body:?}"), + }, + Self::Desc(item) => item.fmt(f), } - f.write_char(' ')?; - self.body.fmt(f) + } +} + +/// A description list item. +#[derive(Clone, PartialEq, Hash)] +pub struct DescItem { + /// The term described by the list item. + pub term: Content, + /// The description of the term. + pub body: Content, +} + +castable! { + DescItem, + Expected: "dictionary with `term` and `body` keys", + Value::Dict(dict) => { + let term: Content = dict.get("term")?.clone().cast()?; + let body: Content = dict.get("body")?.clone().cast()?; + Self { term, body } + }, +} + +impl Debug for DescItem { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "/ {:?}: {:?}", self.term, self.body) } } /// How to label a list. pub type ListKind = usize; -/// Unordered list labelling style. -pub const UNORDERED: ListKind = 0; +/// An unordered list. +pub const LIST: ListKind = 0; + +/// An ordered list. +pub const ENUM: ListKind = 1; -/// Ordered list labelling style. -pub const ORDERED: ListKind = 1; +/// A description list. +pub const DESC: ListKind = 2; /// How to label a list or enumeration. #[derive(Debug, Clone, PartialEq, Hash)] @@ -218,8 +316,9 @@ impl Label { ) -> SourceResult<Content> { Ok(match self { Self::Default => match kind { - UNORDERED => Content::Text('•'.into()), - ORDERED | _ => Content::Text(format_eco!("{}.", number)), + LIST => Content::Text('•'.into()), + ENUM => Content::Text(format_eco!("{}.", number)), + DESC | _ => panic!("description lists don't have a label"), }, Self::Pattern(prefix, numbering, upper, suffix) => { let fmt = numbering.apply(number); diff --git a/src/library/text/link.rs b/src/library/text/link.rs index c06fea55..78ae4d23 100644 --- a/src/library/text/link.rs +++ b/src/library/text/link.rs @@ -10,6 +10,13 @@ pub struct LinkNode { pub body: Option<Content>, } +impl LinkNode { + /// Create a link node from a URL with its bare text. + pub fn from_url(url: EcoString) -> Self { + Self { dest: Destination::Url(url), body: None } + } +} + #[node(showable)] impl LinkNode { /// The fill color of text in the link. Just the surrounding text color diff --git a/src/library/text/par.rs b/src/library/text/par.rs index 6910c23a..859a7c87 100644 --- a/src/library/text/par.rs +++ b/src/library/text/par.rs @@ -181,8 +181,8 @@ pub struct LinebreakNode; #[node] impl LinebreakNode { fn construct(_: &mut Vm, args: &mut Args) -> SourceResult<Content> { - let justified = args.named("justified")?.unwrap_or(false); - Ok(Content::Linebreak { justified }) + let justify = args.named("justify")?.unwrap_or(false); + Ok(Content::Linebreak { justify }) } } diff --git a/src/library/text/raw.rs b/src/library/text/raw.rs index e7c73a91..a64b1a92 100644 --- a/src/library/text/raw.rs +++ b/src/library/text/raw.rs @@ -91,7 +91,7 @@ impl Show for RawNode { let mut highlighter = HighlightLines::new(syntax, &THEME); for (i, line) in self.text.lines().enumerate() { if i != 0 { - seq.push(Content::Linebreak { justified: false }); + seq.push(Content::Linebreak { justify: false }); } for (style, piece) in diff --git a/src/model/content.rs b/src/model/content.rs index 92d592a6..ae074b50 100644 --- a/src/model/content.rs +++ b/src/model/content.rs @@ -14,7 +14,7 @@ use super::{ use crate::diag::StrResult; use crate::library::layout::{FlowChild, FlowNode, PageNode, PlaceNode, Spacing}; use crate::library::prelude::*; -use crate::library::structure::{DocNode, ListItem, ListNode, ORDERED, UNORDERED}; +use crate::library::structure::{DocNode, ListItem, ListNode, DESC, ENUM, LIST}; use crate::library::text::{ DecoNode, EmphNode, ParChild, ParNode, StrongNode, UNDERLINE, }; @@ -62,7 +62,7 @@ pub enum Content { /// A word space. Space, /// A forced line break. - Linebreak { justified: bool }, + Linebreak { justify: bool }, /// Horizontal spacing. Horizontal { amount: Spacing, weak: bool }, /// Plain text. @@ -264,7 +264,7 @@ impl Debug for Content { match self { Self::Empty => f.pad("Empty"), Self::Space => f.pad("Space"), - Self::Linebreak { justified } => write!(f, "Linebreak({justified})"), + Self::Linebreak { justify } => write!(f, "Linebreak({justify})"), Self::Horizontal { amount, weak } => { write!(f, "Horizontal({amount:?}, {weak})") } @@ -633,8 +633,8 @@ impl<'a> ParBuilder<'a> { Content::Space => { self.0.weak(ParChild::Text(' '.into()), styles, 2); } - &Content::Linebreak { justified } => { - let c = if justified { '\u{2028}' } else { '\n' }; + &Content::Linebreak { justify } => { + let c = if justify { '\u{2028}' } else { '\n' }; self.0.destructive(ParChild::Text(c.into()), styles); } &Content::Horizontal { amount, weak } => { @@ -734,7 +734,7 @@ impl<'a> ListBuilder<'a> { .items .items() .next() - .map_or(true, |first| item.kind == first.kind) => + .map_or(true, |first| item.kind() == first.kind()) => { self.items.push(item.clone(), styles); self.tight &= self.staged.drain(..).all(|(t, _)| *t != Content::Parbreak); @@ -751,21 +751,16 @@ impl<'a> ListBuilder<'a> { fn finish(self, parent: &mut Builder<'a>) -> SourceResult<()> { let (items, shared) = self.items.finish(); let kind = match items.items().next() { - Some(item) => item.kind, + Some(item) => item.kind(), None => return Ok(()), }; - let start = 1; let tight = self.tight; let attached = tight && self.attachable; - let content = match kind { - UNORDERED => { - Content::show(ListNode::<UNORDERED> { start, tight, attached, items }) - } - ORDERED | _ => { - Content::show(ListNode::<ORDERED> { start, tight, attached, items }) - } + LIST => Content::show(ListNode::<LIST> { tight, attached, items }), + ENUM => Content::show(ListNode::<ENUM> { tight, attached, items }), + DESC | _ => Content::show(ListNode::<DESC> { tight, attached, items }), }; let stored = parent.scratch.templates.alloc(content); diff --git a/src/model/property.rs b/src/model/property.rs index 18f41eee..ab4f02e3 100644 --- a/src/model/property.rs +++ b/src/model/property.rs @@ -9,7 +9,7 @@ use super::{Interruption, NodeId, StyleChain}; use crate::eval::{RawLength, Smart}; use crate::geom::{Corners, Length, Numeric, Relative, Sides, Spec}; use crate::library::layout::PageNode; -use crate::library::structure::{EnumNode, ListNode}; +use crate::library::structure::{DescNode, EnumNode, ListNode}; use crate::library::text::ParNode; use crate::util::ReadableTypeId; @@ -68,7 +68,10 @@ impl Property { Some(Interruption::Page) } else if self.is_of::<ParNode>() { Some(Interruption::Par) - } else if self.is_of::<ListNode>() || self.is_of::<EnumNode>() { + } else if self.is_of::<ListNode>() + || self.is_of::<EnumNode>() + || self.is_of::<DescNode>() + { Some(Interruption::List) } else { None diff --git a/src/model/recipe.rs b/src/model/recipe.rs index 6b21ccf2..27b1be42 100644 --- a/src/model/recipe.rs +++ b/src/model/recipe.rs @@ -5,7 +5,7 @@ use comemo::Tracked; use super::{Content, Interruption, NodeId, Show, ShowNode, StyleChain, StyleEntry}; use crate::diag::SourceResult; use crate::eval::{Args, Func, Regex, Value}; -use crate::library::structure::{EnumNode, ListNode}; +use crate::library::structure::{DescNode, EnumNode, ListNode}; use crate::syntax::Spanned; use crate::World; @@ -93,7 +93,10 @@ impl Recipe { /// What kind of structure the property interrupts. pub fn interruption(&self) -> Option<Interruption> { if let Pattern::Node(id) = self.pattern { - if id == NodeId::of::<ListNode>() || id == NodeId::of::<EnumNode>() { + if id == NodeId::of::<ListNode>() + || id == NodeId::of::<EnumNode>() + || id == NodeId::of::<DescNode>() + { return Some(Interruption::List); } } diff --git a/src/parse/incremental.rs b/src/parse/incremental.rs index 9272dcfc..06096a75 100644 --- a/src/parse/incremental.rs +++ b/src/parse/incremental.rs @@ -407,18 +407,18 @@ mod tests { test("", 0..0, "do it", 0..5); test("a d e", 1 .. 3, " b c d", 0 .. 9); test("*~ *", 2..2, "*", 0..5); - test("_1_\n2a\n3", 5..5, "4", 0..7); - test("_1_\n2a\n3~", 8..8, "4", 5..10); + test("_1_\n2a\n3", 5..5, "4", 4..7); + test("_1_\n2a\n3~", 8..8, "4", 4..10); test("_1_ 2 3a\n4", 7..7, "5", 0..9); test("* {1+2} *", 5..6, "3", 2..7); test("a #f() e", 1 .. 6, " b c d", 0 .. 9); test("a\nb\nc\nd\ne\n", 5 .. 5, "c", 2 .. 7); test("a\n\nb\n\nc\n\nd\n\ne\n", 7 .. 7, "c", 3 .. 10); - test("a\nb\nc *hel a b lo* d\nd\ne", 13..13, "c ", 6..20); + test("a\nb\nc *hel a b lo* d\nd\ne", 13..13, "c ", 4..20); test("~~ {a} ~~", 4 .. 5, "b", 3 .. 6); test("{(0, 1, 2)}", 5 .. 6, "11pt", 0..14); test("\n= A heading", 4 .. 4, "n evocative", 0 .. 23); - test("for~your~thing", 9 .. 9, "a", 4 .. 15); + test("for~your~thing", 9 .. 9, "a", 0 .. 15); test("a your thing a", 6 .. 7, "a", 0 .. 14); test("{call(); abc}", 7 .. 7, "[]", 0 .. 15); test("#call() abc", 7 .. 7, "[]", 0 .. 10); @@ -429,17 +429,17 @@ mod tests { test("#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", 34 .. 41, "_bar_", 33 .. 40); test("{let i=1; for x in range(5) {i}}", 6 .. 6, " ", 0 .. 33); test("{let i=1; for x in range(5) {i}}", 13 .. 14, " ", 0 .. 33); - test("hello~~{x}", 7 .. 10, "#f()", 5 .. 11); - test("this~is -- in my opinion -- spectacular", 8 .. 10, "---", 5 .. 25); - test("understanding `code` is complicated", 15 .. 15, "C ", 14 .. 22); + test("hello~~{x}", 7 .. 10, "#f()", 0 .. 11); + test("this~is -- in my opinion -- spectacular", 8 .. 10, "---", 0 .. 25); + test("understanding `code` is complicated", 15 .. 15, "C ", 0 .. 22); test("{ let x = g() }", 10 .. 12, "f(54", 0 .. 17); - test(r#"a ```typst hello``` b"#, 16 .. 17, "", 2 .. 18); - test(r#"a ```typst hello```"#, 16 .. 17, "", 2 .. 18); + test(r#"a ```typst hello``` b"#, 16 .. 17, "", 0 .. 18); + test(r#"a ```typst hello```"#, 16 .. 17, "", 0 .. 18); test("#for", 4 .. 4, "//", 0 .. 6); test("#show a: f as b..", 16..16, "c", 0..18); test("a\n#let \nb", 7 .. 7, "i", 2 .. 9); test("a\n#for i \nb", 9 .. 9, "in", 2 .. 12); - test("a~https://fun/html", 13..14, "n", 2..18); + test("a~https://fun/html", 13..14, "n", 0..18); } #[test] @@ -452,7 +452,7 @@ mod tests { test("abc\n= a heading\njoke", 3 .. 4, "\nnot ", 0 .. 19); test("#let x = (1, 2 + ;~ Five\r\n\r", 20 .. 23, "2.", 0 .. 23); test("hey #myfriend", 4 .. 4, "\\", 0 .. 14); - test("hey #myfriend", 4 .. 4, "\\", 3 .. 6); + test("hey #myfriend", 4 .. 4, "\\", 0 .. 6); test("= foo\nbar\n - a\n - b", 6 .. 9, "", 0 .. 11); test("= foo\n bar\n baz", 6 .. 8, "", 0 .. 9); test(" // hi", 1 .. 1, " ", 0 .. 7); @@ -461,12 +461,12 @@ mod tests { #[test] fn test_parse_incremental_type_invariants() { - test("a #for x in array {x}", 18 .. 21, "[#x]", 2 .. 22); - test("a #let x = 1 {5}", 3 .. 6, "if", 2 .. 11); + test("a #for x in array {x}", 18 .. 21, "[#x]", 0 .. 22); + test("a #let x = 1 {5}", 3 .. 6, "if", 0 .. 11); test("a {let x = 1 {5}} b", 3 .. 6, "if", 2 .. 16); test("#let x = 1 {5}", 4 .. 4, " if", 0 .. 13); test("{let x = 1 {5}}", 4 .. 4, " if", 0 .. 18); - test("a // b c #f()", 3 .. 4, "", 2 .. 12); + test("a // b c #f()", 3 .. 4, "", 0 .. 12); test("{\nf()\n//g(a)\n}", 6 .. 8, "", 0 .. 12); test("a{\nf()\n//g(a)\n}b", 7 .. 9, "", 1 .. 13); test("a #while x {\n g(x) \n} b", 11 .. 11, "//", 0 .. 26); @@ -477,8 +477,8 @@ mod tests { #[test] fn test_parse_incremental_wrongly_or_unclosed_things() { test(r#"{"hi"}"#, 4 .. 5, "c", 0 .. 6); - test(r"this \u{abcd}", 8 .. 9, "", 5 .. 12); - test(r"this \u{abcd} that", 12 .. 13, "", 5 .. 17); + test(r"this \u{abcd}", 8 .. 9, "", 0 .. 12); + test(r"this \u{abcd} that", 12 .. 13, "", 0 .. 17); test(r"{{let x = z}; a = 1} b", 6 .. 6, "//", 0 .. 24); test("a b c", 1 .. 1, " /* letters */", 0 .. 19); test("a b c", 1 .. 1, " /* letters", 0 .. 16); diff --git a/src/parse/mod.rs b/src/parse/mod.rs index a8dee7f3..ed8bc5ce 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -162,11 +162,6 @@ fn markup(p: &mut Parser, mut at_start: bool) { }); } -/// Parse a single line of markup. -fn markup_line(p: &mut Parser) { - markup_indented(p, usize::MAX); -} - /// Parse markup that stays right of the given `column`. fn markup_indented(p: &mut Parser, min_indent: usize) { p.eat_while(|t| match t { @@ -185,7 +180,6 @@ fn markup_indented(p: &mut Parser, min_indent: usize) { { break; } - Some(NodeKind::Label(_)) => break, _ => {} } @@ -195,6 +189,33 @@ fn markup_indented(p: &mut Parser, min_indent: usize) { marker.end(p, NodeKind::Markup { min_indent }); } +/// Parse a line of markup that can prematurely end if `f` returns true. +fn markup_line<F>(p: &mut Parser, mut f: F) +where + F: FnMut(&NodeKind) -> bool, +{ + p.eat_while(|t| match t { + NodeKind::Space { newlines } => *newlines == 0, + NodeKind::LineComment | NodeKind::BlockComment => true, + _ => false, + }); + + p.perform(NodeKind::Markup { min_indent: usize::MAX }, |p| { + let mut at_start = false; + while let Some(kind) = p.peek() { + if let NodeKind::Space { newlines: (1 ..) } = kind { + break; + } + + if f(kind) { + break; + } + + markup_node(p, &mut at_start); + } + }); +} + /// Parse a markup node. fn markup_node(p: &mut Parser, at_start: &mut bool) { let token = match p.peek() { @@ -226,6 +247,7 @@ fn markup_node(p: &mut Parser, at_start: &mut bool) { | NodeKind::Ellipsis | NodeKind::Quote { .. } | NodeKind::Escape(_) + | NodeKind::Link(_) | NodeKind::Raw(_) | NodeKind::Math(_) | NodeKind::Label(_) @@ -233,12 +255,22 @@ fn markup_node(p: &mut Parser, at_start: &mut bool) { p.eat(); } - // Grouping markup. + // Strong, emph, heading. NodeKind::Star => strong(p), NodeKind::Underscore => emph(p), NodeKind::Eq => heading(p, *at_start), + + // Lists. NodeKind::Minus => list_node(p, *at_start), - NodeKind::EnumNumbering(_) => enum_node(p, *at_start), + NodeKind::Plus | NodeKind::EnumNumbering(_) => enum_node(p, *at_start), + NodeKind::Slash => { + desc_node(p, *at_start).ok(); + } + NodeKind::Colon => { + let marker = p.marker(); + p.eat(); + marker.convert(p, NodeKind::Text(':'.into())); + } // Hashtag + keyword / identifier. NodeKind::Ident(_) @@ -293,7 +325,7 @@ fn heading(p: &mut Parser, at_start: bool) { if at_start && p.peek().map_or(true, |kind| kind.is_space()) { p.eat_while(|kind| *kind == NodeKind::Space { newlines: 0 }); - markup_line(p); + markup_line(p, |kind| matches!(kind, NodeKind::Label(_))); marker.end(p, NodeKind::Heading); } else { let text = p.get(current_start .. p.prev_end()).into(); @@ -331,6 +363,25 @@ fn enum_node(p: &mut Parser, at_start: bool) { } } +/// Parse a single description list item. +fn desc_node(p: &mut Parser, at_start: bool) -> ParseResult { + let marker = p.marker(); + let text: EcoString = p.peek_src().into(); + p.eat(); + + let min_indent = p.column(p.prev_end()); + if at_start && p.eat_if(NodeKind::Space { newlines: 0 }) && !p.eof() { + markup_line(p, |node| matches!(node, NodeKind::Colon)); + p.expect(NodeKind::Colon)?; + markup_indented(p, min_indent); + marker.end(p, NodeKind::Desc); + } else { + marker.convert(p, NodeKind::Text(text)); + } + + Ok(()) +} + /// Parse an expression within a markup mode. fn markup_expr(p: &mut Parser) { // Does the expression need termination or can content follow directly? diff --git a/src/parse/resolve.rs b/src/parse/resolve.rs index bce2da3c..d68282c0 100644 --- a/src/parse/resolve.rs +++ b/src/parse/resolve.rs @@ -26,14 +26,12 @@ pub fn resolve_string(string: &str) -> EcoString { // TODO: Error if closing brace is missing. let sequence = s.eat_while(char::is_ascii_hexdigit); let _terminated = s.eat_if('}'); - match resolve_hex(sequence) { Some(c) => out.push(c), None => out.push_str(s.from(start)), } } - // TODO: Error for invalid escape sequence. _ => out.push_str(s.from(start)), } } diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs index 84a63ba1..f6d4b0e8 100644 --- a/src/parse/tokens.rs +++ b/src/parse/tokens.rs @@ -103,6 +103,11 @@ impl<'s> Iterator for Tokens<'s> { let start = self.s.cursor(); let c = self.s.eat()?; Some(match c { + // Comments. + '/' if self.s.eat_if('/') => self.line_comment(), + '/' if self.s.eat_if('*') => self.block_comment(), + '*' if self.s.eat_if('/') => NodeKind::Unknown("*/".into()), + // Blocks. '{' => NodeKind::LeftBrace, '}' => NodeKind::RightBrace, @@ -110,15 +115,7 @@ impl<'s> Iterator for Tokens<'s> { ']' => NodeKind::RightBracket, // Whitespace. - ' ' if self.s.done() || !self.s.at(char::is_whitespace) => { - NodeKind::Space { newlines: 0 } - } - c if c.is_whitespace() => self.whitespace(), - - // Comments with special case for URLs. - '/' if self.s.eat_if('*') => self.block_comment(), - '/' if !self.maybe_in_url() && self.s.eat_if('/') => self.line_comment(), - '*' if self.s.eat_if('/') => NodeKind::Unknown(self.s.from(start).into()), + c if c.is_whitespace() => self.whitespace(c), // Other things. _ => match self.mode { @@ -130,85 +127,110 @@ impl<'s> Iterator for Tokens<'s> { } impl<'s> Tokens<'s> { + fn line_comment(&mut self) -> NodeKind { + self.s.eat_until(is_newline); + if self.s.peek().is_none() { + self.terminated = false; + } + NodeKind::LineComment + } + + fn block_comment(&mut self) -> NodeKind { + let mut state = '_'; + let mut depth = 1; + self.terminated = false; + + // Find the first `*/` that does not correspond to a nested `/*`. + while let Some(c) = self.s.eat() { + state = match (state, c) { + ('*', '/') => { + depth -= 1; + if depth == 0 { + self.terminated = true; + break; + } + '_' + } + ('/', '*') => { + depth += 1; + '_' + } + ('/', '/') => { + self.line_comment(); + '_' + } + _ => c, + } + } + + NodeKind::BlockComment + } + + fn whitespace(&mut self, c: char) -> NodeKind { + if c == ' ' && !self.s.at(char::is_whitespace) { + return NodeKind::Space { newlines: 0 }; + } + + self.s.uneat(); + + // Count the number of newlines. + let mut newlines = 0; + while let Some(c) = self.s.eat() { + if !c.is_whitespace() { + self.s.uneat(); + break; + } + + if is_newline(c) { + if c == '\r' { + self.s.eat_if('\n'); + } + newlines += 1; + } + } + + NodeKind::Space { newlines } + } + #[inline] fn markup(&mut self, start: usize, c: char) -> NodeKind { match c { // Escape sequences. '\\' => self.backslash(), - // Keywords and identifiers. - '#' => self.hash(), - - // Markup. + // Single-char things. '~' => NodeKind::NonBreakingSpace, - '-' => self.hyph(), '.' if self.s.eat_if("..") => NodeKind::Ellipsis, '\'' => NodeKind::Quote { double: false }, '"' => NodeKind::Quote { double: true }, '*' if !self.in_word() => NodeKind::Star, '_' if !self.in_word() => NodeKind::Underscore, - '`' => self.raw(), '=' => NodeKind::Eq, - '$' => self.math(), - '<' => self.label(), - '@' => self.reference(), - c if c == '.' || c.is_ascii_digit() => self.numbering(start, c), - - // Plain text. - _ => self.text(start), - } - } - - fn code(&mut self, start: usize, c: char) -> NodeKind { - match c { - // Parens. - '(' => NodeKind::LeftParen, - ')' => NodeKind::RightParen, - - // Length two. - '=' if self.s.eat_if('=') => NodeKind::EqEq, - '!' if self.s.eat_if('=') => NodeKind::ExclEq, - '<' if self.s.eat_if('=') => NodeKind::LtEq, - '>' if self.s.eat_if('=') => NodeKind::GtEq, - '+' if self.s.eat_if('=') => NodeKind::PlusEq, - '-' if self.s.eat_if('=') => NodeKind::HyphEq, - '*' if self.s.eat_if('=') => NodeKind::StarEq, - '/' if self.s.eat_if('=') => NodeKind::SlashEq, - '.' if self.s.eat_if('.') => NodeKind::Dots, - '=' if self.s.eat_if('>') => NodeKind::Arrow, - - // Length one. - ',' => NodeKind::Comma, - ';' => NodeKind::Semicolon, - ':' => NodeKind::Colon, '+' => NodeKind::Plus, - '-' => NodeKind::Minus, - '*' => NodeKind::Star, '/' => NodeKind::Slash, - '=' => NodeKind::Eq, - '<' => NodeKind::Lt, - '>' => NodeKind::Gt, - '.' if self.s.done() || !self.s.at(char::is_ascii_digit) => NodeKind::Dot, - - // Identifiers. - c if is_id_start(c) => self.ident(start), + ':' => NodeKind::Colon, - // Numbers. - c if c.is_ascii_digit() || (c == '.' && self.s.at(char::is_ascii_digit)) => { - self.number(start, c) + // Multi-char things. + '#' => self.hash(start), + '-' => self.hyph(), + 'h' if self.s.eat_if("ttp://") || self.s.eat_if("ttps://") => { + self.link(start) } + '`' => self.raw(), + '$' => self.math(), + c if c.is_ascii_digit() => self.numbering(start), + '<' => self.label(), + '@' => self.reference(start), - // Strings. - '"' => self.string(), - - _ => NodeKind::Unknown(self.s.from(start).into()), + // Plain text. + _ => self.text(start), } } #[inline] fn text(&mut self, start: usize) -> NodeKind { macro_rules! table { - ($($c:literal)|*) => {{ + ($(|$c:literal)*) => {{ let mut t = [false; 128]; $(t[$c as usize] = true;)* t @@ -216,12 +238,9 @@ impl<'s> Tokens<'s> { } const TABLE: [bool; 128] = table! { - // Ascii whitespace. - ' ' | '\t' | '\n' | '\x0b' | '\x0c' | '\r' | - // Comments, parentheses, code. - '/' | '[' | ']' | '{' | '}' | '#' | - // Markup - '~' | '-' | '.' | '\'' | '"' | '*' | '_' | '`' | '$' | '\\' + | ' ' | '\t' | '\n' | '\x0b' | '\x0c' | '\r' | '\\' | '/' + | '[' | ']' | '{' | '}' | '~' | '-' | '.' | '\'' | '"' + | '*' | '_' | ':' | 'h' | '`' | '$' | '<' | '>' | '@' | '#' }; loop { @@ -229,14 +248,17 @@ impl<'s> Tokens<'s> { TABLE.get(c as usize).copied().unwrap_or_else(|| c.is_whitespace()) }); - // Allow a single space, optionally preceded by . or - if something - // alphanumeric follows directly. This leads to less text nodes, - // which is good for performance. + // Continue with the same text node if the thing would become text + // anyway. let mut s = self.s; - s.eat_if(['.', '-']); - s.eat_if(' '); - if !s.at(char::is_alphanumeric) { - break; + match s.eat() { + Some('/') if !s.at(['/', '*']) => {} + Some(' ') if s.at(char::is_alphanumeric) => {} + Some('-') if !s.at(['-', '?']) => {} + Some('.') if !s.at("..") => {} + Some('h') if !s.at("ttp://") && !s.at("ttps://") => {} + Some('@' | '#') if !s.at(is_id_start) => {} + _ => break, } self.s = s; @@ -245,47 +267,9 @@ impl<'s> Tokens<'s> { NodeKind::Text(self.s.from(start).into()) } - fn whitespace(&mut self) -> NodeKind { - self.s.uneat(); - - // Count the number of newlines. - let mut newlines = 0; - while let Some(c) = self.s.eat() { - if !c.is_whitespace() { - self.s.uneat(); - break; - } - - if is_newline(c) { - if c == '\r' { - self.s.eat_if('\n'); - } - newlines += 1; - } - } - - NodeKind::Space { newlines } - } - fn backslash(&mut self) -> NodeKind { - let c = match self.s.peek() { - Some(c) => c, - None => return NodeKind::Linebreak { justified: false }, - }; - - match c { - // Backslash and comments. - '\\' | '/' | - // Parenthesis and hashtag. - '[' | ']' | '{' | '}' | '#' | - // Markup. - '~' | '-' | '.' | ':' | - '\'' | '"' | '*' | '_' | '`' | '$' | '=' | - '<' | '>' | '@' => { - self.s.expect(c); - NodeKind::Escape(c) - } - 'u' if self.s.eat_if("u{") => { + match self.s.peek() { + Some('u') if self.s.eat_if("u{") => { let sequence = self.s.eat_while(char::is_ascii_alphanumeric); if self.s.eat_if('}') { if let Some(c) = resolve_hex(sequence) { @@ -298,26 +282,23 @@ impl<'s> Tokens<'s> { } } else { self.terminated = false; - NodeKind::Error( - SpanPos::End, - "expected closing brace".into(), - ) + NodeKind::Error(SpanPos::End, "expected closing brace".into()) } } // Linebreaks. - c if c.is_whitespace() => NodeKind::Linebreak { justified: false }, - '+' => { + Some(c) if c.is_whitespace() => NodeKind::Linebreak, + None => NodeKind::Linebreak, + + // Escapes. + Some(c) => { self.s.expect(c); - NodeKind::Linebreak { justified: true } + NodeKind::Escape(c) } - - // Just the backslash. - _ => NodeKind::Text('\\'.into()), } } - fn hash(&mut self) -> NodeKind { + fn hash(&mut self, start: usize) -> NodeKind { if self.s.at(is_id_start) { let read = self.s.eat_while(is_id_continue); match keyword(read) { @@ -325,7 +306,7 @@ impl<'s> Tokens<'s> { None => NodeKind::Ident(read.into()), } } else { - NodeKind::Text('#'.into()) + self.text(start) } } @@ -343,19 +324,26 @@ impl<'s> Tokens<'s> { } } - fn numbering(&mut self, start: usize, c: char) -> NodeKind { - let number = if c != '.' { - self.s.eat_while(char::is_ascii_digit); - let read = self.s.from(start); - if !self.s.eat_if('.') { - return NodeKind::Text(self.s.from(start).into()); - } - read.parse().ok() - } else { - None - }; + fn in_word(&self) -> bool { + let alphanumeric = |c: Option<char>| c.map_or(false, |c| c.is_alphanumeric()); + let prev = self.s.scout(-2); + let next = self.s.peek(); + alphanumeric(prev) && alphanumeric(next) + } - NodeKind::EnumNumbering(number) + fn link(&mut self, start: usize) -> NodeKind { + #[rustfmt::skip] + self.s.eat_while(|c: char| matches!(c, + | '0' ..= '9' + | 'a' ..= 'z' + | 'A' ..= 'Z' + | '~' | '/' | '%' | '?' | '#' | '&' | '+' | '=' + | '\'' | '.' | ',' | ';' + )); + if self.s.scout(-1) == Some('.') { + self.s.uneat(); + } + NodeKind::Link(self.s.from(start).into()) } fn raw(&mut self) -> NodeKind { @@ -376,7 +364,6 @@ impl<'s> Tokens<'s> { } let start = self.s.cursor(); - let mut found = 0; while found < backticks { match self.s.eat() { @@ -394,10 +381,9 @@ impl<'s> Tokens<'s> { self.s.get(start .. end), ))) } else { + self.terminated = false; let remaining = backticks - found; let noun = if remaining == 1 { "backtick" } else { "backticks" }; - - self.terminated = false; NodeKind::Error( SpanPos::End, if found == 0 { @@ -410,51 +396,38 @@ impl<'s> Tokens<'s> { } fn math(&mut self) -> NodeKind { - let mut display = false; - if self.s.eat_if('[') { - display = true; - } - - let start = self.s.cursor(); - let mut escaped = false; - let mut dollar = !display; - - let terminated = loop { - match self.s.eat() { - Some('$') if !escaped && dollar => break true, - Some(']') if !escaped => dollar = true, - Some(c) => { - dollar = !display; - escaped = c == '\\' && !escaped; - } - None => break false, + let formula = self.s.eat_until(|c| { + if c == '$' && !escaped { + true + } else { + escaped = c == '\\' && !escaped; + false } - }; + }); + + let display = formula.len() >= 2 + && formula.starts_with(char::is_whitespace) + && formula.ends_with(char::is_whitespace); - let end = self.s.cursor() - - match (terminated, display) { - (false, _) => 0, - (true, false) => 1, - (true, true) => 2, - }; - - if terminated { - NodeKind::Math(Arc::new(MathNode { - formula: self.s.get(start .. end).into(), - display, - })) + if self.s.eat_if('$') { + NodeKind::Math(Arc::new(MathNode { formula: formula.into(), display })) } else { self.terminated = false; - NodeKind::Error( - SpanPos::End, - if !display || (!escaped && dollar) { - "expected closing dollar sign".into() - } else { - "expected closing bracket and dollar sign".into() - }, - ) + NodeKind::Error(SpanPos::End, "expected dollar sign".into()) + } + } + + fn numbering(&mut self, start: usize) -> NodeKind { + self.s.eat_while(char::is_ascii_digit); + let read = self.s.from(start); + if self.s.eat_if('.') { + if let Ok(number) = read.parse() { + return NodeKind::EnumNumbering(number); + } } + + self.text(start) } fn label(&mut self) -> NodeKind { @@ -471,12 +444,59 @@ impl<'s> Tokens<'s> { } } - fn reference(&mut self) -> NodeKind { + fn reference(&mut self, start: usize) -> NodeKind { let label = self.s.eat_while(is_id_continue); if !label.is_empty() { NodeKind::Ref(label.into()) } else { - NodeKind::Error(SpanPos::Full, "label cannot be empty".into()) + self.text(start) + } + } + + fn code(&mut self, start: usize, c: char) -> NodeKind { + match c { + // Parentheses. + '(' => NodeKind::LeftParen, + ')' => NodeKind::RightParen, + + // Two-char operators. + '=' if self.s.eat_if('=') => NodeKind::EqEq, + '!' if self.s.eat_if('=') => NodeKind::ExclEq, + '<' if self.s.eat_if('=') => NodeKind::LtEq, + '>' if self.s.eat_if('=') => NodeKind::GtEq, + '+' if self.s.eat_if('=') => NodeKind::PlusEq, + '-' if self.s.eat_if('=') => NodeKind::HyphEq, + '*' if self.s.eat_if('=') => NodeKind::StarEq, + '/' if self.s.eat_if('=') => NodeKind::SlashEq, + '.' if self.s.eat_if('.') => NodeKind::Dots, + '=' if self.s.eat_if('>') => NodeKind::Arrow, + + // Single-char operators. + ',' => NodeKind::Comma, + ';' => NodeKind::Semicolon, + ':' => NodeKind::Colon, + '+' => NodeKind::Plus, + '-' => NodeKind::Minus, + '*' => NodeKind::Star, + '/' => NodeKind::Slash, + '=' => NodeKind::Eq, + '<' => NodeKind::Lt, + '>' => NodeKind::Gt, + '.' if !self.s.at(char::is_ascii_digit) => NodeKind::Dot, + + // Identifiers. + c if is_id_start(c) => self.ident(start), + + // Numbers. + c if c.is_ascii_digit() || (c == '.' && self.s.at(char::is_ascii_digit)) => { + self.number(start, c) + } + + // Strings. + '"' => self.string(), + + // Invalid token. + _ => NodeKind::Unknown(self.s.from(start).into()), } } @@ -543,18 +563,18 @@ impl<'s> Tokens<'s> { } } - fn string(&mut self) -> NodeKind { let mut escaped = false; - let string = resolve_string(self.s.eat_until(|c| { + let verbatim = self.s.eat_until(|c| { if c == '"' && !escaped { true } else { escaped = c == '\\' && !escaped; false } - })); + }); + let string = resolve_string(verbatim); if self.s.eat_if('"') { NodeKind::Str(string) } else { @@ -562,56 +582,6 @@ impl<'s> Tokens<'s> { NodeKind::Error(SpanPos::End, "expected quote".into()) } } - - fn line_comment(&mut self) -> NodeKind { - self.s.eat_until(is_newline); - if self.s.peek().is_none() { - self.terminated = false; - } - NodeKind::LineComment - } - - fn block_comment(&mut self) -> NodeKind { - let mut state = '_'; - let mut depth = 1; - self.terminated = false; - - // Find the first `*/` that does not correspond to a nested `/*`. - while let Some(c) = self.s.eat() { - state = match (state, c) { - ('*', '/') => { - depth -= 1; - if depth == 0 { - self.terminated = true; - break; - } - '_' - } - ('/', '*') => { - depth += 1; - '_' - } - ('/', '/') => { - self.line_comment(); - '_' - } - _ => c, - } - } - - NodeKind::BlockComment - } - - fn in_word(&self) -> bool { - let alphanumeric = |c: Option<char>| c.map_or(false, |c| c.is_alphanumeric()); - let prev = self.s.scout(-2); - let next = self.s.peek(); - alphanumeric(prev) && alphanumeric(next) - } - - fn maybe_in_url(&self) -> bool { - self.mode == TokenMode::Markup && self.s.before().ends_with(":/") - } } fn keyword(ident: &str) -> Option<NodeKind> { @@ -872,14 +842,14 @@ mod tests { #[test] fn test_tokenize_text() { // Test basic text. - t!(Markup[" /"]: "hello" => Text("hello")); - t!(Markup[" /"]: "hello-world" => Text("hello-world")); + t!(Markup[" /"]: "hello" => Text("hello")); + t!(Markup[" /"]: "reha-world" => Text("reha-world")); // Test code symbols in text. - t!(Markup[" /"]: "a():\"b" => Text("a():"), Quote { double: true }, Text("b")); - t!(Markup[" /"]: ";:,|/+" => Text(";:,|"), Text("/+")); + t!(Markup[" /"]: "a():\"b" => Text("a()"), Colon, Quote { double: true }, Text("b")); + t!(Markup[" /"]: ";,|/+" => Text(";,|/+")); t!(Markup[" /"]: "=-a" => Eq, Minus, Text("a")); - t!(Markup[" "]: "#123" => Text("#"), Text("123")); + t!(Markup[" "]: "#123" => Text("#123")); // Test text ends. t!(Markup[""]: "hello " => Text("hello"), Space(0)); @@ -904,11 +874,9 @@ mod tests { t!(Markup: r"\`" => Escape('`')); t!(Markup: r"\$" => Escape('$')); t!(Markup: r"\#" => Escape('#')); - - // Test unescapable symbols. - t!(Markup[" /"]: r"\a" => Text(r"\"), Text("a")); - t!(Markup[" /"]: r"\u" => Text(r"\"), Text("u")); - t!(Markup[" /"]: r"\1" => Text(r"\"), Text("1")); + t!(Markup: r"\a" => Escape('a')); + t!(Markup: r"\u" => Escape('u')); + t!(Markup: r"\1" => Escape('1')); // Test basic unicode escapes. t!(Markup: r"\u{}" => Error(Full, "invalid unicode escape sequence")); @@ -930,16 +898,15 @@ mod tests { t!(Markup: "_" => Underscore); t!(Markup[""]: "===" => Eq, Eq, Eq); t!(Markup["a1/"]: "= " => Eq, Space(0)); - t!(Markup[" "]: r"\" => Linebreak { justified: false }); - t!(Markup[" "]: r"\+" => Linebreak { justified: true }); + t!(Markup[" "]: r"\" => Linebreak); t!(Markup: "~" => NonBreakingSpace); t!(Markup["a1/"]: "-?" => Shy); t!(Markup["a "]: r"a--" => Text("a"), EnDash); t!(Markup["a1/"]: "- " => Minus, Space(0)); - t!(Markup[" "]: "." => EnumNumbering(None)); - t!(Markup[" "]: "1." => EnumNumbering(Some(1))); - t!(Markup[" "]: "1.a" => EnumNumbering(Some(1)), Text("a")); - t!(Markup[" /"]: "a1." => Text("a1"), EnumNumbering(None)); + t!(Markup[" "]: "+" => Plus); + t!(Markup[" "]: "1." => EnumNumbering(1)); + t!(Markup[" "]: "1.a" => EnumNumbering(1), Text("a")); + t!(Markup[" /"]: "a1." => Text("a1.")); } #[test] @@ -995,7 +962,7 @@ mod tests { for (s, t) in list.clone() { t!(Markup[" "]: format!("#{}", s) => t); t!(Markup[" "]: format!("#{0}#{0}", s) => t, t); - t!(Markup[" /"]: format!("# {}", s) => Text("#"), Space(0), Text(s)); + t!(Markup[" /"]: format!("# {}", s) => Text(&format!("# {s}"))); } for (s, t) in list { @@ -1037,18 +1004,16 @@ mod tests { t!(Markup: "$$" => Math("", false)); t!(Markup: "$x$" => Math("x", false)); t!(Markup: r"$\\$" => Math(r"\\", false)); - t!(Markup: "$[x + y]$" => Math("x + y", true)); - t!(Markup: r"$[\\]$" => Math(r"\\", true)); + t!(Markup: r"$[\\]$" => Math(r"[\\]", false)); + t!(Markup: "$ x + y $" => Math(" x + y ", true)); // Test unterminated. - t!(Markup[""]: "$x" => Error(End, "expected closing dollar sign")); - t!(Markup[""]: "$[x" => Error(End, "expected closing bracket and dollar sign")); - t!(Markup[""]: "$[x]\n$" => Error(End, "expected closing bracket and dollar sign")); + t!(Markup[""]: "$x" => Error(End, "expected dollar sign")); + t!(Markup[""]: "$[x]\n" => Error(End, "expected dollar sign")); // Test escape sequences. - t!(Markup: r"$\$x$" => Math(r"\$x", false)); - t!(Markup: r"$[\\\]$]$" => Math(r"\\\]$", true)); - t!(Markup[""]: r"$[ ]\\$" => Error(End, "expected closing bracket and dollar sign")); + t!(Markup: r"$\$x$" => Math(r"\$x", false)); + t!(Markup: r"$\ \$ $" => Math(r"\ \$ ", false)); } #[test] diff --git a/src/syntax/ast.rs b/src/syntax/ast.rs index 10bee4e8..8d3696a8 100644 --- a/src/syntax/ast.rs +++ b/src/syntax/ast.rs @@ -63,9 +63,7 @@ impl Markup { self.0.children().filter_map(|node| match node.kind() { NodeKind::Space { newlines: (2 ..) } => Some(MarkupNode::Parbreak), NodeKind::Space { .. } => Some(MarkupNode::Space), - &NodeKind::Linebreak { justified } => { - Some(MarkupNode::Linebreak { justified }) - } + NodeKind::Linebreak => Some(MarkupNode::Linebreak), NodeKind::Text(s) => Some(MarkupNode::Text(s.clone())), NodeKind::Escape(c) => Some(MarkupNode::Text((*c).into())), NodeKind::NonBreakingSpace => Some(MarkupNode::Text('\u{00A0}'.into())), @@ -76,6 +74,7 @@ impl Markup { &NodeKind::Quote { double } => Some(MarkupNode::Quote { double }), NodeKind::Strong => node.cast().map(MarkupNode::Strong), NodeKind::Emph => node.cast().map(MarkupNode::Emph), + NodeKind::Link(url) => Some(MarkupNode::Link(url.clone())), NodeKind::Raw(raw) => Some(MarkupNode::Raw(raw.as_ref().clone())), NodeKind::Math(math) => Some(MarkupNode::Math(Spanned::new( math.as_ref().clone(), @@ -84,6 +83,7 @@ impl Markup { NodeKind::Heading => node.cast().map(MarkupNode::Heading), NodeKind::List => node.cast().map(MarkupNode::List), NodeKind::Enum => node.cast().map(MarkupNode::Enum), + NodeKind::Desc => node.cast().map(MarkupNode::Desc), NodeKind::Label(v) => Some(MarkupNode::Label(v.clone())), NodeKind::Ref(v) => Some(MarkupNode::Ref(v.clone())), _ => node.cast().map(MarkupNode::Expr), @@ -96,8 +96,8 @@ impl Markup { pub enum MarkupNode { /// Whitespace containing less than two newlines. Space, - /// A forced line break: `\` or `\+` if justified. - Linebreak { justified: bool }, + /// A forced line break. + Linebreak, /// A paragraph break: Two or more newlines. Parbreak, /// Plain text. @@ -108,6 +108,8 @@ pub enum MarkupNode { Strong(StrongNode), /// Emphasized content: `_Emphasized_`. Emph(EmphNode), + /// A hyperlink. + Link(EcoString), /// A raw block with optional syntax highlighting: `` `...` ``. Raw(RawNode), /// A math formula: `$a^2 = b^2 + c^2$`. @@ -116,8 +118,10 @@ pub enum MarkupNode { Heading(HeadingNode), /// An item in an unordered list: `- ...`. List(ListNode), - /// An item in an enumeration (ordered list): `1. ...`. + /// An item in an enumeration (ordered list): `+ ...` or `1. ...`. Enum(EnumNode), + /// An item in a description list: `/ Term: Details. + Desc(DescNode), /// A label. Label(EcoString), /// A reference. @@ -170,8 +174,8 @@ pub struct RawNode { pub struct MathNode { /// The formula between the dollars / brackets. pub formula: EcoString, - /// Whether the formula is display-level, that is, it is surrounded by - /// `$[..]$`. + /// Whether the formula is display-level, that is, it contains whitespace + /// after the starting dollar sign and before the ending dollar sign. pub display: bool, } @@ -205,7 +209,7 @@ node! { impl ListNode { /// The contents of the list item. pub fn body(&self) -> Markup { - self.0.cast_first_child().expect("list node is missing body") + self.0.cast_first_child().expect("list item is missing body") } } @@ -217,18 +221,36 @@ node! { impl EnumNode { /// The contents of the list item. pub fn body(&self) -> Markup { - self.0.cast_first_child().expect("enum node is missing body") + self.0.cast_first_child().expect("enum item is missing body") } /// The number, if any. pub fn number(&self) -> Option<usize> { + self.0.children().find_map(|node| match node.kind() { + NodeKind::EnumNumbering(num) => Some(*num), + _ => None, + }) + } +} + +node! { + /// An item in a description list: `/ Term: Details. + DescNode: Desc +} + +impl DescNode { + /// The term described by the list item. + pub fn term(&self) -> Markup { self.0 - .children() - .find_map(|node| match node.kind() { - NodeKind::EnumNumbering(num) => Some(*num), - _ => None, - }) - .expect("enum node is missing number") + .cast_first_child() + .expect("description list item is missing term") + } + + /// The description of the term. + pub fn body(&self) -> Markup { + self.0 + .cast_last_child() + .expect("description list item is missing body") } } diff --git a/src/syntax/highlight.rs b/src/syntax/highlight.rs index 7f5ee083..de7c70a2 100644 --- a/src/syntax/highlight.rs +++ b/src/syntax/highlight.rs @@ -147,12 +147,12 @@ pub fn highlight_pre(text: &str, mode: TokenMode, theme: &Theme) -> String { /// The syntax highlighting category of a node. #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] pub enum Category { + /// A line or block comment. + Comment, /// Any kind of bracket, parenthesis or brace. Bracket, /// Punctuation in code. Punctuation, - /// A line or block comment. - Comment, /// An easily typable shortcut to a unicode codepoint. Shortcut, /// An escape sequence. @@ -161,14 +161,18 @@ pub enum Category { Strong, /// Emphasized text. Emph, + /// A hyperlink. + Link, /// Raw text or code. Raw, /// A math formula. Math, /// A section heading. Heading, - /// A list or enumeration. + /// A symbol of a list, enumeration, or description list. List, + /// A term in a description list. + Term, /// A label. Label, /// A reference. @@ -204,66 +208,74 @@ impl Category { i: usize, ) -> Option<Category> { match child.kind() { + NodeKind::LineComment => Some(Category::Comment), + NodeKind::BlockComment => Some(Category::Comment), NodeKind::LeftBrace => Some(Category::Bracket), NodeKind::RightBrace => Some(Category::Bracket), NodeKind::LeftBracket => Some(Category::Bracket), NodeKind::RightBracket => Some(Category::Bracket), NodeKind::LeftParen => Some(Category::Bracket), NodeKind::RightParen => Some(Category::Bracket), - NodeKind::Comma => Some(Category::Punctuation), - NodeKind::Semicolon => Some(Category::Punctuation), - NodeKind::Colon => Some(Category::Punctuation), - NodeKind::Dot => Some(Category::Punctuation), - NodeKind::LineComment => Some(Category::Comment), - NodeKind::BlockComment => Some(Category::Comment), + + NodeKind::Markup { .. } => match parent.kind() { + NodeKind::Desc + if parent + .children() + .take_while(|child| child.kind() != &NodeKind::Colon) + .find(|c| matches!(c.kind(), NodeKind::Markup { .. })) + .map_or(false, |ident| std::ptr::eq(ident, child)) => + { + Some(Category::Term) + } + _ => None, + }, + NodeKind::Space { .. } => None, NodeKind::Linebreak { .. } => Some(Category::Shortcut), + NodeKind::Text(_) => None, + NodeKind::Escape(_) => Some(Category::Escape), NodeKind::NonBreakingSpace => Some(Category::Shortcut), NodeKind::Shy => Some(Category::Shortcut), NodeKind::EnDash => Some(Category::Shortcut), NodeKind::EmDash => Some(Category::Shortcut), NodeKind::Ellipsis => Some(Category::Shortcut), - NodeKind::Escape(_) => Some(Category::Escape), + NodeKind::Quote { .. } => None, + NodeKind::Star => match parent.kind() { + NodeKind::Strong => None, + _ => Some(Category::Operator), + }, + NodeKind::Underscore => None, NodeKind::Strong => Some(Category::Strong), NodeKind::Emph => Some(Category::Emph), + NodeKind::Link(_) => Some(Category::Link), NodeKind::Raw(_) => Some(Category::Raw), NodeKind::Math(_) => Some(Category::Math), NodeKind::Heading => Some(Category::Heading), + NodeKind::List => None, + NodeKind::Enum => None, + NodeKind::EnumNumbering(_) => Some(Category::List), + NodeKind::Desc => None, + NodeKind::Label(_) => Some(Category::Label), + NodeKind::Ref(_) => Some(Category::Ref), + + NodeKind::Comma => Some(Category::Punctuation), + NodeKind::Semicolon => Some(Category::Punctuation), + NodeKind::Colon => match parent.kind() { + NodeKind::Desc => Some(Category::Term), + _ => Some(Category::Punctuation), + }, + NodeKind::Plus => match parent.kind() { + NodeKind::Enum => Some(Category::List), + _ => Some(Category::Operator), + }, NodeKind::Minus => match parent.kind() { NodeKind::List => Some(Category::List), _ => Some(Category::Operator), }, - NodeKind::EnumNumbering(_) => Some(Category::List), - NodeKind::Label(_) => Some(Category::Label), - NodeKind::Ref(_) => Some(Category::Ref), - NodeKind::Not => Some(Category::Keyword), - NodeKind::And => Some(Category::Keyword), - NodeKind::Or => Some(Category::Keyword), - NodeKind::Let => Some(Category::Keyword), - NodeKind::Set => Some(Category::Keyword), - NodeKind::Show => Some(Category::Keyword), - NodeKind::Wrap => Some(Category::Keyword), - NodeKind::If => Some(Category::Keyword), - NodeKind::Else => Some(Category::Keyword), - NodeKind::While => Some(Category::Keyword), - NodeKind::For => Some(Category::Keyword), - NodeKind::In => Some(Category::Keyword), - NodeKind::As => Some(Category::Keyword), - NodeKind::Break => Some(Category::Keyword), - NodeKind::Continue => Some(Category::Keyword), - NodeKind::Return => Some(Category::Keyword), - NodeKind::Import => Some(Category::Keyword), - NodeKind::From => Some(Category::Keyword), - NodeKind::Include => Some(Category::Keyword), - NodeKind::Plus => Some(Category::Operator), - NodeKind::Star => match parent.kind() { - NodeKind::Strong => None, + NodeKind::Slash => match parent.kind() { + NodeKind::Desc => Some(Category::List), _ => Some(Category::Operator), }, - NodeKind::Slash => Some(Category::Operator), - NodeKind::PlusEq => Some(Category::Operator), - NodeKind::HyphEq => Some(Category::Operator), - NodeKind::StarEq => Some(Category::Operator), - NodeKind::SlashEq => Some(Category::Operator), + NodeKind::Dot => Some(Category::Punctuation), NodeKind::Eq => match parent.kind() { NodeKind::Heading => None, _ => Some(Category::Operator), @@ -274,10 +286,34 @@ impl Category { NodeKind::LtEq => Some(Category::Operator), NodeKind::Gt => Some(Category::Operator), NodeKind::GtEq => Some(Category::Operator), + NodeKind::PlusEq => Some(Category::Operator), + NodeKind::HyphEq => Some(Category::Operator), + NodeKind::StarEq => Some(Category::Operator), + NodeKind::SlashEq => Some(Category::Operator), NodeKind::Dots => Some(Category::Operator), NodeKind::Arrow => Some(Category::Operator), + NodeKind::Not => Some(Category::Keyword), + NodeKind::And => Some(Category::Keyword), + NodeKind::Or => Some(Category::Keyword), NodeKind::None => Some(Category::None), NodeKind::Auto => Some(Category::Auto), + NodeKind::Let => Some(Category::Keyword), + NodeKind::Set => Some(Category::Keyword), + NodeKind::Show => Some(Category::Keyword), + NodeKind::Wrap => Some(Category::Keyword), + NodeKind::If => Some(Category::Keyword), + NodeKind::Else => Some(Category::Keyword), + NodeKind::For => Some(Category::Keyword), + NodeKind::In => Some(Category::Keyword), + NodeKind::While => Some(Category::Keyword), + NodeKind::Break => Some(Category::Keyword), + NodeKind::Continue => Some(Category::Keyword), + NodeKind::Return => Some(Category::Keyword), + NodeKind::Import => Some(Category::Keyword), + NodeKind::Include => Some(Category::Keyword), + NodeKind::From => Some(Category::Keyword), + NodeKind::As => Some(Category::Keyword), + NodeKind::Ident(_) => match parent.kind() { NodeKind::Markup { .. } => Some(Category::Interpolated), NodeKind::FuncCall => Some(Category::Function), @@ -302,15 +338,6 @@ impl Category { NodeKind::Float(_) => Some(Category::Number), NodeKind::Numeric(_, _) => Some(Category::Number), NodeKind::Str(_) => Some(Category::String), - NodeKind::Error(_, _) => Some(Category::Invalid), - NodeKind::Unknown(_) => Some(Category::Invalid), - NodeKind::Underscore => None, - NodeKind::Markup { .. } => None, - NodeKind::Space { .. } => None, - NodeKind::Text(_) => None, - NodeKind::Quote { .. } => None, - NodeKind::List => None, - NodeKind::Enum => None, NodeKind::CodeBlock => None, NodeKind::ContentBlock => None, NodeKind::GroupExpr => None, @@ -341,6 +368,9 @@ impl Category { NodeKind::BreakExpr => None, NodeKind::ContinueExpr => None, NodeKind::ReturnExpr => None, + + NodeKind::Error(_, _) => Some(Category::Invalid), + NodeKind::Unknown(_) => Some(Category::Invalid), } } @@ -354,10 +384,12 @@ impl Category { Self::Escape => "constant.character.escape.content.typst", Self::Strong => "markup.bold.typst", Self::Emph => "markup.italic.typst", + Self::Link => "markup.underline.link.typst", Self::Raw => "markup.raw.typst", Self::Math => "string.other.math.typst", Self::Heading => "markup.heading.typst", Self::List => "markup.list.typst", + Self::Term => "markup.bold.typst", Self::Label => "entity.name.label.typst", Self::Ref => "markup.other.reference.typst", Self::Keyword => "keyword.typst", diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs index 89937f2c..6c6f690c 100644 --- a/src/syntax/mod.rs +++ b/src/syntax/mod.rs @@ -571,6 +571,14 @@ impl PartialEq for NodeData { /// the parser. #[derive(Debug, Clone, PartialEq)] pub enum NodeKind { + /// A line comment, two slashes followed by inner contents, terminated with + /// a newline: `//<str>\n`. + LineComment, + /// A block comment, a slash and a star followed by inner contents, + /// terminated with a star and a slash: `/*<str>*/`. + /// + /// The comment can contain nested block comments. + BlockComment, /// A left curly brace, starting a code block: `{`. LeftBrace, /// A right curly brace, terminating a code block: `}`. @@ -585,23 +593,83 @@ pub enum NodeKind { /// A right round parenthesis, terminating a grouped expression, collection, /// argument or parameter list: `)`. RightParen, + + /// Markup of which all lines must have a minimal indentation. + /// + /// Notably, the number does not determine in which column the markup + /// started, but to the right of which column all markup elements must be, + /// so it is zero except for headings and lists. + Markup { min_indent: usize }, + /// One or more whitespace characters. Single spaces are collapsed into text + /// nodes if they would otherwise be surrounded by text nodes. + /// + /// Also stores how many newlines are contained. + Space { newlines: usize }, + /// A forced line break. + Linebreak, + /// Consecutive text without markup. While basic text with just single + /// spaces is collapsed into a single node, certain symbols that could + /// possibly be markup force text into multiple nodes. + Text(EcoString), + /// A slash and the letter "u" followed by a hexadecimal unicode entity + /// enclosed in curly braces: `\u{1F5FA}`. + Escape(char), + /// A non-breaking space: `~`. + NonBreakingSpace, + /// A soft hyphen: `-?`. + Shy, + /// An en-dash: `--`. + EnDash, + /// An em-dash: `---`. + EmDash, + /// An ellipsis: `...`. + Ellipsis, + /// A smart quote: `'` or `"`. + Quote { double: bool }, /// The strong text toggle, multiplication operator, and wildcard import /// symbol: `*`. Star, /// Toggles emphasized text: `_`. Underscore, + /// Strong content: `*Strong*`. + Strong, + /// Emphasized content: `_Emphasized_`. + Emph, + /// A hyperlink. + Link(EcoString), + /// A raw block with optional syntax highlighting: `` `...` ``. + Raw(Arc<RawNode>), + /// A math formula: `$x$`, `$[x^2]$`. + Math(Arc<MathNode>), + /// A section heading: `= Introduction`. + Heading, + /// An item in an unordered list: `- ...`. + List, + /// An item in an enumeration (ordered list): `+ ...` or `1. ...`. + Enum, + /// An explicit enumeration numbering: `23.`. + EnumNumbering(usize), + /// An item in a description list: `/ Term: Details. + Desc, + /// A label: `<label>`. + Label(EcoString), + /// A reference: `@label`. + Ref(EcoString), + /// A comma separator in a sequence: `,`. Comma, /// A semicolon terminating an expression: `;`. Semicolon, /// A colon between name / key and value in a dictionary, argument or - /// parameter list: `:`. + /// parameter list, or between the term and body of a description list + /// term: `:`. Colon, - /// The unary plus and addition operator: `+`. + /// The unary plus and addition operator, and start of enum items: `+`. Plus, - /// The unary negation and subtraction operator: `-`. + /// The unary negation and subtraction operator, and start of list + /// items: `-`. Minus, - /// The division operator: `/`. + /// The division operator and start of description list items: `/`. Slash, /// A field access and method call operator: `.`. Dot, @@ -627,16 +695,16 @@ pub enum NodeKind { StarEq, /// The divide-assign operator: `/=`. SlashEq, + /// The spread operator: `..`. + Dots, + /// An arrow between a closure's parameters and body: `=>`. + Arrow, /// The `not` operator. Not, /// The `and` operator. And, /// The `or` operator. Or, - /// The spread operator: `..`. - Dots, - /// An arrow between a closure's parameters and body: `=>`. - Arrow, /// The none literal: `none`. None, /// The auto literal: `auto`. @@ -673,60 +741,7 @@ pub enum NodeKind { From, /// The `as` keyword. As, - /// Markup of which all lines must have a minimal indentation. - /// - /// Notably, the number does not determine in which column the markup - /// started, but to the right of which column all markup elements must be, - /// so it is zero except for headings and lists. - Markup { min_indent: usize }, - /// One or more whitespace characters. Single spaces are collapsed into text - /// nodes if they would otherwise be surrounded by text nodes. - /// - /// Also stores how many newlines are contained. - Space { newlines: usize }, - /// Consecutive text without markup. While basic text with just single - /// spaces is collapsed into a single node, certain symbols that could - /// possibly be markup force text into multiple nodes. - Text(EcoString), - /// A forced line break: `\` or `\+` if justified. - Linebreak { justified: bool }, - /// A non-breaking space: `~`. - NonBreakingSpace, - /// A soft hyphen: `-?`. - Shy, - /// An en-dash: `--`. - EnDash, - /// An em-dash: `---`. - EmDash, - /// An ellipsis: `...`. - Ellipsis, - /// A smart quote: `'` or `"`. - Quote { double: bool }, - /// A slash and the letter "u" followed by a hexadecimal unicode entity - /// enclosed in curly braces: `\u{1F5FA}`. - Escape(char), - /// Strong content: `*Strong*`. - Strong, - /// Emphasized content: `_Emphasized_`. - Emph, - /// A raw block with optional syntax highlighting: `` `...` ``. - Raw(Arc<RawNode>), - /// A math formula: `$x$`, `$[x^2]$`. - Math(Arc<MathNode>), - /// A section heading: `= Introduction`. - Heading, - /// An item in an unordered list: `- ...`. - List, - /// An item in an enumeration (ordered list): `1. ...`. - Enum, - /// A numbering: `23.`. - /// - /// Can also exist without the number: `.`. - EnumNumbering(Option<usize>), - /// A label: `<label>`. - Label(EcoString), - /// A reference: `@label`. - Ref(EcoString), + /// An identifier: `center`. Ident(EcoString), /// A boolean: `true`, `false`. @@ -799,14 +814,7 @@ pub enum NodeKind { ContinueExpr, /// A return expression: `return x + 1`. ReturnExpr, - /// A line comment, two slashes followed by inner contents, terminated with - /// a newline: `//<str>\n`. - LineComment, - /// A block comment, a slash and a star followed by inner contents, - /// terminated with a star and a slash: `/*<str>*/`. - /// - /// The comment can contain nested block comments. - BlockComment, + /// Tokens that appear in the wrong place. Error(SpanPos, EcoString), /// Unknown character sequences. @@ -844,7 +852,7 @@ impl NodeKind { } } - /// Whether changes _inside_ this node are safely encapuslated, so that only + /// Whether changes _inside_ this node are safely encapsulated, so that only /// this node must be reparsed. pub fn is_bounded(&self) -> bool { match self { @@ -860,7 +868,6 @@ impl NodeKind { | Self::BlockComment | Self::Space { .. } | Self::Escape(_) => true, - Self::Text(t) => t != "-" && !t.ends_with('.'), _ => false, } } @@ -868,14 +875,43 @@ impl NodeKind { /// A human-readable name for the kind. pub fn as_str(&self) -> &'static str { match self { + Self::LineComment => "line comment", + Self::BlockComment => "block comment", Self::LeftBrace => "opening brace", Self::RightBrace => "closing brace", Self::LeftBracket => "opening bracket", Self::RightBracket => "closing bracket", Self::LeftParen => "opening paren", Self::RightParen => "closing paren", + + Self::Markup { .. } => "markup", + Self::Space { newlines: (2 ..) } => "paragraph break", + Self::Space { .. } => "space", + Self::Linebreak => "linebreak", + Self::Text(_) => "text", + Self::Escape(_) => "escape sequence", + Self::NonBreakingSpace => "non-breaking space", + Self::Shy => "soft hyphen", + Self::EnDash => "en dash", + Self::EmDash => "em dash", + Self::Ellipsis => "ellipsis", + Self::Quote { double: false } => "single quote", + Self::Quote { double: true } => "double quote", Self::Star => "star", Self::Underscore => "underscore", + Self::Strong => "strong content", + Self::Emph => "emphasized content", + Self::Link(_) => "link", + Self::Raw(_) => "raw block", + Self::Math(_) => "math formula", + Self::Heading => "heading", + Self::List => "list item", + Self::Enum => "enumeration item", + Self::EnumNumbering(_) => "enumeration item numbering", + Self::Desc => "description list item", + Self::Label(_) => "label", + Self::Ref(_) => "reference", + Self::Comma => "comma", Self::Semicolon => "semicolon", Self::Colon => "colon", @@ -894,11 +930,11 @@ impl NodeKind { Self::HyphEq => "subtract-assign operator", Self::StarEq => "multiply-assign operator", Self::SlashEq => "divide-assign operator", + Self::Dots => "dots", + Self::Arrow => "arrow", Self::Not => "operator `not`", Self::And => "operator `and`", Self::Or => "operator `or`", - Self::Dots => "dots", - Self::Arrow => "arrow", Self::None => "`none`", Self::Auto => "`auto`", Self::Let => "keyword `let`", @@ -909,7 +945,6 @@ impl NodeKind { Self::Else => "keyword `else`", Self::For => "keyword `for`", Self::In => "keyword `in`", - Self::As => "keyword `as`", Self::While => "keyword `while`", Self::Break => "keyword `break`", Self::Continue => "keyword `continue`", @@ -917,30 +952,8 @@ impl NodeKind { Self::Import => "keyword `import`", Self::Include => "keyword `include`", Self::From => "keyword `from`", - Self::Markup { .. } => "markup", - Self::Space { newlines: (2 ..) } => "paragraph break", - Self::Space { .. } => "space", - Self::Linebreak { justified: false } => "linebreak", - Self::Linebreak { justified: true } => "justified linebreak", - Self::Text(_) => "text", - Self::NonBreakingSpace => "non-breaking space", - Self::Shy => "soft hyphen", - Self::EnDash => "en dash", - Self::EmDash => "em dash", - Self::Ellipsis => "ellipsis", - Self::Quote { double: false } => "single quote", - Self::Quote { double: true } => "double quote", - Self::Escape(_) => "escape sequence", - Self::Strong => "strong content", - Self::Emph => "emphasized content", - Self::Raw(_) => "raw block", - Self::Math(_) => "math formula", - Self::List => "list item", - Self::Heading => "heading", - Self::Enum => "enumeration item", - Self::EnumNumbering(_) => "enumeration item numbering", - Self::Label(_) => "label", - Self::Ref(_) => "reference", + Self::As => "keyword `as`", + Self::Ident(_) => "identifier", Self::Bool(_) => "boolean", Self::Int(_) => "integer", @@ -977,8 +990,7 @@ impl NodeKind { Self::BreakExpr => "`break` expression", Self::ContinueExpr => "`continue` expression", Self::ReturnExpr => "`return` expression", - Self::LineComment => "line comment", - Self::BlockComment => "block comment", + Self::Error(_, _) => "parse error", Self::Unknown(text) => match text.as_str() { "*/" => "end of block comment", @@ -998,14 +1010,41 @@ impl Hash for NodeKind { fn hash<H: Hasher>(&self, state: &mut H) { std::mem::discriminant(self).hash(state); match self { + Self::LineComment => {} + Self::BlockComment => {} Self::LeftBrace => {} Self::RightBrace => {} Self::LeftBracket => {} Self::RightBracket => {} Self::LeftParen => {} Self::RightParen => {} + + Self::Markup { min_indent } => min_indent.hash(state), + Self::Space { newlines } => newlines.hash(state), + Self::Linebreak => {} + Self::Text(s) => s.hash(state), + Self::Escape(c) => c.hash(state), + Self::NonBreakingSpace => {} + Self::Shy => {} + Self::EnDash => {} + Self::EmDash => {} + Self::Ellipsis => {} + Self::Quote { double } => double.hash(state), Self::Star => {} Self::Underscore => {} + Self::Strong => {} + Self::Emph => {} + Self::Link(link) => link.hash(state), + Self::Raw(raw) => raw.hash(state), + Self::Math(math) => math.hash(state), + Self::Heading => {} + Self::List => {} + Self::Enum => {} + Self::EnumNumbering(num) => num.hash(state), + Self::Desc => {} + Self::Label(c) => c.hash(state), + Self::Ref(c) => c.hash(state), + Self::Comma => {} Self::Semicolon => {} Self::Colon => {} @@ -1024,11 +1063,11 @@ impl Hash for NodeKind { Self::HyphEq => {} Self::StarEq => {} Self::SlashEq => {} + Self::Dots => {} + Self::Arrow => {} Self::Not => {} Self::And => {} Self::Or => {} - Self::Dots => {} - Self::Arrow => {} Self::None => {} Self::Auto => {} Self::Let => {} @@ -1039,7 +1078,6 @@ impl Hash for NodeKind { Self::Else => {} Self::For => {} Self::In => {} - Self::As => {} Self::While => {} Self::Break => {} Self::Continue => {} @@ -1047,27 +1085,8 @@ impl Hash for NodeKind { Self::Import => {} Self::Include => {} Self::From => {} - Self::Markup { min_indent } => min_indent.hash(state), - Self::Space { newlines } => newlines.hash(state), - Self::Linebreak { justified } => justified.hash(state), - Self::Text(s) => s.hash(state), - Self::NonBreakingSpace => {} - Self::Shy => {} - Self::EnDash => {} - Self::EmDash => {} - Self::Ellipsis => {} - Self::Quote { double } => double.hash(state), - Self::Escape(c) => c.hash(state), - Self::Strong => {} - Self::Emph => {} - Self::Raw(raw) => raw.hash(state), - Self::Math(math) => math.hash(state), - Self::List => {} - Self::Heading => {} - Self::Enum => {} - Self::EnumNumbering(num) => num.hash(state), - Self::Label(c) => c.hash(state), - Self::Ref(c) => c.hash(state), + Self::As => {} + Self::Ident(v) => v.hash(state), Self::Bool(v) => v.hash(state), Self::Int(v) => v.hash(state), @@ -1104,8 +1123,7 @@ impl Hash for NodeKind { Self::BreakExpr => {} Self::ContinueExpr => {} Self::ReturnExpr => {} - Self::LineComment => {} - Self::BlockComment => {} + Self::Error(pos, msg) => (pos, msg).hash(state), Self::Unknown(text) => text.hash(state), } |
