summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLaurenz <laurmaedje@gmail.com>2022-09-26 15:39:32 +0200
committerLaurenz <laurmaedje@gmail.com>2022-09-26 16:12:57 +0200
commit704f2fbaf1b4483caa12f249a222c49e44f08961 (patch)
tree146f7813fe63409df2c1bbaa487731e992d3ac71
parent2661f1a5066bd5e3f8a9c68e4a5c304c248efcb7 (diff)
Description lists, link syntax, and new enum syntax
-rw-r--r--src/eval/mod.rs35
-rw-r--r--src/library/mod.rs1
-rw-r--r--src/library/structure/list.rs233
-rw-r--r--src/library/text/link.rs7
-rw-r--r--src/library/text/par.rs4
-rw-r--r--src/library/text/raw.rs2
-rw-r--r--src/model/content.rs25
-rw-r--r--src/model/property.rs7
-rw-r--r--src/model/recipe.rs7
-rw-r--r--src/parse/incremental.rs32
-rw-r--r--src/parse/mod.rs69
-rw-r--r--src/parse/resolve.rs2
-rw-r--r--src/parse/tokens.rs491
-rw-r--r--src/syntax/ast.rs54
-rw-r--r--src/syntax/highlight.rs130
-rw-r--r--src/syntax/mod.rs272
-rw-r--r--tests/ref/code/comment.pngbin2648 -> 797 bytes
-rw-r--r--tests/ref/structure/desc.pngbin0 -> 16889 bytes
-rw-r--r--tests/ref/structure/enum.pngbin28037 -> 29125 bytes
-rw-r--r--tests/ref/style/show-text.pngbin44844 -> 44838 bytes
-rw-r--r--tests/ref/text/escape.pngbin14079 -> 13269 bytes
-rw-r--r--tests/ref/text/link.pngbin40822 -> 49564 bytes
-rw-r--r--tests/typ/code/comment.typ5
-rw-r--r--tests/typ/math/basic.typ10
-rw-r--r--tests/typ/structure/desc.typ48
-rw-r--r--tests/typ/structure/enum.typ29
-rw-r--r--tests/typ/structure/list.typ2
-rw-r--r--tests/typ/style/set.typ6
-rw-r--r--tests/typ/text/escape.typ7
-rw-r--r--tests/typ/text/justify.typ4
-rw-r--r--tests/typ/text/linebreak.typ6
-rw-r--r--tests/typ/text/link.typ13
-rw-r--r--tests/typ/text/par.typ2
33 files changed, 879 insertions, 624 deletions
diff --git a/src/eval/mod.rs b/src/eval/mod.rs
index fb65420d..7ef5bf76 100644
--- a/src/eval/mod.rs
+++ b/src/eval/mod.rs
@@ -195,16 +195,20 @@ impl Eval for MarkupNode {
Ok(match self {
Self::Space => Content::Space,
Self::Parbreak => Content::Parbreak,
- &Self::Linebreak { justified } => Content::Linebreak { justified },
+ &Self::Linebreak => Content::Linebreak { justify: false },
Self::Text(text) => Content::Text(text.clone()),
&Self::Quote { double } => Content::Quote { double },
Self::Strong(strong) => strong.eval(vm)?,
Self::Emph(emph) => emph.eval(vm)?,
+ Self::Link(url) => {
+ Content::show(library::text::LinkNode::from_url(url.clone()))
+ }
Self::Raw(raw) => raw.eval(vm)?,
Self::Math(math) => math.eval(vm)?,
Self::Heading(heading) => heading.eval(vm)?,
Self::List(list) => list.eval(vm)?,
Self::Enum(enum_) => enum_.eval(vm)?,
+ Self::Desc(desc) => desc.eval(vm)?,
Self::Label(_) => Content::Empty,
Self::Ref(label) => Content::show(library::structure::RefNode(label.clone())),
Self::Expr(expr) => expr.eval(vm)?.display(),
@@ -273,11 +277,8 @@ impl Eval for ListNode {
type Output = Content;
fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> {
- Ok(Content::Item(library::structure::ListItem {
- kind: library::structure::UNORDERED,
- number: None,
- body: Box::new(self.body().eval(vm)?),
- }))
+ let body = Box::new(self.body().eval(vm)?);
+ Ok(Content::Item(library::structure::ListItem::List(body)))
}
}
@@ -285,11 +286,23 @@ impl Eval for EnumNode {
type Output = Content;
fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> {
- Ok(Content::Item(library::structure::ListItem {
- kind: library::structure::ORDERED,
- number: self.number(),
- body: Box::new(self.body().eval(vm)?),
- }))
+ let number = self.number();
+ let body = Box::new(self.body().eval(vm)?);
+ Ok(Content::Item(library::structure::ListItem::Enum(
+ number, body,
+ )))
+ }
+}
+
+impl Eval for DescNode {
+ type Output = Content;
+
+ fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> {
+ let term = self.term().eval(vm)?;
+ let body = self.body().eval(vm)?;
+ Ok(Content::Item(library::structure::ListItem::Desc(Box::new(
+ library::structure::DescItem { term, body },
+ ))))
}
}
diff --git a/src/library/mod.rs b/src/library/mod.rs
index 91e4671c..e7617bc0 100644
--- a/src/library/mod.rs
+++ b/src/library/mod.rs
@@ -41,6 +41,7 @@ pub fn new() -> Scope {
std.def_node::<structure::HeadingNode>("heading");
std.def_node::<structure::ListNode>("list");
std.def_node::<structure::EnumNode>("enum");
+ std.def_node::<structure::DescNode>("desc");
std.def_node::<structure::TableNode>("table");
// Layout.
diff --git a/src/library/structure/list.rs b/src/library/structure/list.rs
index 9da14733..7a43e5db 100644
--- a/src/library/structure/list.rs
+++ b/src/library/structure/list.rs
@@ -1,5 +1,3 @@
-use std::fmt::Write;
-
use unscanny::Scanner;
use crate::library::layout::{BlockSpacing, GridNode, TrackSizing};
@@ -9,9 +7,7 @@ use crate::library::utility::Numbering;
/// An unordered (bulleted) or ordered (numbered) list.
#[derive(Debug, Hash)]
-pub struct ListNode<const L: ListKind = UNORDERED> {
- /// Where the list starts.
- pub start: usize,
+pub struct ListNode<const L: ListKind = LIST> {
/// If true, the items are separated by leading instead of list spacing.
pub tight: bool,
/// If true, the spacing above the list is leading instead of above spacing.
@@ -20,19 +16,11 @@ pub struct ListNode<const L: ListKind = UNORDERED> {
pub items: StyleVec<ListItem>,
}
-/// An item in a list.
-#[derive(Clone, PartialEq, Hash)]
-pub struct ListItem {
- /// The kind of item.
- pub kind: ListKind,
- /// The number of the item.
- pub number: Option<usize>,
- /// The node that produces the item's body.
- pub body: Box<Content>,
-}
-
/// An ordered list.
-pub type EnumNode = ListNode<ORDERED>;
+pub type EnumNode = ListNode<ENUM>;
+
+/// A description list.
+pub type DescNode = ListNode<DESC>;
#[node(showable)]
impl<const L: ListKind> ListNode<L> {
@@ -44,7 +32,11 @@ impl<const L: ListKind> ListNode<L> {
pub const INDENT: RawLength = RawLength::zero();
/// The space between the label and the body of each item.
#[property(resolve)]
- pub const BODY_INDENT: RawLength = Em::new(0.5).into();
+ pub const BODY_INDENT: RawLength = Em::new(match L {
+ LIST | ENUM => 0.5,
+ DESC | _ => 1.0,
+ })
+ .into();
/// The spacing above the list.
#[property(resolve, shorthand(around))]
@@ -57,19 +49,34 @@ impl<const L: ListKind> ListNode<L> {
pub const SPACING: BlockSpacing = Ratio::one().into();
fn construct(_: &mut Vm, args: &mut Args) -> SourceResult<Content> {
- Ok(Content::show(Self {
- start: args.named("start")?.unwrap_or(1),
- tight: args.named("tight")?.unwrap_or(true),
- attached: args.named("attached")?.unwrap_or(false),
- items: args
+ let items = match L {
+ LIST => args
.all()?
.into_iter()
- .map(|body| ListItem {
- kind: L,
- number: None,
- body: Box::new(body),
- })
+ .map(|body| ListItem::List(Box::new(body)))
.collect(),
+ ENUM => {
+ let mut number: usize = args.named("start")?.unwrap_or(1);
+ args.all()?
+ .into_iter()
+ .map(|body| {
+ let item = ListItem::Enum(Some(number), Box::new(body));
+ number += 1;
+ item
+ })
+ .collect()
+ }
+ DESC | _ => args
+ .all()?
+ .into_iter()
+ .map(|item| ListItem::Desc(Box::new(item)))
+ .collect(),
+ };
+
+ Ok(Content::show(Self {
+ tight: args.named("tight")?.unwrap_or(true),
+ attached: args.named("attached")?.unwrap_or(false),
+ items,
}))
}
}
@@ -77,10 +84,7 @@ impl<const L: ListKind> ListNode<L> {
impl<const L: ListKind> Show for ListNode<L> {
fn unguard(&self, sel: Selector) -> ShowNode {
Self {
- items: self.items.map(|item| ListItem {
- body: Box::new(item.body.unguard(sel).role(Role::ListItemBody)),
- ..*item
- }),
+ items: self.items.map(|item| item.unguard(sel)),
..*self
}
.pack()
@@ -88,13 +92,12 @@ impl<const L: ListKind> Show for ListNode<L> {
fn encode(&self, _: StyleChain) -> Dict {
dict! {
- "start" => Value::Int(self.start as i64),
"tight" => Value::Bool(self.tight),
"attached" => Value::Bool(self.attached),
"items" => Value::Array(
self.items
.items()
- .map(|item| Value::Content(item.body.as_ref().clone()))
+ .map(|item| item.encode())
.collect()
),
}
@@ -106,34 +109,54 @@ impl<const L: ListKind> Show for ListNode<L> {
styles: StyleChain,
) -> SourceResult<Content> {
let mut cells = vec![];
- let mut number = self.start;
+ let mut number = 1;
let label = styles.get(Self::LABEL);
+ let indent = styles.get(Self::INDENT);
+ let body_indent = styles.get(Self::BODY_INDENT);
+ let gutter = if self.tight {
+ styles.get(ParNode::LEADING)
+ } else {
+ styles.get(Self::SPACING)
+ };
for (item, map) in self.items.iter() {
- number = item.number.unwrap_or(number);
+ if let &ListItem::Enum(Some(n), _) = item {
+ number = n;
+ }
cells.push(LayoutNode::default());
- cells.push(
+
+ let label = if L == LIST || L == ENUM {
label
.resolve(world, L, number)?
.styled_with_map(map.clone())
.role(Role::ListLabel)
- .pack(),
- );
+ .pack()
+ } else {
+ LayoutNode::default()
+ };
+
+ cells.push(label);
cells.push(LayoutNode::default());
- cells.push((*item.body).clone().styled_with_map(map.clone()).pack());
- number += 1;
- }
- let gutter = if self.tight {
- styles.get(ParNode::LEADING)
- } else {
- styles.get(Self::SPACING)
- };
+ let body = match &item {
+ ListItem::List(body) => body.as_ref().clone(),
+ ListItem::Enum(_, body) => body.as_ref().clone(),
+ ListItem::Desc(item) => Content::sequence(vec![
+ Content::Horizontal {
+ amount: (-body_indent).into(),
+ weak: false,
+ },
+ (item.term.clone() + Content::Text(':'.into())).strong(),
+ Content::Space,
+ item.body.clone(),
+ ]),
+ };
- let indent = styles.get(Self::INDENT);
- let body_indent = styles.get(Self::BODY_INDENT);
+ cells.push(body.styled_with_map(map.clone()).pack());
+ number += 1;
+ }
Ok(Content::block(GridNode {
tracks: Spec::with_x(vec![
@@ -165,35 +188,110 @@ impl<const L: ListKind> Show for ListNode<L> {
}
}
- Ok(realized
- .role(Role::List { ordered: L == ORDERED })
- .spaced(above, below))
+ Ok(realized.role(Role::List { ordered: L == ENUM }).spaced(above, below))
+ }
+}
+
+/// An item in a list.
+#[derive(Clone, PartialEq, Hash)]
+pub enum ListItem {
+ /// An item of an unordered list.
+ List(Box<Content>),
+ /// An item of an ordered list.
+ Enum(Option<usize>, Box<Content>),
+ /// An item of a description list.
+ Desc(Box<DescItem>),
+}
+
+impl ListItem {
+ /// What kind of item this is.
+ pub fn kind(&self) -> ListKind {
+ match self {
+ Self::List(_) => LIST,
+ Self::Enum { .. } => ENUM,
+ Self::Desc { .. } => DESC,
+ }
+ }
+
+ fn unguard(&self, sel: Selector) -> Self {
+ match self {
+ Self::List(body) => Self::List(Box::new(body.unguard(sel))),
+ Self::Enum(number, body) => Self::Enum(*number, Box::new(body.unguard(sel))),
+ Self::Desc(item) => Self::Desc(Box::new(DescItem {
+ term: item.term.unguard(sel),
+ body: item.body.unguard(sel),
+ })),
+ }
+ }
+
+ /// Encode the item into a value.
+ fn encode(&self) -> Value {
+ match self {
+ Self::List(body) => Value::Content(body.as_ref().clone()),
+ Self::Enum(number, body) => Value::Dict(dict! {
+ "number" => match *number {
+ Some(n) => Value::Int(n as i64),
+ None => Value::None,
+ },
+ "body" => Value::Content(body.as_ref().clone()),
+ }),
+ Self::Desc(item) => Value::Dict(dict! {
+ "term" => Value::Content(item.term.clone()),
+ "body" => Value::Content(item.body.clone()),
+ }),
+ }
}
}
impl Debug for ListItem {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
- if self.kind == UNORDERED {
- f.write_char('-')?;
- } else {
- if let Some(number) = self.number {
- write!(f, "{}", number)?;
- }
- f.write_char('.')?;
+ match self {
+ Self::List(body) => write!(f, "- {body:?}"),
+ Self::Enum(number, body) => match number {
+ Some(n) => write!(f, "{n}. {body:?}"),
+ None => write!(f, "+ {body:?}"),
+ },
+ Self::Desc(item) => item.fmt(f),
}
- f.write_char(' ')?;
- self.body.fmt(f)
+ }
+}
+
+/// A description list item.
+#[derive(Clone, PartialEq, Hash)]
+pub struct DescItem {
+ /// The term described by the list item.
+ pub term: Content,
+ /// The description of the term.
+ pub body: Content,
+}
+
+castable! {
+ DescItem,
+ Expected: "dictionary with `term` and `body` keys",
+ Value::Dict(dict) => {
+ let term: Content = dict.get("term")?.clone().cast()?;
+ let body: Content = dict.get("body")?.clone().cast()?;
+ Self { term, body }
+ },
+}
+
+impl Debug for DescItem {
+ fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+ write!(f, "/ {:?}: {:?}", self.term, self.body)
}
}
/// How to label a list.
pub type ListKind = usize;
-/// Unordered list labelling style.
-pub const UNORDERED: ListKind = 0;
+/// An unordered list.
+pub const LIST: ListKind = 0;
+
+/// An ordered list.
+pub const ENUM: ListKind = 1;
-/// Ordered list labelling style.
-pub const ORDERED: ListKind = 1;
+/// A description list.
+pub const DESC: ListKind = 2;
/// How to label a list or enumeration.
#[derive(Debug, Clone, PartialEq, Hash)]
@@ -218,8 +316,9 @@ impl Label {
) -> SourceResult<Content> {
Ok(match self {
Self::Default => match kind {
- UNORDERED => Content::Text('•'.into()),
- ORDERED | _ => Content::Text(format_eco!("{}.", number)),
+ LIST => Content::Text('•'.into()),
+ ENUM => Content::Text(format_eco!("{}.", number)),
+ DESC | _ => panic!("description lists don't have a label"),
},
Self::Pattern(prefix, numbering, upper, suffix) => {
let fmt = numbering.apply(number);
diff --git a/src/library/text/link.rs b/src/library/text/link.rs
index c06fea55..78ae4d23 100644
--- a/src/library/text/link.rs
+++ b/src/library/text/link.rs
@@ -10,6 +10,13 @@ pub struct LinkNode {
pub body: Option<Content>,
}
+impl LinkNode {
+ /// Create a link node from a URL with its bare text.
+ pub fn from_url(url: EcoString) -> Self {
+ Self { dest: Destination::Url(url), body: None }
+ }
+}
+
#[node(showable)]
impl LinkNode {
/// The fill color of text in the link. Just the surrounding text color
diff --git a/src/library/text/par.rs b/src/library/text/par.rs
index 6910c23a..859a7c87 100644
--- a/src/library/text/par.rs
+++ b/src/library/text/par.rs
@@ -181,8 +181,8 @@ pub struct LinebreakNode;
#[node]
impl LinebreakNode {
fn construct(_: &mut Vm, args: &mut Args) -> SourceResult<Content> {
- let justified = args.named("justified")?.unwrap_or(false);
- Ok(Content::Linebreak { justified })
+ let justify = args.named("justify")?.unwrap_or(false);
+ Ok(Content::Linebreak { justify })
}
}
diff --git a/src/library/text/raw.rs b/src/library/text/raw.rs
index e7c73a91..a64b1a92 100644
--- a/src/library/text/raw.rs
+++ b/src/library/text/raw.rs
@@ -91,7 +91,7 @@ impl Show for RawNode {
let mut highlighter = HighlightLines::new(syntax, &THEME);
for (i, line) in self.text.lines().enumerate() {
if i != 0 {
- seq.push(Content::Linebreak { justified: false });
+ seq.push(Content::Linebreak { justify: false });
}
for (style, piece) in
diff --git a/src/model/content.rs b/src/model/content.rs
index 92d592a6..ae074b50 100644
--- a/src/model/content.rs
+++ b/src/model/content.rs
@@ -14,7 +14,7 @@ use super::{
use crate::diag::StrResult;
use crate::library::layout::{FlowChild, FlowNode, PageNode, PlaceNode, Spacing};
use crate::library::prelude::*;
-use crate::library::structure::{DocNode, ListItem, ListNode, ORDERED, UNORDERED};
+use crate::library::structure::{DocNode, ListItem, ListNode, DESC, ENUM, LIST};
use crate::library::text::{
DecoNode, EmphNode, ParChild, ParNode, StrongNode, UNDERLINE,
};
@@ -62,7 +62,7 @@ pub enum Content {
/// A word space.
Space,
/// A forced line break.
- Linebreak { justified: bool },
+ Linebreak { justify: bool },
/// Horizontal spacing.
Horizontal { amount: Spacing, weak: bool },
/// Plain text.
@@ -264,7 +264,7 @@ impl Debug for Content {
match self {
Self::Empty => f.pad("Empty"),
Self::Space => f.pad("Space"),
- Self::Linebreak { justified } => write!(f, "Linebreak({justified})"),
+ Self::Linebreak { justify } => write!(f, "Linebreak({justify})"),
Self::Horizontal { amount, weak } => {
write!(f, "Horizontal({amount:?}, {weak})")
}
@@ -633,8 +633,8 @@ impl<'a> ParBuilder<'a> {
Content::Space => {
self.0.weak(ParChild::Text(' '.into()), styles, 2);
}
- &Content::Linebreak { justified } => {
- let c = if justified { '\u{2028}' } else { '\n' };
+ &Content::Linebreak { justify } => {
+ let c = if justify { '\u{2028}' } else { '\n' };
self.0.destructive(ParChild::Text(c.into()), styles);
}
&Content::Horizontal { amount, weak } => {
@@ -734,7 +734,7 @@ impl<'a> ListBuilder<'a> {
.items
.items()
.next()
- .map_or(true, |first| item.kind == first.kind) =>
+ .map_or(true, |first| item.kind() == first.kind()) =>
{
self.items.push(item.clone(), styles);
self.tight &= self.staged.drain(..).all(|(t, _)| *t != Content::Parbreak);
@@ -751,21 +751,16 @@ impl<'a> ListBuilder<'a> {
fn finish(self, parent: &mut Builder<'a>) -> SourceResult<()> {
let (items, shared) = self.items.finish();
let kind = match items.items().next() {
- Some(item) => item.kind,
+ Some(item) => item.kind(),
None => return Ok(()),
};
- let start = 1;
let tight = self.tight;
let attached = tight && self.attachable;
-
let content = match kind {
- UNORDERED => {
- Content::show(ListNode::<UNORDERED> { start, tight, attached, items })
- }
- ORDERED | _ => {
- Content::show(ListNode::<ORDERED> { start, tight, attached, items })
- }
+ LIST => Content::show(ListNode::<LIST> { tight, attached, items }),
+ ENUM => Content::show(ListNode::<ENUM> { tight, attached, items }),
+ DESC | _ => Content::show(ListNode::<DESC> { tight, attached, items }),
};
let stored = parent.scratch.templates.alloc(content);
diff --git a/src/model/property.rs b/src/model/property.rs
index 18f41eee..ab4f02e3 100644
--- a/src/model/property.rs
+++ b/src/model/property.rs
@@ -9,7 +9,7 @@ use super::{Interruption, NodeId, StyleChain};
use crate::eval::{RawLength, Smart};
use crate::geom::{Corners, Length, Numeric, Relative, Sides, Spec};
use crate::library::layout::PageNode;
-use crate::library::structure::{EnumNode, ListNode};
+use crate::library::structure::{DescNode, EnumNode, ListNode};
use crate::library::text::ParNode;
use crate::util::ReadableTypeId;
@@ -68,7 +68,10 @@ impl Property {
Some(Interruption::Page)
} else if self.is_of::<ParNode>() {
Some(Interruption::Par)
- } else if self.is_of::<ListNode>() || self.is_of::<EnumNode>() {
+ } else if self.is_of::<ListNode>()
+ || self.is_of::<EnumNode>()
+ || self.is_of::<DescNode>()
+ {
Some(Interruption::List)
} else {
None
diff --git a/src/model/recipe.rs b/src/model/recipe.rs
index 6b21ccf2..27b1be42 100644
--- a/src/model/recipe.rs
+++ b/src/model/recipe.rs
@@ -5,7 +5,7 @@ use comemo::Tracked;
use super::{Content, Interruption, NodeId, Show, ShowNode, StyleChain, StyleEntry};
use crate::diag::SourceResult;
use crate::eval::{Args, Func, Regex, Value};
-use crate::library::structure::{EnumNode, ListNode};
+use crate::library::structure::{DescNode, EnumNode, ListNode};
use crate::syntax::Spanned;
use crate::World;
@@ -93,7 +93,10 @@ impl Recipe {
/// What kind of structure the property interrupts.
pub fn interruption(&self) -> Option<Interruption> {
if let Pattern::Node(id) = self.pattern {
- if id == NodeId::of::<ListNode>() || id == NodeId::of::<EnumNode>() {
+ if id == NodeId::of::<ListNode>()
+ || id == NodeId::of::<EnumNode>()
+ || id == NodeId::of::<DescNode>()
+ {
return Some(Interruption::List);
}
}
diff --git a/src/parse/incremental.rs b/src/parse/incremental.rs
index 9272dcfc..06096a75 100644
--- a/src/parse/incremental.rs
+++ b/src/parse/incremental.rs
@@ -407,18 +407,18 @@ mod tests {
test("", 0..0, "do it", 0..5);
test("a d e", 1 .. 3, " b c d", 0 .. 9);
test("*~ *", 2..2, "*", 0..5);
- test("_1_\n2a\n3", 5..5, "4", 0..7);
- test("_1_\n2a\n3~", 8..8, "4", 5..10);
+ test("_1_\n2a\n3", 5..5, "4", 4..7);
+ test("_1_\n2a\n3~", 8..8, "4", 4..10);
test("_1_ 2 3a\n4", 7..7, "5", 0..9);
test("* {1+2} *", 5..6, "3", 2..7);
test("a #f() e", 1 .. 6, " b c d", 0 .. 9);
test("a\nb\nc\nd\ne\n", 5 .. 5, "c", 2 .. 7);
test("a\n\nb\n\nc\n\nd\n\ne\n", 7 .. 7, "c", 3 .. 10);
- test("a\nb\nc *hel a b lo* d\nd\ne", 13..13, "c ", 6..20);
+ test("a\nb\nc *hel a b lo* d\nd\ne", 13..13, "c ", 4..20);
test("~~ {a} ~~", 4 .. 5, "b", 3 .. 6);
test("{(0, 1, 2)}", 5 .. 6, "11pt", 0..14);
test("\n= A heading", 4 .. 4, "n evocative", 0 .. 23);
- test("for~your~thing", 9 .. 9, "a", 4 .. 15);
+ test("for~your~thing", 9 .. 9, "a", 0 .. 15);
test("a your thing a", 6 .. 7, "a", 0 .. 14);
test("{call(); abc}", 7 .. 7, "[]", 0 .. 15);
test("#call() abc", 7 .. 7, "[]", 0 .. 10);
@@ -429,17 +429,17 @@ mod tests {
test("#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", 34 .. 41, "_bar_", 33 .. 40);
test("{let i=1; for x in range(5) {i}}", 6 .. 6, " ", 0 .. 33);
test("{let i=1; for x in range(5) {i}}", 13 .. 14, " ", 0 .. 33);
- test("hello~~{x}", 7 .. 10, "#f()", 5 .. 11);
- test("this~is -- in my opinion -- spectacular", 8 .. 10, "---", 5 .. 25);
- test("understanding `code` is complicated", 15 .. 15, "C ", 14 .. 22);
+ test("hello~~{x}", 7 .. 10, "#f()", 0 .. 11);
+ test("this~is -- in my opinion -- spectacular", 8 .. 10, "---", 0 .. 25);
+ test("understanding `code` is complicated", 15 .. 15, "C ", 0 .. 22);
test("{ let x = g() }", 10 .. 12, "f(54", 0 .. 17);
- test(r#"a ```typst hello``` b"#, 16 .. 17, "", 2 .. 18);
- test(r#"a ```typst hello```"#, 16 .. 17, "", 2 .. 18);
+ test(r#"a ```typst hello``` b"#, 16 .. 17, "", 0 .. 18);
+ test(r#"a ```typst hello```"#, 16 .. 17, "", 0 .. 18);
test("#for", 4 .. 4, "//", 0 .. 6);
test("#show a: f as b..", 16..16, "c", 0..18);
test("a\n#let \nb", 7 .. 7, "i", 2 .. 9);
test("a\n#for i \nb", 9 .. 9, "in", 2 .. 12);
- test("a~https://fun/html", 13..14, "n", 2..18);
+ test("a~https://fun/html", 13..14, "n", 0..18);
}
#[test]
@@ -452,7 +452,7 @@ mod tests {
test("abc\n= a heading\njoke", 3 .. 4, "\nnot ", 0 .. 19);
test("#let x = (1, 2 + ;~ Five\r\n\r", 20 .. 23, "2.", 0 .. 23);
test("hey #myfriend", 4 .. 4, "\\", 0 .. 14);
- test("hey #myfriend", 4 .. 4, "\\", 3 .. 6);
+ test("hey #myfriend", 4 .. 4, "\\", 0 .. 6);
test("= foo\nbar\n - a\n - b", 6 .. 9, "", 0 .. 11);
test("= foo\n bar\n baz", 6 .. 8, "", 0 .. 9);
test(" // hi", 1 .. 1, " ", 0 .. 7);
@@ -461,12 +461,12 @@ mod tests {
#[test]
fn test_parse_incremental_type_invariants() {
- test("a #for x in array {x}", 18 .. 21, "[#x]", 2 .. 22);
- test("a #let x = 1 {5}", 3 .. 6, "if", 2 .. 11);
+ test("a #for x in array {x}", 18 .. 21, "[#x]", 0 .. 22);
+ test("a #let x = 1 {5}", 3 .. 6, "if", 0 .. 11);
test("a {let x = 1 {5}} b", 3 .. 6, "if", 2 .. 16);
test("#let x = 1 {5}", 4 .. 4, " if", 0 .. 13);
test("{let x = 1 {5}}", 4 .. 4, " if", 0 .. 18);
- test("a // b c #f()", 3 .. 4, "", 2 .. 12);
+ test("a // b c #f()", 3 .. 4, "", 0 .. 12);
test("{\nf()\n//g(a)\n}", 6 .. 8, "", 0 .. 12);
test("a{\nf()\n//g(a)\n}b", 7 .. 9, "", 1 .. 13);
test("a #while x {\n g(x) \n} b", 11 .. 11, "//", 0 .. 26);
@@ -477,8 +477,8 @@ mod tests {
#[test]
fn test_parse_incremental_wrongly_or_unclosed_things() {
test(r#"{"hi"}"#, 4 .. 5, "c", 0 .. 6);
- test(r"this \u{abcd}", 8 .. 9, "", 5 .. 12);
- test(r"this \u{abcd} that", 12 .. 13, "", 5 .. 17);
+ test(r"this \u{abcd}", 8 .. 9, "", 0 .. 12);
+ test(r"this \u{abcd} that", 12 .. 13, "", 0 .. 17);
test(r"{{let x = z}; a = 1} b", 6 .. 6, "//", 0 .. 24);
test("a b c", 1 .. 1, " /* letters */", 0 .. 19);
test("a b c", 1 .. 1, " /* letters", 0 .. 16);
diff --git a/src/parse/mod.rs b/src/parse/mod.rs
index a8dee7f3..ed8bc5ce 100644
--- a/src/parse/mod.rs
+++ b/src/parse/mod.rs
@@ -162,11 +162,6 @@ fn markup(p: &mut Parser, mut at_start: bool) {
});
}
-/// Parse a single line of markup.
-fn markup_line(p: &mut Parser) {
- markup_indented(p, usize::MAX);
-}
-
/// Parse markup that stays right of the given `column`.
fn markup_indented(p: &mut Parser, min_indent: usize) {
p.eat_while(|t| match t {
@@ -185,7 +180,6 @@ fn markup_indented(p: &mut Parser, min_indent: usize) {
{
break;
}
- Some(NodeKind::Label(_)) => break,
_ => {}
}
@@ -195,6 +189,33 @@ fn markup_indented(p: &mut Parser, min_indent: usize) {
marker.end(p, NodeKind::Markup { min_indent });
}
+/// Parse a line of markup that can prematurely end if `f` returns true.
+fn markup_line<F>(p: &mut Parser, mut f: F)
+where
+ F: FnMut(&NodeKind) -> bool,
+{
+ p.eat_while(|t| match t {
+ NodeKind::Space { newlines } => *newlines == 0,
+ NodeKind::LineComment | NodeKind::BlockComment => true,
+ _ => false,
+ });
+
+ p.perform(NodeKind::Markup { min_indent: usize::MAX }, |p| {
+ let mut at_start = false;
+ while let Some(kind) = p.peek() {
+ if let NodeKind::Space { newlines: (1 ..) } = kind {
+ break;
+ }
+
+ if f(kind) {
+ break;
+ }
+
+ markup_node(p, &mut at_start);
+ }
+ });
+}
+
/// Parse a markup node.
fn markup_node(p: &mut Parser, at_start: &mut bool) {
let token = match p.peek() {
@@ -226,6 +247,7 @@ fn markup_node(p: &mut Parser, at_start: &mut bool) {
| NodeKind::Ellipsis
| NodeKind::Quote { .. }
| NodeKind::Escape(_)
+ | NodeKind::Link(_)
| NodeKind::Raw(_)
| NodeKind::Math(_)
| NodeKind::Label(_)
@@ -233,12 +255,22 @@ fn markup_node(p: &mut Parser, at_start: &mut bool) {
p.eat();
}
- // Grouping markup.
+ // Strong, emph, heading.
NodeKind::Star => strong(p),
NodeKind::Underscore => emph(p),
NodeKind::Eq => heading(p, *at_start),
+
+ // Lists.
NodeKind::Minus => list_node(p, *at_start),
- NodeKind::EnumNumbering(_) => enum_node(p, *at_start),
+ NodeKind::Plus | NodeKind::EnumNumbering(_) => enum_node(p, *at_start),
+ NodeKind::Slash => {
+ desc_node(p, *at_start).ok();
+ }
+ NodeKind::Colon => {
+ let marker = p.marker();
+ p.eat();
+ marker.convert(p, NodeKind::Text(':'.into()));
+ }
// Hashtag + keyword / identifier.
NodeKind::Ident(_)
@@ -293,7 +325,7 @@ fn heading(p: &mut Parser, at_start: bool) {
if at_start && p.peek().map_or(true, |kind| kind.is_space()) {
p.eat_while(|kind| *kind == NodeKind::Space { newlines: 0 });
- markup_line(p);
+ markup_line(p, |kind| matches!(kind, NodeKind::Label(_)));
marker.end(p, NodeKind::Heading);
} else {
let text = p.get(current_start .. p.prev_end()).into();
@@ -331,6 +363,25 @@ fn enum_node(p: &mut Parser, at_start: bool) {
}
}
+/// Parse a single description list item.
+fn desc_node(p: &mut Parser, at_start: bool) -> ParseResult {
+ let marker = p.marker();
+ let text: EcoString = p.peek_src().into();
+ p.eat();
+
+ let min_indent = p.column(p.prev_end());
+ if at_start && p.eat_if(NodeKind::Space { newlines: 0 }) && !p.eof() {
+ markup_line(p, |node| matches!(node, NodeKind::Colon));
+ p.expect(NodeKind::Colon)?;
+ markup_indented(p, min_indent);
+ marker.end(p, NodeKind::Desc);
+ } else {
+ marker.convert(p, NodeKind::Text(text));
+ }
+
+ Ok(())
+}
+
/// Parse an expression within a markup mode.
fn markup_expr(p: &mut Parser) {
// Does the expression need termination or can content follow directly?
diff --git a/src/parse/resolve.rs b/src/parse/resolve.rs
index bce2da3c..d68282c0 100644
--- a/src/parse/resolve.rs
+++ b/src/parse/resolve.rs
@@ -26,14 +26,12 @@ pub fn resolve_string(string: &str) -> EcoString {
// TODO: Error if closing brace is missing.
let sequence = s.eat_while(char::is_ascii_hexdigit);
let _terminated = s.eat_if('}');
-
match resolve_hex(sequence) {
Some(c) => out.push(c),
None => out.push_str(s.from(start)),
}
}
- // TODO: Error for invalid escape sequence.
_ => out.push_str(s.from(start)),
}
}
diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs
index 84a63ba1..f6d4b0e8 100644
--- a/src/parse/tokens.rs
+++ b/src/parse/tokens.rs
@@ -103,6 +103,11 @@ impl<'s> Iterator for Tokens<'s> {
let start = self.s.cursor();
let c = self.s.eat()?;
Some(match c {
+ // Comments.
+ '/' if self.s.eat_if('/') => self.line_comment(),
+ '/' if self.s.eat_if('*') => self.block_comment(),
+ '*' if self.s.eat_if('/') => NodeKind::Unknown("*/".into()),
+
// Blocks.
'{' => NodeKind::LeftBrace,
'}' => NodeKind::RightBrace,
@@ -110,15 +115,7 @@ impl<'s> Iterator for Tokens<'s> {
']' => NodeKind::RightBracket,
// Whitespace.
- ' ' if self.s.done() || !self.s.at(char::is_whitespace) => {
- NodeKind::Space { newlines: 0 }
- }
- c if c.is_whitespace() => self.whitespace(),
-
- // Comments with special case for URLs.
- '/' if self.s.eat_if('*') => self.block_comment(),
- '/' if !self.maybe_in_url() && self.s.eat_if('/') => self.line_comment(),
- '*' if self.s.eat_if('/') => NodeKind::Unknown(self.s.from(start).into()),
+ c if c.is_whitespace() => self.whitespace(c),
// Other things.
_ => match self.mode {
@@ -130,85 +127,110 @@ impl<'s> Iterator for Tokens<'s> {
}
impl<'s> Tokens<'s> {
+ fn line_comment(&mut self) -> NodeKind {
+ self.s.eat_until(is_newline);
+ if self.s.peek().is_none() {
+ self.terminated = false;
+ }
+ NodeKind::LineComment
+ }
+
+ fn block_comment(&mut self) -> NodeKind {
+ let mut state = '_';
+ let mut depth = 1;
+ self.terminated = false;
+
+ // Find the first `*/` that does not correspond to a nested `/*`.
+ while let Some(c) = self.s.eat() {
+ state = match (state, c) {
+ ('*', '/') => {
+ depth -= 1;
+ if depth == 0 {
+ self.terminated = true;
+ break;
+ }
+ '_'
+ }
+ ('/', '*') => {
+ depth += 1;
+ '_'
+ }
+ ('/', '/') => {
+ self.line_comment();
+ '_'
+ }
+ _ => c,
+ }
+ }
+
+ NodeKind::BlockComment
+ }
+
+ fn whitespace(&mut self, c: char) -> NodeKind {
+ if c == ' ' && !self.s.at(char::is_whitespace) {
+ return NodeKind::Space { newlines: 0 };
+ }
+
+ self.s.uneat();
+
+ // Count the number of newlines.
+ let mut newlines = 0;
+ while let Some(c) = self.s.eat() {
+ if !c.is_whitespace() {
+ self.s.uneat();
+ break;
+ }
+
+ if is_newline(c) {
+ if c == '\r' {
+ self.s.eat_if('\n');
+ }
+ newlines += 1;
+ }
+ }
+
+ NodeKind::Space { newlines }
+ }
+
#[inline]
fn markup(&mut self, start: usize, c: char) -> NodeKind {
match c {
// Escape sequences.
'\\' => self.backslash(),
- // Keywords and identifiers.
- '#' => self.hash(),
-
- // Markup.
+ // Single-char things.
'~' => NodeKind::NonBreakingSpace,
- '-' => self.hyph(),
'.' if self.s.eat_if("..") => NodeKind::Ellipsis,
'\'' => NodeKind::Quote { double: false },
'"' => NodeKind::Quote { double: true },
'*' if !self.in_word() => NodeKind::Star,
'_' if !self.in_word() => NodeKind::Underscore,
- '`' => self.raw(),
'=' => NodeKind::Eq,
- '$' => self.math(),
- '<' => self.label(),
- '@' => self.reference(),
- c if c == '.' || c.is_ascii_digit() => self.numbering(start, c),
-
- // Plain text.
- _ => self.text(start),
- }
- }
-
- fn code(&mut self, start: usize, c: char) -> NodeKind {
- match c {
- // Parens.
- '(' => NodeKind::LeftParen,
- ')' => NodeKind::RightParen,
-
- // Length two.
- '=' if self.s.eat_if('=') => NodeKind::EqEq,
- '!' if self.s.eat_if('=') => NodeKind::ExclEq,
- '<' if self.s.eat_if('=') => NodeKind::LtEq,
- '>' if self.s.eat_if('=') => NodeKind::GtEq,
- '+' if self.s.eat_if('=') => NodeKind::PlusEq,
- '-' if self.s.eat_if('=') => NodeKind::HyphEq,
- '*' if self.s.eat_if('=') => NodeKind::StarEq,
- '/' if self.s.eat_if('=') => NodeKind::SlashEq,
- '.' if self.s.eat_if('.') => NodeKind::Dots,
- '=' if self.s.eat_if('>') => NodeKind::Arrow,
-
- // Length one.
- ',' => NodeKind::Comma,
- ';' => NodeKind::Semicolon,
- ':' => NodeKind::Colon,
'+' => NodeKind::Plus,
- '-' => NodeKind::Minus,
- '*' => NodeKind::Star,
'/' => NodeKind::Slash,
- '=' => NodeKind::Eq,
- '<' => NodeKind::Lt,
- '>' => NodeKind::Gt,
- '.' if self.s.done() || !self.s.at(char::is_ascii_digit) => NodeKind::Dot,
-
- // Identifiers.
- c if is_id_start(c) => self.ident(start),
+ ':' => NodeKind::Colon,
- // Numbers.
- c if c.is_ascii_digit() || (c == '.' && self.s.at(char::is_ascii_digit)) => {
- self.number(start, c)
+ // Multi-char things.
+ '#' => self.hash(start),
+ '-' => self.hyph(),
+ 'h' if self.s.eat_if("ttp://") || self.s.eat_if("ttps://") => {
+ self.link(start)
}
+ '`' => self.raw(),
+ '$' => self.math(),
+ c if c.is_ascii_digit() => self.numbering(start),
+ '<' => self.label(),
+ '@' => self.reference(start),
- // Strings.
- '"' => self.string(),
-
- _ => NodeKind::Unknown(self.s.from(start).into()),
+ // Plain text.
+ _ => self.text(start),
}
}
#[inline]
fn text(&mut self, start: usize) -> NodeKind {
macro_rules! table {
- ($($c:literal)|*) => {{
+ ($(|$c:literal)*) => {{
let mut t = [false; 128];
$(t[$c as usize] = true;)*
t
@@ -216,12 +238,9 @@ impl<'s> Tokens<'s> {
}
const TABLE: [bool; 128] = table! {
- // Ascii whitespace.
- ' ' | '\t' | '\n' | '\x0b' | '\x0c' | '\r' |
- // Comments, parentheses, code.
- '/' | '[' | ']' | '{' | '}' | '#' |
- // Markup
- '~' | '-' | '.' | '\'' | '"' | '*' | '_' | '`' | '$' | '\\'
+ | ' ' | '\t' | '\n' | '\x0b' | '\x0c' | '\r' | '\\' | '/'
+ | '[' | ']' | '{' | '}' | '~' | '-' | '.' | '\'' | '"'
+ | '*' | '_' | ':' | 'h' | '`' | '$' | '<' | '>' | '@' | '#'
};
loop {
@@ -229,14 +248,17 @@ impl<'s> Tokens<'s> {
TABLE.get(c as usize).copied().unwrap_or_else(|| c.is_whitespace())
});
- // Allow a single space, optionally preceded by . or - if something
- // alphanumeric follows directly. This leads to less text nodes,
- // which is good for performance.
+ // Continue with the same text node if the thing would become text
+ // anyway.
let mut s = self.s;
- s.eat_if(['.', '-']);
- s.eat_if(' ');
- if !s.at(char::is_alphanumeric) {
- break;
+ match s.eat() {
+ Some('/') if !s.at(['/', '*']) => {}
+ Some(' ') if s.at(char::is_alphanumeric) => {}
+ Some('-') if !s.at(['-', '?']) => {}
+ Some('.') if !s.at("..") => {}
+ Some('h') if !s.at("ttp://") && !s.at("ttps://") => {}
+ Some('@' | '#') if !s.at(is_id_start) => {}
+ _ => break,
}
self.s = s;
@@ -245,47 +267,9 @@ impl<'s> Tokens<'s> {
NodeKind::Text(self.s.from(start).into())
}
- fn whitespace(&mut self) -> NodeKind {
- self.s.uneat();
-
- // Count the number of newlines.
- let mut newlines = 0;
- while let Some(c) = self.s.eat() {
- if !c.is_whitespace() {
- self.s.uneat();
- break;
- }
-
- if is_newline(c) {
- if c == '\r' {
- self.s.eat_if('\n');
- }
- newlines += 1;
- }
- }
-
- NodeKind::Space { newlines }
- }
-
fn backslash(&mut self) -> NodeKind {
- let c = match self.s.peek() {
- Some(c) => c,
- None => return NodeKind::Linebreak { justified: false },
- };
-
- match c {
- // Backslash and comments.
- '\\' | '/' |
- // Parenthesis and hashtag.
- '[' | ']' | '{' | '}' | '#' |
- // Markup.
- '~' | '-' | '.' | ':' |
- '\'' | '"' | '*' | '_' | '`' | '$' | '=' |
- '<' | '>' | '@' => {
- self.s.expect(c);
- NodeKind::Escape(c)
- }
- 'u' if self.s.eat_if("u{") => {
+ match self.s.peek() {
+ Some('u') if self.s.eat_if("u{") => {
let sequence = self.s.eat_while(char::is_ascii_alphanumeric);
if self.s.eat_if('}') {
if let Some(c) = resolve_hex(sequence) {
@@ -298,26 +282,23 @@ impl<'s> Tokens<'s> {
}
} else {
self.terminated = false;
- NodeKind::Error(
- SpanPos::End,
- "expected closing brace".into(),
- )
+ NodeKind::Error(SpanPos::End, "expected closing brace".into())
}
}
// Linebreaks.
- c if c.is_whitespace() => NodeKind::Linebreak { justified: false },
- '+' => {
+ Some(c) if c.is_whitespace() => NodeKind::Linebreak,
+ None => NodeKind::Linebreak,
+
+ // Escapes.
+ Some(c) => {
self.s.expect(c);
- NodeKind::Linebreak { justified: true }
+ NodeKind::Escape(c)
}
-
- // Just the backslash.
- _ => NodeKind::Text('\\'.into()),
}
}
- fn hash(&mut self) -> NodeKind {
+ fn hash(&mut self, start: usize) -> NodeKind {
if self.s.at(is_id_start) {
let read = self.s.eat_while(is_id_continue);
match keyword(read) {
@@ -325,7 +306,7 @@ impl<'s> Tokens<'s> {
None => NodeKind::Ident(read.into()),
}
} else {
- NodeKind::Text('#'.into())
+ self.text(start)
}
}
@@ -343,19 +324,26 @@ impl<'s> Tokens<'s> {
}
}
- fn numbering(&mut self, start: usize, c: char) -> NodeKind {
- let number = if c != '.' {
- self.s.eat_while(char::is_ascii_digit);
- let read = self.s.from(start);
- if !self.s.eat_if('.') {
- return NodeKind::Text(self.s.from(start).into());
- }
- read.parse().ok()
- } else {
- None
- };
+ fn in_word(&self) -> bool {
+ let alphanumeric = |c: Option<char>| c.map_or(false, |c| c.is_alphanumeric());
+ let prev = self.s.scout(-2);
+ let next = self.s.peek();
+ alphanumeric(prev) && alphanumeric(next)
+ }
- NodeKind::EnumNumbering(number)
+ fn link(&mut self, start: usize) -> NodeKind {
+ #[rustfmt::skip]
+ self.s.eat_while(|c: char| matches!(c,
+ | '0' ..= '9'
+ | 'a' ..= 'z'
+ | 'A' ..= 'Z'
+ | '~' | '/' | '%' | '?' | '#' | '&' | '+' | '='
+ | '\'' | '.' | ',' | ';'
+ ));
+ if self.s.scout(-1) == Some('.') {
+ self.s.uneat();
+ }
+ NodeKind::Link(self.s.from(start).into())
}
fn raw(&mut self) -> NodeKind {
@@ -376,7 +364,6 @@ impl<'s> Tokens<'s> {
}
let start = self.s.cursor();
-
let mut found = 0;
while found < backticks {
match self.s.eat() {
@@ -394,10 +381,9 @@ impl<'s> Tokens<'s> {
self.s.get(start .. end),
)))
} else {
+ self.terminated = false;
let remaining = backticks - found;
let noun = if remaining == 1 { "backtick" } else { "backticks" };
-
- self.terminated = false;
NodeKind::Error(
SpanPos::End,
if found == 0 {
@@ -410,51 +396,38 @@ impl<'s> Tokens<'s> {
}
fn math(&mut self) -> NodeKind {
- let mut display = false;
- if self.s.eat_if('[') {
- display = true;
- }
-
- let start = self.s.cursor();
-
let mut escaped = false;
- let mut dollar = !display;
-
- let terminated = loop {
- match self.s.eat() {
- Some('$') if !escaped && dollar => break true,
- Some(']') if !escaped => dollar = true,
- Some(c) => {
- dollar = !display;
- escaped = c == '\\' && !escaped;
- }
- None => break false,
+ let formula = self.s.eat_until(|c| {
+ if c == '$' && !escaped {
+ true
+ } else {
+ escaped = c == '\\' && !escaped;
+ false
}
- };
+ });
+
+ let display = formula.len() >= 2
+ && formula.starts_with(char::is_whitespace)
+ && formula.ends_with(char::is_whitespace);
- let end = self.s.cursor()
- - match (terminated, display) {
- (false, _) => 0,
- (true, false) => 1,
- (true, true) => 2,
- };
-
- if terminated {
- NodeKind::Math(Arc::new(MathNode {
- formula: self.s.get(start .. end).into(),
- display,
- }))
+ if self.s.eat_if('$') {
+ NodeKind::Math(Arc::new(MathNode { formula: formula.into(), display }))
} else {
self.terminated = false;
- NodeKind::Error(
- SpanPos::End,
- if !display || (!escaped && dollar) {
- "expected closing dollar sign".into()
- } else {
- "expected closing bracket and dollar sign".into()
- },
- )
+ NodeKind::Error(SpanPos::End, "expected dollar sign".into())
+ }
+ }
+
+ fn numbering(&mut self, start: usize) -> NodeKind {
+ self.s.eat_while(char::is_ascii_digit);
+ let read = self.s.from(start);
+ if self.s.eat_if('.') {
+ if let Ok(number) = read.parse() {
+ return NodeKind::EnumNumbering(number);
+ }
}
+
+ self.text(start)
}
fn label(&mut self) -> NodeKind {
@@ -471,12 +444,59 @@ impl<'s> Tokens<'s> {
}
}
- fn reference(&mut self) -> NodeKind {
+ fn reference(&mut self, start: usize) -> NodeKind {
let label = self.s.eat_while(is_id_continue);
if !label.is_empty() {
NodeKind::Ref(label.into())
} else {
- NodeKind::Error(SpanPos::Full, "label cannot be empty".into())
+ self.text(start)
+ }
+ }
+
+ fn code(&mut self, start: usize, c: char) -> NodeKind {
+ match c {
+ // Parentheses.
+ '(' => NodeKind::LeftParen,
+ ')' => NodeKind::RightParen,
+
+ // Two-char operators.
+ '=' if self.s.eat_if('=') => NodeKind::EqEq,
+ '!' if self.s.eat_if('=') => NodeKind::ExclEq,
+ '<' if self.s.eat_if('=') => NodeKind::LtEq,
+ '>' if self.s.eat_if('=') => NodeKind::GtEq,
+ '+' if self.s.eat_if('=') => NodeKind::PlusEq,
+ '-' if self.s.eat_if('=') => NodeKind::HyphEq,
+ '*' if self.s.eat_if('=') => NodeKind::StarEq,
+ '/' if self.s.eat_if('=') => NodeKind::SlashEq,
+ '.' if self.s.eat_if('.') => NodeKind::Dots,
+ '=' if self.s.eat_if('>') => NodeKind::Arrow,
+
+ // Single-char operators.
+ ',' => NodeKind::Comma,
+ ';' => NodeKind::Semicolon,
+ ':' => NodeKind::Colon,
+ '+' => NodeKind::Plus,
+ '-' => NodeKind::Minus,
+ '*' => NodeKind::Star,
+ '/' => NodeKind::Slash,
+ '=' => NodeKind::Eq,
+ '<' => NodeKind::Lt,
+ '>' => NodeKind::Gt,
+ '.' if !self.s.at(char::is_ascii_digit) => NodeKind::Dot,
+
+ // Identifiers.
+ c if is_id_start(c) => self.ident(start),
+
+ // Numbers.
+ c if c.is_ascii_digit() || (c == '.' && self.s.at(char::is_ascii_digit)) => {
+ self.number(start, c)
+ }
+
+ // Strings.
+ '"' => self.string(),
+
+ // Invalid token.
+ _ => NodeKind::Unknown(self.s.from(start).into()),
}
}
@@ -543,18 +563,18 @@ impl<'s> Tokens<'s> {
}
}
-
fn string(&mut self) -> NodeKind {
let mut escaped = false;
- let string = resolve_string(self.s.eat_until(|c| {
+ let verbatim = self.s.eat_until(|c| {
if c == '"' && !escaped {
true
} else {
escaped = c == '\\' && !escaped;
false
}
- }));
+ });
+ let string = resolve_string(verbatim);
if self.s.eat_if('"') {
NodeKind::Str(string)
} else {
@@ -562,56 +582,6 @@ impl<'s> Tokens<'s> {
NodeKind::Error(SpanPos::End, "expected quote".into())
}
}
-
- fn line_comment(&mut self) -> NodeKind {
- self.s.eat_until(is_newline);
- if self.s.peek().is_none() {
- self.terminated = false;
- }
- NodeKind::LineComment
- }
-
- fn block_comment(&mut self) -> NodeKind {
- let mut state = '_';
- let mut depth = 1;
- self.terminated = false;
-
- // Find the first `*/` that does not correspond to a nested `/*`.
- while let Some(c) = self.s.eat() {
- state = match (state, c) {
- ('*', '/') => {
- depth -= 1;
- if depth == 0 {
- self.terminated = true;
- break;
- }
- '_'
- }
- ('/', '*') => {
- depth += 1;
- '_'
- }
- ('/', '/') => {
- self.line_comment();
- '_'
- }
- _ => c,
- }
- }
-
- NodeKind::BlockComment
- }
-
- fn in_word(&self) -> bool {
- let alphanumeric = |c: Option<char>| c.map_or(false, |c| c.is_alphanumeric());
- let prev = self.s.scout(-2);
- let next = self.s.peek();
- alphanumeric(prev) && alphanumeric(next)
- }
-
- fn maybe_in_url(&self) -> bool {
- self.mode == TokenMode::Markup && self.s.before().ends_with(":/")
- }
}
fn keyword(ident: &str) -> Option<NodeKind> {
@@ -872,14 +842,14 @@ mod tests {
#[test]
fn test_tokenize_text() {
// Test basic text.
- t!(Markup[" /"]: "hello" => Text("hello"));
- t!(Markup[" /"]: "hello-world" => Text("hello-world"));
+ t!(Markup[" /"]: "hello" => Text("hello"));
+ t!(Markup[" /"]: "reha-world" => Text("reha-world"));
// Test code symbols in text.
- t!(Markup[" /"]: "a():\"b" => Text("a():"), Quote { double: true }, Text("b"));
- t!(Markup[" /"]: ";:,|/+" => Text(";:,|"), Text("/+"));
+ t!(Markup[" /"]: "a():\"b" => Text("a()"), Colon, Quote { double: true }, Text("b"));
+ t!(Markup[" /"]: ";,|/+" => Text(";,|/+"));
t!(Markup[" /"]: "=-a" => Eq, Minus, Text("a"));
- t!(Markup[" "]: "#123" => Text("#"), Text("123"));
+ t!(Markup[" "]: "#123" => Text("#123"));
// Test text ends.
t!(Markup[""]: "hello " => Text("hello"), Space(0));
@@ -904,11 +874,9 @@ mod tests {
t!(Markup: r"\`" => Escape('`'));
t!(Markup: r"\$" => Escape('$'));
t!(Markup: r"\#" => Escape('#'));
-
- // Test unescapable symbols.
- t!(Markup[" /"]: r"\a" => Text(r"\"), Text("a"));
- t!(Markup[" /"]: r"\u" => Text(r"\"), Text("u"));
- t!(Markup[" /"]: r"\1" => Text(r"\"), Text("1"));
+ t!(Markup: r"\a" => Escape('a'));
+ t!(Markup: r"\u" => Escape('u'));
+ t!(Markup: r"\1" => Escape('1'));
// Test basic unicode escapes.
t!(Markup: r"\u{}" => Error(Full, "invalid unicode escape sequence"));
@@ -930,16 +898,15 @@ mod tests {
t!(Markup: "_" => Underscore);
t!(Markup[""]: "===" => Eq, Eq, Eq);
t!(Markup["a1/"]: "= " => Eq, Space(0));
- t!(Markup[" "]: r"\" => Linebreak { justified: false });
- t!(Markup[" "]: r"\+" => Linebreak { justified: true });
+ t!(Markup[" "]: r"\" => Linebreak);
t!(Markup: "~" => NonBreakingSpace);
t!(Markup["a1/"]: "-?" => Shy);
t!(Markup["a "]: r"a--" => Text("a"), EnDash);
t!(Markup["a1/"]: "- " => Minus, Space(0));
- t!(Markup[" "]: "." => EnumNumbering(None));
- t!(Markup[" "]: "1." => EnumNumbering(Some(1)));
- t!(Markup[" "]: "1.a" => EnumNumbering(Some(1)), Text("a"));
- t!(Markup[" /"]: "a1." => Text("a1"), EnumNumbering(None));
+ t!(Markup[" "]: "+" => Plus);
+ t!(Markup[" "]: "1." => EnumNumbering(1));
+ t!(Markup[" "]: "1.a" => EnumNumbering(1), Text("a"));
+ t!(Markup[" /"]: "a1." => Text("a1."));
}
#[test]
@@ -995,7 +962,7 @@ mod tests {
for (s, t) in list.clone() {
t!(Markup[" "]: format!("#{}", s) => t);
t!(Markup[" "]: format!("#{0}#{0}", s) => t, t);
- t!(Markup[" /"]: format!("# {}", s) => Text("#"), Space(0), Text(s));
+ t!(Markup[" /"]: format!("# {}", s) => Text(&format!("# {s}")));
}
for (s, t) in list {
@@ -1037,18 +1004,16 @@ mod tests {
t!(Markup: "$$" => Math("", false));
t!(Markup: "$x$" => Math("x", false));
t!(Markup: r"$\\$" => Math(r"\\", false));
- t!(Markup: "$[x + y]$" => Math("x + y", true));
- t!(Markup: r"$[\\]$" => Math(r"\\", true));
+ t!(Markup: r"$[\\]$" => Math(r"[\\]", false));
+ t!(Markup: "$ x + y $" => Math(" x + y ", true));
// Test unterminated.
- t!(Markup[""]: "$x" => Error(End, "expected closing dollar sign"));
- t!(Markup[""]: "$[x" => Error(End, "expected closing bracket and dollar sign"));
- t!(Markup[""]: "$[x]\n$" => Error(End, "expected closing bracket and dollar sign"));
+ t!(Markup[""]: "$x" => Error(End, "expected dollar sign"));
+ t!(Markup[""]: "$[x]\n" => Error(End, "expected dollar sign"));
// Test escape sequences.
- t!(Markup: r"$\$x$" => Math(r"\$x", false));
- t!(Markup: r"$[\\\]$]$" => Math(r"\\\]$", true));
- t!(Markup[""]: r"$[ ]\\$" => Error(End, "expected closing bracket and dollar sign"));
+ t!(Markup: r"$\$x$" => Math(r"\$x", false));
+ t!(Markup: r"$\ \$ $" => Math(r"\ \$ ", false));
}
#[test]
diff --git a/src/syntax/ast.rs b/src/syntax/ast.rs
index 10bee4e8..8d3696a8 100644
--- a/src/syntax/ast.rs
+++ b/src/syntax/ast.rs
@@ -63,9 +63,7 @@ impl Markup {
self.0.children().filter_map(|node| match node.kind() {
NodeKind::Space { newlines: (2 ..) } => Some(MarkupNode::Parbreak),
NodeKind::Space { .. } => Some(MarkupNode::Space),
- &NodeKind::Linebreak { justified } => {
- Some(MarkupNode::Linebreak { justified })
- }
+ NodeKind::Linebreak => Some(MarkupNode::Linebreak),
NodeKind::Text(s) => Some(MarkupNode::Text(s.clone())),
NodeKind::Escape(c) => Some(MarkupNode::Text((*c).into())),
NodeKind::NonBreakingSpace => Some(MarkupNode::Text('\u{00A0}'.into())),
@@ -76,6 +74,7 @@ impl Markup {
&NodeKind::Quote { double } => Some(MarkupNode::Quote { double }),
NodeKind::Strong => node.cast().map(MarkupNode::Strong),
NodeKind::Emph => node.cast().map(MarkupNode::Emph),
+ NodeKind::Link(url) => Some(MarkupNode::Link(url.clone())),
NodeKind::Raw(raw) => Some(MarkupNode::Raw(raw.as_ref().clone())),
NodeKind::Math(math) => Some(MarkupNode::Math(Spanned::new(
math.as_ref().clone(),
@@ -84,6 +83,7 @@ impl Markup {
NodeKind::Heading => node.cast().map(MarkupNode::Heading),
NodeKind::List => node.cast().map(MarkupNode::List),
NodeKind::Enum => node.cast().map(MarkupNode::Enum),
+ NodeKind::Desc => node.cast().map(MarkupNode::Desc),
NodeKind::Label(v) => Some(MarkupNode::Label(v.clone())),
NodeKind::Ref(v) => Some(MarkupNode::Ref(v.clone())),
_ => node.cast().map(MarkupNode::Expr),
@@ -96,8 +96,8 @@ impl Markup {
pub enum MarkupNode {
/// Whitespace containing less than two newlines.
Space,
- /// A forced line break: `\` or `\+` if justified.
- Linebreak { justified: bool },
+ /// A forced line break.
+ Linebreak,
/// A paragraph break: Two or more newlines.
Parbreak,
/// Plain text.
@@ -108,6 +108,8 @@ pub enum MarkupNode {
Strong(StrongNode),
/// Emphasized content: `_Emphasized_`.
Emph(EmphNode),
+ /// A hyperlink.
+ Link(EcoString),
/// A raw block with optional syntax highlighting: `` `...` ``.
Raw(RawNode),
/// A math formula: `$a^2 = b^2 + c^2$`.
@@ -116,8 +118,10 @@ pub enum MarkupNode {
Heading(HeadingNode),
/// An item in an unordered list: `- ...`.
List(ListNode),
- /// An item in an enumeration (ordered list): `1. ...`.
+ /// An item in an enumeration (ordered list): `+ ...` or `1. ...`.
Enum(EnumNode),
+ /// An item in a description list: `/ Term: Details.
+ Desc(DescNode),
/// A label.
Label(EcoString),
/// A reference.
@@ -170,8 +174,8 @@ pub struct RawNode {
pub struct MathNode {
/// The formula between the dollars / brackets.
pub formula: EcoString,
- /// Whether the formula is display-level, that is, it is surrounded by
- /// `$[..]$`.
+ /// Whether the formula is display-level, that is, it contains whitespace
+ /// after the starting dollar sign and before the ending dollar sign.
pub display: bool,
}
@@ -205,7 +209,7 @@ node! {
impl ListNode {
/// The contents of the list item.
pub fn body(&self) -> Markup {
- self.0.cast_first_child().expect("list node is missing body")
+ self.0.cast_first_child().expect("list item is missing body")
}
}
@@ -217,18 +221,36 @@ node! {
impl EnumNode {
/// The contents of the list item.
pub fn body(&self) -> Markup {
- self.0.cast_first_child().expect("enum node is missing body")
+ self.0.cast_first_child().expect("enum item is missing body")
}
/// The number, if any.
pub fn number(&self) -> Option<usize> {
+ self.0.children().find_map(|node| match node.kind() {
+ NodeKind::EnumNumbering(num) => Some(*num),
+ _ => None,
+ })
+ }
+}
+
+node! {
+ /// An item in a description list: `/ Term: Details.
+ DescNode: Desc
+}
+
+impl DescNode {
+ /// The term described by the list item.
+ pub fn term(&self) -> Markup {
self.0
- .children()
- .find_map(|node| match node.kind() {
- NodeKind::EnumNumbering(num) => Some(*num),
- _ => None,
- })
- .expect("enum node is missing number")
+ .cast_first_child()
+ .expect("description list item is missing term")
+ }
+
+ /// The description of the term.
+ pub fn body(&self) -> Markup {
+ self.0
+ .cast_last_child()
+ .expect("description list item is missing body")
}
}
diff --git a/src/syntax/highlight.rs b/src/syntax/highlight.rs
index 7f5ee083..de7c70a2 100644
--- a/src/syntax/highlight.rs
+++ b/src/syntax/highlight.rs
@@ -147,12 +147,12 @@ pub fn highlight_pre(text: &str, mode: TokenMode, theme: &Theme) -> String {
/// The syntax highlighting category of a node.
#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
pub enum Category {
+ /// A line or block comment.
+ Comment,
/// Any kind of bracket, parenthesis or brace.
Bracket,
/// Punctuation in code.
Punctuation,
- /// A line or block comment.
- Comment,
/// An easily typable shortcut to a unicode codepoint.
Shortcut,
/// An escape sequence.
@@ -161,14 +161,18 @@ pub enum Category {
Strong,
/// Emphasized text.
Emph,
+ /// A hyperlink.
+ Link,
/// Raw text or code.
Raw,
/// A math formula.
Math,
/// A section heading.
Heading,
- /// A list or enumeration.
+ /// A symbol of a list, enumeration, or description list.
List,
+ /// A term in a description list.
+ Term,
/// A label.
Label,
/// A reference.
@@ -204,66 +208,74 @@ impl Category {
i: usize,
) -> Option<Category> {
match child.kind() {
+ NodeKind::LineComment => Some(Category::Comment),
+ NodeKind::BlockComment => Some(Category::Comment),
NodeKind::LeftBrace => Some(Category::Bracket),
NodeKind::RightBrace => Some(Category::Bracket),
NodeKind::LeftBracket => Some(Category::Bracket),
NodeKind::RightBracket => Some(Category::Bracket),
NodeKind::LeftParen => Some(Category::Bracket),
NodeKind::RightParen => Some(Category::Bracket),
- NodeKind::Comma => Some(Category::Punctuation),
- NodeKind::Semicolon => Some(Category::Punctuation),
- NodeKind::Colon => Some(Category::Punctuation),
- NodeKind::Dot => Some(Category::Punctuation),
- NodeKind::LineComment => Some(Category::Comment),
- NodeKind::BlockComment => Some(Category::Comment),
+
+ NodeKind::Markup { .. } => match parent.kind() {
+ NodeKind::Desc
+ if parent
+ .children()
+ .take_while(|child| child.kind() != &NodeKind::Colon)
+ .find(|c| matches!(c.kind(), NodeKind::Markup { .. }))
+ .map_or(false, |ident| std::ptr::eq(ident, child)) =>
+ {
+ Some(Category::Term)
+ }
+ _ => None,
+ },
+ NodeKind::Space { .. } => None,
NodeKind::Linebreak { .. } => Some(Category::Shortcut),
+ NodeKind::Text(_) => None,
+ NodeKind::Escape(_) => Some(Category::Escape),
NodeKind::NonBreakingSpace => Some(Category::Shortcut),
NodeKind::Shy => Some(Category::Shortcut),
NodeKind::EnDash => Some(Category::Shortcut),
NodeKind::EmDash => Some(Category::Shortcut),
NodeKind::Ellipsis => Some(Category::Shortcut),
- NodeKind::Escape(_) => Some(Category::Escape),
+ NodeKind::Quote { .. } => None,
+ NodeKind::Star => match parent.kind() {
+ NodeKind::Strong => None,
+ _ => Some(Category::Operator),
+ },
+ NodeKind::Underscore => None,
NodeKind::Strong => Some(Category::Strong),
NodeKind::Emph => Some(Category::Emph),
+ NodeKind::Link(_) => Some(Category::Link),
NodeKind::Raw(_) => Some(Category::Raw),
NodeKind::Math(_) => Some(Category::Math),
NodeKind::Heading => Some(Category::Heading),
+ NodeKind::List => None,
+ NodeKind::Enum => None,
+ NodeKind::EnumNumbering(_) => Some(Category::List),
+ NodeKind::Desc => None,
+ NodeKind::Label(_) => Some(Category::Label),
+ NodeKind::Ref(_) => Some(Category::Ref),
+
+ NodeKind::Comma => Some(Category::Punctuation),
+ NodeKind::Semicolon => Some(Category::Punctuation),
+ NodeKind::Colon => match parent.kind() {
+ NodeKind::Desc => Some(Category::Term),
+ _ => Some(Category::Punctuation),
+ },
+ NodeKind::Plus => match parent.kind() {
+ NodeKind::Enum => Some(Category::List),
+ _ => Some(Category::Operator),
+ },
NodeKind::Minus => match parent.kind() {
NodeKind::List => Some(Category::List),
_ => Some(Category::Operator),
},
- NodeKind::EnumNumbering(_) => Some(Category::List),
- NodeKind::Label(_) => Some(Category::Label),
- NodeKind::Ref(_) => Some(Category::Ref),
- NodeKind::Not => Some(Category::Keyword),
- NodeKind::And => Some(Category::Keyword),
- NodeKind::Or => Some(Category::Keyword),
- NodeKind::Let => Some(Category::Keyword),
- NodeKind::Set => Some(Category::Keyword),
- NodeKind::Show => Some(Category::Keyword),
- NodeKind::Wrap => Some(Category::Keyword),
- NodeKind::If => Some(Category::Keyword),
- NodeKind::Else => Some(Category::Keyword),
- NodeKind::While => Some(Category::Keyword),
- NodeKind::For => Some(Category::Keyword),
- NodeKind::In => Some(Category::Keyword),
- NodeKind::As => Some(Category::Keyword),
- NodeKind::Break => Some(Category::Keyword),
- NodeKind::Continue => Some(Category::Keyword),
- NodeKind::Return => Some(Category::Keyword),
- NodeKind::Import => Some(Category::Keyword),
- NodeKind::From => Some(Category::Keyword),
- NodeKind::Include => Some(Category::Keyword),
- NodeKind::Plus => Some(Category::Operator),
- NodeKind::Star => match parent.kind() {
- NodeKind::Strong => None,
+ NodeKind::Slash => match parent.kind() {
+ NodeKind::Desc => Some(Category::List),
_ => Some(Category::Operator),
},
- NodeKind::Slash => Some(Category::Operator),
- NodeKind::PlusEq => Some(Category::Operator),
- NodeKind::HyphEq => Some(Category::Operator),
- NodeKind::StarEq => Some(Category::Operator),
- NodeKind::SlashEq => Some(Category::Operator),
+ NodeKind::Dot => Some(Category::Punctuation),
NodeKind::Eq => match parent.kind() {
NodeKind::Heading => None,
_ => Some(Category::Operator),
@@ -274,10 +286,34 @@ impl Category {
NodeKind::LtEq => Some(Category::Operator),
NodeKind::Gt => Some(Category::Operator),
NodeKind::GtEq => Some(Category::Operator),
+ NodeKind::PlusEq => Some(Category::Operator),
+ NodeKind::HyphEq => Some(Category::Operator),
+ NodeKind::StarEq => Some(Category::Operator),
+ NodeKind::SlashEq => Some(Category::Operator),
NodeKind::Dots => Some(Category::Operator),
NodeKind::Arrow => Some(Category::Operator),
+ NodeKind::Not => Some(Category::Keyword),
+ NodeKind::And => Some(Category::Keyword),
+ NodeKind::Or => Some(Category::Keyword),
NodeKind::None => Some(Category::None),
NodeKind::Auto => Some(Category::Auto),
+ NodeKind::Let => Some(Category::Keyword),
+ NodeKind::Set => Some(Category::Keyword),
+ NodeKind::Show => Some(Category::Keyword),
+ NodeKind::Wrap => Some(Category::Keyword),
+ NodeKind::If => Some(Category::Keyword),
+ NodeKind::Else => Some(Category::Keyword),
+ NodeKind::For => Some(Category::Keyword),
+ NodeKind::In => Some(Category::Keyword),
+ NodeKind::While => Some(Category::Keyword),
+ NodeKind::Break => Some(Category::Keyword),
+ NodeKind::Continue => Some(Category::Keyword),
+ NodeKind::Return => Some(Category::Keyword),
+ NodeKind::Import => Some(Category::Keyword),
+ NodeKind::Include => Some(Category::Keyword),
+ NodeKind::From => Some(Category::Keyword),
+ NodeKind::As => Some(Category::Keyword),
+
NodeKind::Ident(_) => match parent.kind() {
NodeKind::Markup { .. } => Some(Category::Interpolated),
NodeKind::FuncCall => Some(Category::Function),
@@ -302,15 +338,6 @@ impl Category {
NodeKind::Float(_) => Some(Category::Number),
NodeKind::Numeric(_, _) => Some(Category::Number),
NodeKind::Str(_) => Some(Category::String),
- NodeKind::Error(_, _) => Some(Category::Invalid),
- NodeKind::Unknown(_) => Some(Category::Invalid),
- NodeKind::Underscore => None,
- NodeKind::Markup { .. } => None,
- NodeKind::Space { .. } => None,
- NodeKind::Text(_) => None,
- NodeKind::Quote { .. } => None,
- NodeKind::List => None,
- NodeKind::Enum => None,
NodeKind::CodeBlock => None,
NodeKind::ContentBlock => None,
NodeKind::GroupExpr => None,
@@ -341,6 +368,9 @@ impl Category {
NodeKind::BreakExpr => None,
NodeKind::ContinueExpr => None,
NodeKind::ReturnExpr => None,
+
+ NodeKind::Error(_, _) => Some(Category::Invalid),
+ NodeKind::Unknown(_) => Some(Category::Invalid),
}
}
@@ -354,10 +384,12 @@ impl Category {
Self::Escape => "constant.character.escape.content.typst",
Self::Strong => "markup.bold.typst",
Self::Emph => "markup.italic.typst",
+ Self::Link => "markup.underline.link.typst",
Self::Raw => "markup.raw.typst",
Self::Math => "string.other.math.typst",
Self::Heading => "markup.heading.typst",
Self::List => "markup.list.typst",
+ Self::Term => "markup.bold.typst",
Self::Label => "entity.name.label.typst",
Self::Ref => "markup.other.reference.typst",
Self::Keyword => "keyword.typst",
diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs
index 89937f2c..6c6f690c 100644
--- a/src/syntax/mod.rs
+++ b/src/syntax/mod.rs
@@ -571,6 +571,14 @@ impl PartialEq for NodeData {
/// the parser.
#[derive(Debug, Clone, PartialEq)]
pub enum NodeKind {
+ /// A line comment, two slashes followed by inner contents, terminated with
+ /// a newline: `//<str>\n`.
+ LineComment,
+ /// A block comment, a slash and a star followed by inner contents,
+ /// terminated with a star and a slash: `/*<str>*/`.
+ ///
+ /// The comment can contain nested block comments.
+ BlockComment,
/// A left curly brace, starting a code block: `{`.
LeftBrace,
/// A right curly brace, terminating a code block: `}`.
@@ -585,23 +593,83 @@ pub enum NodeKind {
/// A right round parenthesis, terminating a grouped expression, collection,
/// argument or parameter list: `)`.
RightParen,
+
+ /// Markup of which all lines must have a minimal indentation.
+ ///
+ /// Notably, the number does not determine in which column the markup
+ /// started, but to the right of which column all markup elements must be,
+ /// so it is zero except for headings and lists.
+ Markup { min_indent: usize },
+ /// One or more whitespace characters. Single spaces are collapsed into text
+ /// nodes if they would otherwise be surrounded by text nodes.
+ ///
+ /// Also stores how many newlines are contained.
+ Space { newlines: usize },
+ /// A forced line break.
+ Linebreak,
+ /// Consecutive text without markup. While basic text with just single
+ /// spaces is collapsed into a single node, certain symbols that could
+ /// possibly be markup force text into multiple nodes.
+ Text(EcoString),
+ /// A slash and the letter "u" followed by a hexadecimal unicode entity
+ /// enclosed in curly braces: `\u{1F5FA}`.
+ Escape(char),
+ /// A non-breaking space: `~`.
+ NonBreakingSpace,
+ /// A soft hyphen: `-?`.
+ Shy,
+ /// An en-dash: `--`.
+ EnDash,
+ /// An em-dash: `---`.
+ EmDash,
+ /// An ellipsis: `...`.
+ Ellipsis,
+ /// A smart quote: `'` or `"`.
+ Quote { double: bool },
/// The strong text toggle, multiplication operator, and wildcard import
/// symbol: `*`.
Star,
/// Toggles emphasized text: `_`.
Underscore,
+ /// Strong content: `*Strong*`.
+ Strong,
+ /// Emphasized content: `_Emphasized_`.
+ Emph,
+ /// A hyperlink.
+ Link(EcoString),
+ /// A raw block with optional syntax highlighting: `` `...` ``.
+ Raw(Arc<RawNode>),
+ /// A math formula: `$x$`, `$[x^2]$`.
+ Math(Arc<MathNode>),
+ /// A section heading: `= Introduction`.
+ Heading,
+ /// An item in an unordered list: `- ...`.
+ List,
+ /// An item in an enumeration (ordered list): `+ ...` or `1. ...`.
+ Enum,
+ /// An explicit enumeration numbering: `23.`.
+ EnumNumbering(usize),
+ /// An item in a description list: `/ Term: Details.
+ Desc,
+ /// A label: `<label>`.
+ Label(EcoString),
+ /// A reference: `@label`.
+ Ref(EcoString),
+
/// A comma separator in a sequence: `,`.
Comma,
/// A semicolon terminating an expression: `;`.
Semicolon,
/// A colon between name / key and value in a dictionary, argument or
- /// parameter list: `:`.
+ /// parameter list, or between the term and body of a description list
+ /// term: `:`.
Colon,
- /// The unary plus and addition operator: `+`.
+ /// The unary plus and addition operator, and start of enum items: `+`.
Plus,
- /// The unary negation and subtraction operator: `-`.
+ /// The unary negation and subtraction operator, and start of list
+ /// items: `-`.
Minus,
- /// The division operator: `/`.
+ /// The division operator and start of description list items: `/`.
Slash,
/// A field access and method call operator: `.`.
Dot,
@@ -627,16 +695,16 @@ pub enum NodeKind {
StarEq,
/// The divide-assign operator: `/=`.
SlashEq,
+ /// The spread operator: `..`.
+ Dots,
+ /// An arrow between a closure's parameters and body: `=>`.
+ Arrow,
/// The `not` operator.
Not,
/// The `and` operator.
And,
/// The `or` operator.
Or,
- /// The spread operator: `..`.
- Dots,
- /// An arrow between a closure's parameters and body: `=>`.
- Arrow,
/// The none literal: `none`.
None,
/// The auto literal: `auto`.
@@ -673,60 +741,7 @@ pub enum NodeKind {
From,
/// The `as` keyword.
As,
- /// Markup of which all lines must have a minimal indentation.
- ///
- /// Notably, the number does not determine in which column the markup
- /// started, but to the right of which column all markup elements must be,
- /// so it is zero except for headings and lists.
- Markup { min_indent: usize },
- /// One or more whitespace characters. Single spaces are collapsed into text
- /// nodes if they would otherwise be surrounded by text nodes.
- ///
- /// Also stores how many newlines are contained.
- Space { newlines: usize },
- /// Consecutive text without markup. While basic text with just single
- /// spaces is collapsed into a single node, certain symbols that could
- /// possibly be markup force text into multiple nodes.
- Text(EcoString),
- /// A forced line break: `\` or `\+` if justified.
- Linebreak { justified: bool },
- /// A non-breaking space: `~`.
- NonBreakingSpace,
- /// A soft hyphen: `-?`.
- Shy,
- /// An en-dash: `--`.
- EnDash,
- /// An em-dash: `---`.
- EmDash,
- /// An ellipsis: `...`.
- Ellipsis,
- /// A smart quote: `'` or `"`.
- Quote { double: bool },
- /// A slash and the letter "u" followed by a hexadecimal unicode entity
- /// enclosed in curly braces: `\u{1F5FA}`.
- Escape(char),
- /// Strong content: `*Strong*`.
- Strong,
- /// Emphasized content: `_Emphasized_`.
- Emph,
- /// A raw block with optional syntax highlighting: `` `...` ``.
- Raw(Arc<RawNode>),
- /// A math formula: `$x$`, `$[x^2]$`.
- Math(Arc<MathNode>),
- /// A section heading: `= Introduction`.
- Heading,
- /// An item in an unordered list: `- ...`.
- List,
- /// An item in an enumeration (ordered list): `1. ...`.
- Enum,
- /// A numbering: `23.`.
- ///
- /// Can also exist without the number: `.`.
- EnumNumbering(Option<usize>),
- /// A label: `<label>`.
- Label(EcoString),
- /// A reference: `@label`.
- Ref(EcoString),
+
/// An identifier: `center`.
Ident(EcoString),
/// A boolean: `true`, `false`.
@@ -799,14 +814,7 @@ pub enum NodeKind {
ContinueExpr,
/// A return expression: `return x + 1`.
ReturnExpr,
- /// A line comment, two slashes followed by inner contents, terminated with
- /// a newline: `//<str>\n`.
- LineComment,
- /// A block comment, a slash and a star followed by inner contents,
- /// terminated with a star and a slash: `/*<str>*/`.
- ///
- /// The comment can contain nested block comments.
- BlockComment,
+
/// Tokens that appear in the wrong place.
Error(SpanPos, EcoString),
/// Unknown character sequences.
@@ -844,7 +852,7 @@ impl NodeKind {
}
}
- /// Whether changes _inside_ this node are safely encapuslated, so that only
+ /// Whether changes _inside_ this node are safely encapsulated, so that only
/// this node must be reparsed.
pub fn is_bounded(&self) -> bool {
match self {
@@ -860,7 +868,6 @@ impl NodeKind {
| Self::BlockComment
| Self::Space { .. }
| Self::Escape(_) => true,
- Self::Text(t) => t != "-" && !t.ends_with('.'),
_ => false,
}
}
@@ -868,14 +875,43 @@ impl NodeKind {
/// A human-readable name for the kind.
pub fn as_str(&self) -> &'static str {
match self {
+ Self::LineComment => "line comment",
+ Self::BlockComment => "block comment",
Self::LeftBrace => "opening brace",
Self::RightBrace => "closing brace",
Self::LeftBracket => "opening bracket",
Self::RightBracket => "closing bracket",
Self::LeftParen => "opening paren",
Self::RightParen => "closing paren",
+
+ Self::Markup { .. } => "markup",
+ Self::Space { newlines: (2 ..) } => "paragraph break",
+ Self::Space { .. } => "space",
+ Self::Linebreak => "linebreak",
+ Self::Text(_) => "text",
+ Self::Escape(_) => "escape sequence",
+ Self::NonBreakingSpace => "non-breaking space",
+ Self::Shy => "soft hyphen",
+ Self::EnDash => "en dash",
+ Self::EmDash => "em dash",
+ Self::Ellipsis => "ellipsis",
+ Self::Quote { double: false } => "single quote",
+ Self::Quote { double: true } => "double quote",
Self::Star => "star",
Self::Underscore => "underscore",
+ Self::Strong => "strong content",
+ Self::Emph => "emphasized content",
+ Self::Link(_) => "link",
+ Self::Raw(_) => "raw block",
+ Self::Math(_) => "math formula",
+ Self::Heading => "heading",
+ Self::List => "list item",
+ Self::Enum => "enumeration item",
+ Self::EnumNumbering(_) => "enumeration item numbering",
+ Self::Desc => "description list item",
+ Self::Label(_) => "label",
+ Self::Ref(_) => "reference",
+
Self::Comma => "comma",
Self::Semicolon => "semicolon",
Self::Colon => "colon",
@@ -894,11 +930,11 @@ impl NodeKind {
Self::HyphEq => "subtract-assign operator",
Self::StarEq => "multiply-assign operator",
Self::SlashEq => "divide-assign operator",
+ Self::Dots => "dots",
+ Self::Arrow => "arrow",
Self::Not => "operator `not`",
Self::And => "operator `and`",
Self::Or => "operator `or`",
- Self::Dots => "dots",
- Self::Arrow => "arrow",
Self::None => "`none`",
Self::Auto => "`auto`",
Self::Let => "keyword `let`",
@@ -909,7 +945,6 @@ impl NodeKind {
Self::Else => "keyword `else`",
Self::For => "keyword `for`",
Self::In => "keyword `in`",
- Self::As => "keyword `as`",
Self::While => "keyword `while`",
Self::Break => "keyword `break`",
Self::Continue => "keyword `continue`",
@@ -917,30 +952,8 @@ impl NodeKind {
Self::Import => "keyword `import`",
Self::Include => "keyword `include`",
Self::From => "keyword `from`",
- Self::Markup { .. } => "markup",
- Self::Space { newlines: (2 ..) } => "paragraph break",
- Self::Space { .. } => "space",
- Self::Linebreak { justified: false } => "linebreak",
- Self::Linebreak { justified: true } => "justified linebreak",
- Self::Text(_) => "text",
- Self::NonBreakingSpace => "non-breaking space",
- Self::Shy => "soft hyphen",
- Self::EnDash => "en dash",
- Self::EmDash => "em dash",
- Self::Ellipsis => "ellipsis",
- Self::Quote { double: false } => "single quote",
- Self::Quote { double: true } => "double quote",
- Self::Escape(_) => "escape sequence",
- Self::Strong => "strong content",
- Self::Emph => "emphasized content",
- Self::Raw(_) => "raw block",
- Self::Math(_) => "math formula",
- Self::List => "list item",
- Self::Heading => "heading",
- Self::Enum => "enumeration item",
- Self::EnumNumbering(_) => "enumeration item numbering",
- Self::Label(_) => "label",
- Self::Ref(_) => "reference",
+ Self::As => "keyword `as`",
+
Self::Ident(_) => "identifier",
Self::Bool(_) => "boolean",
Self::Int(_) => "integer",
@@ -977,8 +990,7 @@ impl NodeKind {
Self::BreakExpr => "`break` expression",
Self::ContinueExpr => "`continue` expression",
Self::ReturnExpr => "`return` expression",
- Self::LineComment => "line comment",
- Self::BlockComment => "block comment",
+
Self::Error(_, _) => "parse error",
Self::Unknown(text) => match text.as_str() {
"*/" => "end of block comment",
@@ -998,14 +1010,41 @@ impl Hash for NodeKind {
fn hash<H: Hasher>(&self, state: &mut H) {
std::mem::discriminant(self).hash(state);
match self {
+ Self::LineComment => {}
+ Self::BlockComment => {}
Self::LeftBrace => {}
Self::RightBrace => {}
Self::LeftBracket => {}
Self::RightBracket => {}
Self::LeftParen => {}
Self::RightParen => {}
+
+ Self::Markup { min_indent } => min_indent.hash(state),
+ Self::Space { newlines } => newlines.hash(state),
+ Self::Linebreak => {}
+ Self::Text(s) => s.hash(state),
+ Self::Escape(c) => c.hash(state),
+ Self::NonBreakingSpace => {}
+ Self::Shy => {}
+ Self::EnDash => {}
+ Self::EmDash => {}
+ Self::Ellipsis => {}
+ Self::Quote { double } => double.hash(state),
Self::Star => {}
Self::Underscore => {}
+ Self::Strong => {}
+ Self::Emph => {}
+ Self::Link(link) => link.hash(state),
+ Self::Raw(raw) => raw.hash(state),
+ Self::Math(math) => math.hash(state),
+ Self::Heading => {}
+ Self::List => {}
+ Self::Enum => {}
+ Self::EnumNumbering(num) => num.hash(state),
+ Self::Desc => {}
+ Self::Label(c) => c.hash(state),
+ Self::Ref(c) => c.hash(state),
+
Self::Comma => {}
Self::Semicolon => {}
Self::Colon => {}
@@ -1024,11 +1063,11 @@ impl Hash for NodeKind {
Self::HyphEq => {}
Self::StarEq => {}
Self::SlashEq => {}
+ Self::Dots => {}
+ Self::Arrow => {}
Self::Not => {}
Self::And => {}
Self::Or => {}
- Self::Dots => {}
- Self::Arrow => {}
Self::None => {}
Self::Auto => {}
Self::Let => {}
@@ -1039,7 +1078,6 @@ impl Hash for NodeKind {
Self::Else => {}
Self::For => {}
Self::In => {}
- Self::As => {}
Self::While => {}
Self::Break => {}
Self::Continue => {}
@@ -1047,27 +1085,8 @@ impl Hash for NodeKind {
Self::Import => {}
Self::Include => {}
Self::From => {}
- Self::Markup { min_indent } => min_indent.hash(state),
- Self::Space { newlines } => newlines.hash(state),
- Self::Linebreak { justified } => justified.hash(state),
- Self::Text(s) => s.hash(state),
- Self::NonBreakingSpace => {}
- Self::Shy => {}
- Self::EnDash => {}
- Self::EmDash => {}
- Self::Ellipsis => {}
- Self::Quote { double } => double.hash(state),
- Self::Escape(c) => c.hash(state),
- Self::Strong => {}
- Self::Emph => {}
- Self::Raw(raw) => raw.hash(state),
- Self::Math(math) => math.hash(state),
- Self::List => {}
- Self::Heading => {}
- Self::Enum => {}
- Self::EnumNumbering(num) => num.hash(state),
- Self::Label(c) => c.hash(state),
- Self::Ref(c) => c.hash(state),
+ Self::As => {}
+
Self::Ident(v) => v.hash(state),
Self::Bool(v) => v.hash(state),
Self::Int(v) => v.hash(state),
@@ -1104,8 +1123,7 @@ impl Hash for NodeKind {
Self::BreakExpr => {}
Self::ContinueExpr => {}
Self::ReturnExpr => {}
- Self::LineComment => {}
- Self::BlockComment => {}
+
Self::Error(pos, msg) => (pos, msg).hash(state),
Self::Unknown(text) => text.hash(state),
}
diff --git a/tests/ref/code/comment.png b/tests/ref/code/comment.png
index b1c630a9..5928dc87 100644
--- a/tests/ref/code/comment.png
+++ b/tests/ref/code/comment.png
Binary files differ
diff --git a/tests/ref/structure/desc.png b/tests/ref/structure/desc.png
new file mode 100644
index 00000000..e8b1d151
--- /dev/null
+++ b/tests/ref/structure/desc.png
Binary files differ
diff --git a/tests/ref/structure/enum.png b/tests/ref/structure/enum.png
index 0fcae99d..20bb08d4 100644
--- a/tests/ref/structure/enum.png
+++ b/tests/ref/structure/enum.png
Binary files differ
diff --git a/tests/ref/style/show-text.png b/tests/ref/style/show-text.png
index b88e1f30..e901b488 100644
--- a/tests/ref/style/show-text.png
+++ b/tests/ref/style/show-text.png
Binary files differ
diff --git a/tests/ref/text/escape.png b/tests/ref/text/escape.png
index 5ee9dbf8..96860fc5 100644
--- a/tests/ref/text/escape.png
+++ b/tests/ref/text/escape.png
Binary files differ
diff --git a/tests/ref/text/link.png b/tests/ref/text/link.png
index fd1a3f21..59e62bea 100644
--- a/tests/ref/text/link.png
+++ b/tests/ref/text/link.png
Binary files differ
diff --git a/tests/typ/code/comment.typ b/tests/typ/code/comment.typ
index db0a17a4..4a4dc7ab 100644
--- a/tests/typ/code/comment.typ
+++ b/tests/typ/code/comment.typ
@@ -25,11 +25,6 @@ Still comment.
E
---
-// Line comments have a special case for URLs.
-https://example.com \
-https:/* block comments don't ... */
-
----
// End should not appear without start.
// Error: 7-9 unexpected end of block comment
/* */ */
diff --git a/tests/typ/math/basic.typ b/tests/typ/math/basic.typ
index df05a567..77d2e535 100644
--- a/tests/typ/math/basic.typ
+++ b/tests/typ/math/basic.typ
@@ -5,15 +5,15 @@ The sum of $a$ and $b$ is $a + b$.
---
We will show that:
-$[ a^2 + b^2 = c^2 ]$
+$ a^2 + b^2 = c^2 $
---
Prove by induction:
-$[ \sum_{k=0}^n k = \frac{n(n+1)}{2} ]$
+$ \sum_{k=0}^n k = \frac{n(n+1)}{2} $
---
// Test that blackboard style looks nice.
-$[ f: \mathbb{N} \rightarrow \mathbb{R} ]$
+$ f: \mathbb{N} \rightarrow \mathbb{R} $
---
#set math(family: "IBM Plex Sans")
@@ -26,5 +26,5 @@ $a$
$\sqrt{x$
---
-// Error: 2:1 expected closing bracket and dollar sign
-$[a
+// Error: 2:1 expected dollar sign
+$a
diff --git a/tests/typ/structure/desc.typ b/tests/typ/structure/desc.typ
new file mode 100644
index 00000000..e12bbd16
--- /dev/null
+++ b/tests/typ/structure/desc.typ
@@ -0,0 +1,48 @@
+// Test description lists.
+
+---
+/
+No: list \
+/No: list
+
+---
+// Test with constructor.
+#desc(
+ (term: [One], body: [First]),
+ (term: [Two], body: [Second]),
+)
+
+---
+// Test joining.
+#for word in lorem(4).split().map(s => s.trim(".")) [
+ / #word: Latin stuff.
+]
+
+---
+// Test multiline.
+#set text(8pt)
+
+/ Fruit: A tasty, edible thing.
+/ Veggie:
+ An important energy source
+ for vegetarians.
+
+---
+// Test style change.
+#set text(8pt)
+
+/ First list: #lorem(4)
+#set desc(body-indent: 30pt)
+/ Second list: #lorem(4)
+
+---
+// Test grid like show rule.
+#show it: desc as table(
+ columns: 2,
+ padding: 3pt,
+ ..it.items.map(item => (emph(item.term), item.body)).flatten(),
+)
+
+/ A: One letter
+/ BB: Two letters
+/ CCC: Three letters
diff --git a/tests/typ/structure/enum.typ b/tests/typ/structure/enum.typ
index 7bd6fa88..eb1d9a9d 100644
--- a/tests/typ/structure/enum.typ
+++ b/tests/typ/structure/enum.typ
@@ -1,4 +1,4 @@
-// Test enums.
+// Test enumerations.
---
#enum[Embrace][Extend][Extinguish]
@@ -12,28 +12,28 @@
---
2. Second
1. First
- . Indented
+ + Indented
---
// Test automatic numbering in summed content.
#for i in range(5) {
- [. #roman(1 + i)]
+ [+ #roman(1 + i)]
}
---
// Test label pattern.
#set enum(label: "~ A:")
-. First
-. Second
+1. First
+ + Second
#set enum(label: "(*)")
-. A
-. B
-. C
++ A
++ B
++ C
#set enum(label: "i)")
-. A
-. B
++ A
++ B
---
// Test label closure.
@@ -47,12 +47,13 @@
---
#set enum(label: n => n > 1)
-. A
-. B
++ A
++ B
---
-// Lone dot is not a list.
-.
+// Lone plus is not an enum.
++
+No enum
---
// Error: 18-20 invalid pattern
diff --git a/tests/typ/structure/list.typ b/tests/typ/structure/list.typ
index 5c410a3f..9a021f31 100644
--- a/tests/typ/structure/list.typ
+++ b/tests/typ/structure/list.typ
@@ -1,4 +1,4 @@
-// Test lists.
+// Test unordered lists.
---
-
diff --git a/tests/typ/style/set.typ b/tests/typ/style/set.typ
index 830c6661..25dc1b81 100644
--- a/tests/typ/style/set.typ
+++ b/tests/typ/style/set.typ
@@ -44,9 +44,9 @@ Hello *{x}*
move(dy: -0.15em, image(path, width: 1em, height: 1em))
})
-. Monkey
-. Rhino
-. Tiger
++ Monkey
++ Rhino
++ Tiger
---
// Error: 11-25 set is only allowed directly in code and content blocks
diff --git a/tests/typ/text/escape.typ b/tests/typ/text/escape.typ
index ccaf78e9..95a8a027 100644
--- a/tests/typ/text/escape.typ
+++ b/tests/typ/text/escape.typ
@@ -2,15 +2,12 @@
---
// Escapable symbols.
-\\ \/ \[ \] \{ \} \# \* \_ \
-\= \~ \` \$ \" \' \< \> \@
+\\ \/ \[ \] \{ \} \# \* \_ \+ \= \~ \
+\` \$ \" \' \< \> \@ \( \) \A
// No need to escape.
( ) ;
-// Unescapable.
-\a \: \; \( \)
-
// Escaped comments.
\//
\/\* \*\/
diff --git a/tests/typ/text/justify.typ b/tests/typ/text/justify.typ
index aa0878d3..e61817a6 100644
--- a/tests/typ/text/justify.typ
+++ b/tests/typ/text/justify.typ
@@ -21,8 +21,8 @@ D
---
// Test forced justification with justified break.
-A B C \+
-D E F \+
+A B C #linebreak(justify: true)
+D E F #linebreak(justify: true)
---
// Test that justificating chinese text is at least a bit sensible.
diff --git a/tests/typ/text/linebreak.typ b/tests/typ/text/linebreak.typ
index 797e2f7c..d8449bcc 100644
--- a/tests/typ/text/linebreak.typ
+++ b/tests/typ/text/linebreak.typ
@@ -30,7 +30,7 @@ Trailing break \ \
---
// Test justified breaks.
#set par(justify: true)
-With a soft \+
-break you can force a break without #linebreak(justified: true)
-breaking justification. #linebreak(justified: false)
+With a soft #linebreak(justify: true)
+break you can force a break without #linebreak(justify: true)
+breaking justification. #linebreak(justify: false)
Nice!
diff --git a/tests/typ/text/link.typ b/tests/typ/text/link.typ
index 01c0b88d..99b380f0 100644
--- a/tests/typ/text/link.typ
+++ b/tests/typ/text/link.typ
@@ -1,8 +1,8 @@
// Test hyperlinking.
---
-// Link without body.
-#link("https://example.com/")
+// Link syntax.
+https://example.com/
// Link with body.
#link("https://typst.app/")[Some text text text]
@@ -10,11 +10,18 @@
// With line break.
This link appears #link("https://google.com/")[in the middle of] a paragraph.
-// Prefix is trimmed.
+// Certain prefixes are trimmed when using the `link` function.
Contact #link("mailto:hi@typst.app") or
call #link("tel:123") for more information.
---
+// Test that the period is trimmed.
+https://a.b.?q=%10#. \
+Wahttp://link \
+Nohttps:\//link \
+Nohttp\://comment
+
+---
// Styled with underline and color.
#set link(fill: rgb("283663"))
You could also make the
diff --git a/tests/typ/text/par.typ b/tests/typ/text/par.typ
index 059653c5..3c465ca8 100644
--- a/tests/typ/text/par.typ
+++ b/tests/typ/text/par.typ
@@ -39,7 +39,7 @@ Hello
fn main() {}
```
-$[ x + y = z ]$
+$ x + y = z $
- List