summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/eval/capture.rs39
-rw-r--r--src/eval/mod.rs25
-rw-r--r--src/eval/scope.rs3
-rw-r--r--src/eval/value.rs16
-rw-r--r--src/exec/context.rs20
-rw-r--r--src/exec/mod.rs91
-rw-r--r--src/exec/state.rs2
-rw-r--r--src/library/markup.rs170
-rw-r--r--src/library/mod.rs12
-rw-r--r--src/library/pad.rs2
-rw-r--r--src/library/shapes.rs4
-rw-r--r--src/library/stack.rs2
-rw-r--r--src/parse/lines.rs28
-rw-r--r--src/parse/mod.rs147
-rw-r--r--src/parse/parser.rs124
-rw-r--r--src/parse/scanner.rs2
-rw-r--r--src/parse/tokens.rs508
-rw-r--r--src/pretty.rs50
-rw-r--r--src/syntax/node.rs110
-rw-r--r--src/syntax/token.rs14
-rw-r--r--src/syntax/visit.rs15
21 files changed, 642 insertions, 742 deletions
diff --git a/src/eval/capture.rs b/src/eval/capture.rs
index 74da4747..64275e93 100644
--- a/src/eval/capture.rs
+++ b/src/eval/capture.rs
@@ -2,7 +2,7 @@ use std::rc::Rc;
use super::{Scope, Scopes, Value};
use crate::syntax::visit::{visit_expr, Visit};
-use crate::syntax::{Expr, Ident, Node};
+use crate::syntax::{Expr, Ident};
/// A visitor that captures variable slots.
#[derive(Debug)]
@@ -26,37 +26,20 @@ impl<'a> CapturesVisitor<'a> {
pub fn finish(self) -> Scope {
self.captures
}
-
- /// Find out whether the name is not locally defined and if so if it can be
- /// captured.
- fn process(&mut self, name: &str) {
- if self.internal.get(name).is_none() {
- if let Some(slot) = self.external.get(name) {
- self.captures.def_slot(name, Rc::clone(slot));
- }
- }
- }
}
impl<'ast> Visit<'ast> for CapturesVisitor<'_> {
- fn visit_node(&mut self, node: &'ast Node) {
- match node {
- Node::Text(_) => {}
- Node::Space => {}
- Node::Linebreak(_) => self.process(Node::LINEBREAK),
- Node::Parbreak(_) => self.process(Node::PARBREAK),
- Node::Strong(_) => self.process(Node::STRONG),
- Node::Emph(_) => self.process(Node::EMPH),
- Node::Heading(_) => self.process(Node::HEADING),
- Node::Raw(_) => self.process(Node::RAW),
- Node::Expr(expr) => self.visit_expr(expr),
- }
- }
-
fn visit_expr(&mut self, node: &'ast Expr) {
- match node {
- Expr::Ident(ident) => self.process(ident),
- expr => visit_expr(self, expr),
+ if let Expr::Ident(ident) = node {
+ // Find out whether the name is not locally defined and if so if it
+ // can be captured.
+ if self.internal.get(ident).is_none() {
+ if let Some(slot) = self.external.get(ident) {
+ self.captures.def_slot(ident.as_str(), Rc::clone(slot));
+ }
+ }
+ } else {
+ visit_expr(self, node);
}
}
diff --git a/src/eval/mod.rs b/src/eval/mod.rs
index c480ddfe..d1307b6d 100644
--- a/src/eval/mod.rs
+++ b/src/eval/mod.rs
@@ -218,24 +218,23 @@ pub trait Eval {
}
impl Eval for Tree {
- type Output = NodeMap;
+ type Output = ExprMap;
fn eval(&self, ctx: &mut EvalContext) -> Self::Output {
- let mut map = NodeMap::new();
-
- for node in self {
- let value = if let Some(call) = node.desugar() {
- call.eval(ctx)
- } else if let Node::Expr(expr) = node {
- expr.eval(ctx)
- } else {
- continue;
- };
+ struct ExprVisitor<'a, 'b> {
+ ctx: &'a mut EvalContext<'b>,
+ map: ExprMap,
+ }
- map.insert(node as *const _, value);
+ impl<'ast> Visit<'ast> for ExprVisitor<'_, '_> {
+ fn visit_expr(&mut self, node: &'ast Expr) {
+ self.map.insert(node as *const _, node.eval(self.ctx));
+ }
}
- map
+ let mut visitor = ExprVisitor { ctx, map: ExprMap::new() };
+ visitor.visit_tree(self);
+ visitor.map
}
}
diff --git a/src/eval/scope.rs b/src/eval/scope.rs
index e5afb6b0..3f7c4c62 100644
--- a/src/eval/scope.rs
+++ b/src/eval/scope.rs
@@ -33,8 +33,7 @@ impl<'a> Scopes<'a> {
/// Exit the topmost scope.
///
- /// # Panics
- /// Panics if no scope was entered.
+ /// This panics if no scope was entered.
pub fn exit(&mut self) {
self.top = self.scopes.pop().expect("no pushed scope");
}
diff --git a/src/eval/value.rs b/src/eval/value.rs
index e2ff5383..94f7f569 100644
--- a/src/eval/value.rs
+++ b/src/eval/value.rs
@@ -9,7 +9,7 @@ use super::EvalContext;
use crate::color::{Color, RgbaColor};
use crate::exec::ExecContext;
use crate::geom::{Angle, Length, Linear, Relative};
-use crate::syntax::{Node, Span, Spanned, Tree};
+use crate::syntax::{Expr, Span, Spanned, Tree};
/// A computational value.
#[derive(Debug, Clone, PartialEq)]
@@ -148,7 +148,7 @@ pub enum TemplateNode {
/// The syntax tree of the corresponding template expression.
tree: Rc<Tree>,
/// The evaluated expressions for the `tree`.
- map: NodeMap,
+ map: ExprMap,
},
/// A template that was converted from a string.
Str(String),
@@ -163,13 +163,13 @@ impl PartialEq for TemplateNode {
}
}
-/// A map from nodes to the values they evaluated to.
+/// A map from expressions to the values they evaluated to.
///
-/// The raw pointers point into the nodes contained in some [`Tree`]. Since the
-/// lifetime is erased, the tree could go out of scope while the hash map still
-/// lives. Although this could lead to lookup panics, it is not unsafe since the
-/// pointers are never dereferenced.
-pub type NodeMap = HashMap<*const Node, Value>;
+/// The raw pointers point into the expressions contained in some [`Tree`].
+/// Since the lifetime is erased, the tree could go out of scope while the hash
+/// map still lives. Although this could lead to lookup panics, it is not unsafe
+/// since the pointers are never dereferenced.
+pub type ExprMap = HashMap<*const Expr, Value>;
/// A reference-counted dynamic template node that can implement custom
/// behaviour.
diff --git a/src/exec/context.rs b/src/exec/context.rs
index 016b092a..63008260 100644
--- a/src/exec/context.rs
+++ b/src/exec/context.rs
@@ -1,13 +1,13 @@
use std::mem;
-use super::{Exec, FontFamily, State};
+use super::{Exec, ExecWithMap, FontFamily, State};
use crate::diag::{Diag, DiagSet, Pass};
-use crate::eval::TemplateValue;
+use crate::eval::{ExprMap, TemplateValue};
use crate::geom::{Align, Dir, Gen, GenAxis, Length, Linear, Sides, Size};
use crate::layout::{
AnyNode, PadNode, PageRun, ParChild, ParNode, StackChild, StackNode, Tree,
};
-use crate::syntax::Span;
+use crate::syntax::{self, Span};
/// The context for execution.
pub struct ExecContext {
@@ -48,12 +48,22 @@ impl ExecContext {
}
/// Execute a template and return the result as a stack node.
- pub fn exec_template(&mut self, template: &TemplateValue) -> StackNode {
+ pub fn exec_template_stack(&mut self, template: &TemplateValue) -> StackNode {
+ self.exec_stack(|ctx| template.exec(ctx))
+ }
+
+ /// Execute a tree with a map and return the result as a stack node.
+ pub fn exec_tree_stack(&mut self, tree: &syntax::Tree, map: &ExprMap) -> StackNode {
+ self.exec_stack(|ctx| tree.exec_with_map(ctx, map))
+ }
+
+ /// Execute something and return the result as a stack node.
+ pub fn exec_stack(&mut self, f: impl FnOnce(&mut Self)) -> StackNode {
let snapshot = self.state.clone();
let page = self.page.take();
let stack = mem::replace(&mut self.stack, StackBuilder::new(&self.state));
- template.exec(self);
+ f(self);
self.state = snapshot;
self.page = page;
diff --git a/src/exec/mod.rs b/src/exec/mod.rs
index 35e6b55c..a13d8c5e 100644
--- a/src/exec/mod.rs
+++ b/src/exec/mod.rs
@@ -9,10 +9,11 @@ pub use state::*;
use std::rc::Rc;
use crate::diag::Pass;
-use crate::eval::{NodeMap, TemplateFunc, TemplateNode, TemplateValue, Value};
-use crate::layout;
+use crate::eval::{ExprMap, TemplateFunc, TemplateNode, TemplateValue, Value};
+use crate::geom::{Dir, Gen};
+use crate::layout::{self, FixedNode, StackChild, StackNode};
use crate::pretty::pretty;
-use crate::syntax::*;
+use crate::syntax;
/// Execute a template to produce a layout tree.
pub fn exec(template: &TemplateValue, state: State) -> Pass<layout::Tree> {
@@ -33,30 +34,96 @@ pub trait Exec {
fn exec(&self, ctx: &mut ExecContext);
}
-/// Execute a node with a node map that applies to it.
+/// Execute a node with an expression map that applies to it.
pub trait ExecWithMap {
/// Execute the node.
- fn exec_with_map(&self, ctx: &mut ExecContext, map: &NodeMap);
+ fn exec_with_map(&self, ctx: &mut ExecContext, map: &ExprMap);
}
-impl ExecWithMap for Tree {
- fn exec_with_map(&self, ctx: &mut ExecContext, map: &NodeMap) {
+impl ExecWithMap for syntax::Tree {
+ fn exec_with_map(&self, ctx: &mut ExecContext, map: &ExprMap) {
for node in self {
node.exec_with_map(ctx, map);
}
}
}
-impl ExecWithMap for Node {
- fn exec_with_map(&self, ctx: &mut ExecContext, map: &NodeMap) {
+impl ExecWithMap for syntax::Node {
+ fn exec_with_map(&self, ctx: &mut ExecContext, map: &ExprMap) {
match self {
- Node::Text(text) => ctx.push_text(text),
- Node::Space => ctx.push_word_space(),
- _ => map[&(self as *const _)].exec(ctx),
+ Self::Text(text) => ctx.push_text(text),
+ Self::Space => ctx.push_word_space(),
+ Self::Linebreak(_) => ctx.linebreak(),
+ Self::Parbreak(_) => ctx.parbreak(),
+ Self::Strong(_) => ctx.state.font.strong ^= true,
+ Self::Emph(_) => ctx.state.font.emph ^= true,
+ Self::Raw(raw) => raw.exec(ctx),
+ Self::Heading(heading) => heading.exec_with_map(ctx, map),
+ Self::List(list) => list.exec_with_map(ctx, map),
+ Self::Expr(expr) => map[&(expr as *const _)].exec(ctx),
}
}
}
+impl Exec for syntax::RawNode {
+ fn exec(&self, ctx: &mut ExecContext) {
+ if self.block {
+ ctx.parbreak();
+ }
+
+ let snapshot = ctx.state.clone();
+ ctx.set_monospace();
+ ctx.push_text(&self.text);
+ ctx.state = snapshot;
+
+ if self.block {
+ ctx.parbreak();
+ }
+ }
+}
+
+impl ExecWithMap for syntax::HeadingNode {
+ fn exec_with_map(&self, ctx: &mut ExecContext, map: &ExprMap) {
+ let snapshot = ctx.state.clone();
+
+ let upscale = 1.6 - 0.1 * self.level as f64;
+ ctx.state.font.scale *= upscale;
+ ctx.state.font.strong = true;
+
+ self.body.exec_with_map(ctx, map);
+
+ ctx.state = snapshot;
+ ctx.parbreak();
+ }
+}
+
+impl ExecWithMap for syntax::ListNode {
+ fn exec_with_map(&self, ctx: &mut ExecContext, map: &ExprMap) {
+ ctx.parbreak();
+
+ let bullet = ctx.exec_stack(|ctx| ctx.push_text("•"));
+ let body = ctx.exec_tree_stack(&self.body, map);
+
+ let stack = StackNode {
+ dirs: Gen::new(Dir::TTB, ctx.state.lang.dir),
+ aspect: None,
+ children: vec![
+ StackChild::Any(bullet.into(), Gen::default()),
+ StackChild::Spacing(ctx.state.font.resolve_size() / 2.0),
+ StackChild::Any(body.into(), Gen::default()),
+ ],
+ };
+
+ ctx.push(FixedNode {
+ width: None,
+ height: None,
+ child: stack.into(),
+ });
+
+ ctx.parbreak();
+ }
+}
+
impl Exec for Value {
fn exec(&self, ctx: &mut ExecContext) {
match self {
diff --git a/src/exec/state.rs b/src/exec/state.rs
index 9a8971cc..aeeeaed5 100644
--- a/src/exec/state.rs
+++ b/src/exec/state.rs
@@ -7,7 +7,7 @@ use crate::geom::*;
use crate::layout::Fill;
use crate::paper::{Paper, PaperClass, PAPER_A4};
-/// The evaluation state.
+/// The execution state.
#[derive(Debug, Clone, PartialEq)]
pub struct State {
/// The current language-related settings.
diff --git a/src/library/markup.rs b/src/library/markup.rs
deleted file mode 100644
index 0a80fe74..00000000
--- a/src/library/markup.rs
+++ /dev/null
@@ -1,170 +0,0 @@
-use super::*;
-use crate::syntax::{HeadingNode, RawNode};
-
-/// `linebreak`: Start a new line.
-///
-/// # Syntax
-/// This function has dedicated syntax:
-/// ```typst
-/// This line ends here, \
-/// And a new one begins.
-/// ```
-///
-/// # Return value
-/// A template that inserts a line break.
-pub fn linebreak(_: &mut EvalContext, _: &mut FuncArgs) -> Value {
- Value::template(Node::LINEBREAK, move |ctx| {
- ctx.linebreak();
- })
-}
-
-/// `parbreak`: Start a new paragraph.
-///
-/// # Return value
-/// A template that inserts a paragraph break.
-pub fn parbreak(_: &mut EvalContext, _: &mut FuncArgs) -> Value {
- Value::template(Node::PARBREAK, move |ctx| {
- ctx.parbreak();
- })
-}
-
-/// `strong`: Strong text.
-///
-/// # Syntax
-/// This function has dedicated syntax.
-/// ```typst
-/// This is *important*!
-/// ```
-///
-/// # Positional parameters
-/// - Body: optional, of type `template`.
-///
-/// # Return value
-/// A template that flips the boldness of text. The effect is scoped to the
-/// body if present.
-pub fn strong(ctx: &mut EvalContext, args: &mut FuncArgs) -> Value {
- let body = args.eat::<TemplateValue>(ctx);
- Value::template(Node::STRONG, move |ctx| {
- let snapshot = ctx.state.clone();
- ctx.state.font.strong ^= true;
-
- if let Some(body) = &body {
- body.exec(ctx);
- ctx.state = snapshot;
- }
- })
-}
-
-/// `emph`: Emphasized text.
-///
-/// # Syntax
-/// This function has dedicated syntax.
-/// ```typst
-/// I would have _never_ thought so!
-/// ```
-///
-/// # Positional parameters
-/// - Body: optional, of type `template`.
-///
-/// # Return value
-/// A template that flips whether text is set in italics. The effect is scoped
-/// to the body if present.
-pub fn emph(ctx: &mut EvalContext, args: &mut FuncArgs) -> Value {
- let body = args.eat::<TemplateValue>(ctx);
- Value::template(Node::EMPH, move |ctx| {
- let snapshot = ctx.state.clone();
- ctx.state.font.emph ^= true;
-
- if let Some(body) = &body {
- body.exec(ctx);
- ctx.state = snapshot;
- }
- })
-}
-
-/// `heading`: A section heading.
-///
-/// # Syntax
-/// This function has dedicated syntax.
-/// ```typst
-/// = Section
-/// ...
-///
-/// == Subsection
-/// ...
-/// ```
-///
-/// # Positional parameters
-/// - Body, of type `template`.
-///
-/// # Named parameters
-/// - Section depth: `level`, of type `integer` between 1 and 6.
-///
-/// # Return value
-/// A template that sets the body as a section heading, that is, large and in
-/// bold.
-pub fn heading(ctx: &mut EvalContext, args: &mut FuncArgs) -> Value {
- let level = args.eat_named(ctx, HeadingNode::LEVEL).unwrap_or(1);
- let body = args
- .eat_expect::<TemplateValue>(ctx, HeadingNode::BODY)
- .unwrap_or_default();
-
- Value::template(Node::HEADING, move |ctx| {
- let snapshot = ctx.state.clone();
- let upscale = 1.6 - 0.1 * level as f64;
- ctx.state.font.scale *= upscale;
- ctx.state.font.strong = true;
-
- body.exec(ctx);
- ctx.state = snapshot;
-
- ctx.parbreak();
- })
-}
-
-/// `raw`: Raw text.
-///
-/// # Syntax
-/// This function has dedicated syntax:
-/// - For inline-level raw text:
-/// ```typst
-/// `...`
-/// ```
-/// - For block-level raw text:
-/// ````typst
-/// ```rust
-/// println!("Hello World!");
-/// ```
-/// ````
-///
-/// # Positional parameters
-/// - Text, of type `string`.
-///
-/// # Named parameters
-/// - Language for syntax highlighting: `lang`, of type `string`.
-/// - Whether the item is block level (split in its own paragraph): `block`, of
-/// type `boolean`.
-///
-/// # Return value
-/// A template that sets the text raw, that is, in monospace and optionally with
-/// syntax highlighting.
-pub fn raw(ctx: &mut EvalContext, args: &mut FuncArgs) -> Value {
- let text = args.eat_expect::<String>(ctx, RawNode::TEXT).unwrap_or_default();
- let _lang = args.eat_named::<String>(ctx, RawNode::LANG);
- let block = args.eat_named(ctx, RawNode::BLOCK).unwrap_or(false);
-
- Value::template(Node::RAW, move |ctx| {
- if block {
- ctx.parbreak();
- }
-
- let snapshot = ctx.state.clone();
- ctx.set_monospace();
- ctx.push_text(&text);
- ctx.state = snapshot;
-
- if block {
- ctx.parbreak();
- }
- })
-}
diff --git a/src/library/mod.rs b/src/library/mod.rs
index 6a46314c..f9e4f68a 100644
--- a/src/library/mod.rs
+++ b/src/library/mod.rs
@@ -8,7 +8,6 @@ mod basic;
mod font;
mod image;
mod lang;
-mod markup;
mod math;
mod pad;
mod page;
@@ -22,7 +21,6 @@ pub use align::*;
pub use basic::*;
pub use font::*;
pub use lang::*;
-pub use markup::*;
pub use math::*;
pub use pad::*;
pub use page::*;
@@ -38,20 +36,12 @@ use crate::eval::{EvalContext, FuncArgs, Scope, TemplateValue, Value};
use crate::exec::{Exec, FontFamily};
use crate::font::{FontStyle, FontWeight, VerticalFontMetric};
use crate::geom::*;
-use crate::syntax::{Node, Spanned};
+use crate::syntax::Spanned;
/// Construct a scope containing all standard library definitions.
pub fn new() -> Scope {
let mut std = Scope::new();
- // Syntax functions.
- std.def_func(Node::LINEBREAK, linebreak);
- std.def_func(Node::PARBREAK, parbreak);
- std.def_func(Node::STRONG, strong);
- std.def_func(Node::EMPH, emph);
- std.def_func(Node::HEADING, heading);
- std.def_func(Node::RAW, raw);
-
// Library functions.
std.def_func("align", align);
std.def_func("circle", circle);
diff --git a/src/library/pad.rs b/src/library/pad.rs
index 4b6236d0..6b944c9c 100644
--- a/src/library/pad.rs
+++ b/src/library/pad.rs
@@ -31,7 +31,7 @@ pub fn pad(ctx: &mut EvalContext, args: &mut FuncArgs) -> Value {
);
Value::template("pad", move |ctx| {
- let child = ctx.exec_template(&body).into();
+ let child = ctx.exec_template_stack(&body).into();
ctx.push(PadNode { padding, child });
})
}
diff --git a/src/library/shapes.rs b/src/library/shapes.rs
index a47203c4..287b1c10 100644
--- a/src/library/shapes.rs
+++ b/src/library/shapes.rs
@@ -61,7 +61,7 @@ fn rect_impl(
body: TemplateValue,
) -> Value {
Value::template(name, move |ctx| {
- let mut stack = ctx.exec_template(&body);
+ let mut stack = ctx.exec_template_stack(&body);
stack.aspect = aspect;
let fixed = FixedNode { width, height, child: stack.into() };
@@ -137,7 +137,7 @@ fn ellipse_impl(
// perfectly into the ellipse.
const PAD: f64 = 0.5 - SQRT_2 / 4.0;
- let mut stack = ctx.exec_template(&body);
+ let mut stack = ctx.exec_template_stack(&body);
stack.aspect = aspect;
let fixed = FixedNode {
diff --git a/src/library/stack.rs b/src/library/stack.rs
index b6f92e16..03bdc6a1 100644
--- a/src/library/stack.rs
+++ b/src/library/stack.rs
@@ -26,7 +26,7 @@ pub fn stack(ctx: &mut EvalContext, args: &mut FuncArgs) -> Value {
let children = children
.iter()
.map(|child| {
- let child = ctx.exec_template(child).into();
+ let child = ctx.exec_template_stack(child).into();
StackChild::Any(child, ctx.state.aligns)
})
.collect();
diff --git a/src/parse/lines.rs b/src/parse/lines.rs
index 8693af44..bbdedaa5 100644
--- a/src/parse/lines.rs
+++ b/src/parse/lines.rs
@@ -32,6 +32,8 @@ impl<'s> LineMap<'s> {
let start = self.line_starts.get(line_index)?;
let head = self.src.get(start.to_usize() .. pos.to_usize())?;
+
+ // TODO: What about tabs?
let column_index = head.chars().count();
Some(Location {
@@ -52,12 +54,14 @@ impl<'s> LineMap<'s> {
let line = self.src.get(line_start.to_usize() .. line_end)?;
- // Find the index in the line. For the first column, the index is always zero. For
- // other columns, we have to look at which byte the char directly before the
- // column in question ends. We can't do `nth(column_idx)` directly since the
- // column may be behind the last char.
+ // Find the index in the line. For the first column, the index is always
+ // zero. For other columns, we have to look at which byte the char
+ // directly before the column in question ends. We can't do
+ // `nth(column_idx)` directly since the column may be behind the last
+ // char.
let column_idx = location.column.checked_sub(1)? as usize;
let line_offset = if let Some(prev_idx) = column_idx.checked_sub(1) {
+ // TODO: What about tabs?
let (idx, prev) = line.char_indices().nth(prev_idx)?;
idx + prev.len_utf8()
} else {
@@ -68,6 +72,22 @@ impl<'s> LineMap<'s> {
}
}
+/// Determine the column at the end of the string.
+pub fn search_column(src: &str) -> usize {
+ let mut column = 0;
+ for c in src.chars().rev() {
+ if is_newline(c) {
+ break;
+ } else if c == '\t' {
+ // TODO: How many columns per tab?
+ column += 2;
+ } else {
+ column += 1;
+ }
+ }
+ column
+}
+
/// Whether this character denotes a newline.
pub fn is_newline(character: char) -> bool {
matches!(
diff --git a/src/parse/mod.rs b/src/parse/mod.rs
index 1b32b31e..048bcb1c 100644
--- a/src/parse/mod.rs
+++ b/src/parse/mod.rs
@@ -25,14 +25,32 @@ pub fn parse(src: &str) -> Pass<Tree> {
/// Parse a syntax tree.
fn tree(p: &mut Parser) -> Tree {
+ tree_while(p, |_| true)
+}
+
+/// Parse a syntax tree that stays right of the column at the start of the next
+/// non-whitespace token.
+fn tree_indented(p: &mut Parser) -> Tree {
+ p.skip_white();
+ let column = p.column(p.next_start());
+ tree_while(p, |p| match p.peek() {
+ Some(Token::Space(n)) if n >= 1 => p.column(p.next_end()) >= column,
+ _ => true,
+ })
+}
+
+/// Parse a syntax tree.
+fn tree_while(p: &mut Parser, mut f: impl FnMut(&mut Parser) -> bool) -> Tree {
// We keep track of whether we are at the start of a block or paragraph
- // to know whether headings are allowed.
+ // to know whether things like headings are allowed.
let mut at_start = true;
let mut tree = vec![];
- while !p.eof() {
+ while !p.eof() && f(p) {
if let Some(node) = node(p, &mut at_start) {
- if !matches!(node, Node::Parbreak(_) | Node::Space) {
- at_start = false;
+ match node {
+ Node::Space => {}
+ Node::Parbreak(_) => {}
+ _ => at_start = false,
}
tree.push(node);
}
@@ -57,10 +75,16 @@ fn node(p: &mut Parser, at_start: &mut bool) -> Option<Node> {
// Text.
Token::Text(text) => Node::Text(text.into()),
+ Token::Tilde => Node::Text("\u{00A0}".into()),
+ Token::HyphHyph => Node::Text("\u{2013}".into()),
+ Token::HyphHyphHyph => Node::Text("\u{2014}".into()),
+ Token::UnicodeEscape(t) => Node::Text(unicode_escape(p, t)),
// Markup.
+ Token::Backslash => Node::Linebreak(span),
Token::Star => Node::Strong(span),
Token::Underscore => Node::Emph(span),
+ Token::Raw(t) => raw(p, t),
Token::Hashtag => {
if *at_start {
return Some(heading(p));
@@ -68,10 +92,13 @@ fn node(p: &mut Parser, at_start: &mut bool) -> Option<Node> {
Node::Text(p.peek_src().into())
}
}
- Token::Tilde => Node::Text("\u{00A0}".into()),
- Token::Backslash => Node::Linebreak(span),
- Token::Raw(t) => raw(p, t),
- Token::UnicodeEscape(t) => Node::Text(unicode_escape(p, t)),
+ Token::Hyph => {
+ if *at_start {
+ return Some(list(p));
+ } else {
+ Node::Text(p.peek_src().into())
+ }
+ }
// Hashtag + keyword / identifier.
Token::Ident(_)
@@ -81,31 +108,27 @@ fn node(p: &mut Parser, at_start: &mut bool) -> Option<Node> {
| Token::For
| Token::Import
| Token::Include => {
- *at_start = false;
let stmt = matches!(token, Token::Let | Token::Import);
let group = if stmt { Group::Stmt } else { Group::Expr };
p.start_group(group, TokenMode::Code);
let expr = expr_with(p, true, 0);
if stmt && expr.is_some() && !p.eof() {
- p.expected_at("semicolon or line break", p.end());
+ p.expected_at("semicolon or line break", p.prev_end());
}
p.end_group();
// Uneat spaces we might have eaten eagerly.
- p.jump(p.end());
return expr.map(Node::Expr);
}
// Block.
Token::LeftBrace => {
- *at_start = false;
return Some(Node::Expr(block(p, false)));
}
// Template.
Token::LeftBracket => {
- *at_start = false;
return Some(Node::Expr(template(p)));
}
@@ -125,9 +148,37 @@ fn node(p: &mut Parser, at_start: &mut bool) -> Option<Node> {
Some(node)
}
+/// Handle a unicode escape sequence.
+fn unicode_escape(p: &mut Parser, token: UnicodeEscapeToken) -> String {
+ let span = p.peek_span();
+ let text = if let Some(c) = resolve::resolve_hex(token.sequence) {
+ c.to_string()
+ } else {
+ // Print out the escape sequence verbatim if it is invalid.
+ p.diag(error!(span, "invalid unicode escape sequence"));
+ p.peek_src().into()
+ };
+
+ if !token.terminated {
+ p.diag(error!(span.end, "expected closing brace"));
+ }
+
+ text
+}
+
+/// Handle a raw block.
+fn raw(p: &mut Parser, token: RawToken) -> Node {
+ let span = p.peek_span();
+ let raw = resolve::resolve_raw(span, token.text, token.backticks);
+ if !token.terminated {
+ p.diag(error!(p.peek_span().end, "expected backtick(s)"));
+ }
+ Node::Raw(raw)
+}
+
/// Parse a heading.
fn heading(p: &mut Parser) -> Node {
- let start = p.start();
+ let start = p.next_start();
p.assert(Token::Hashtag);
// Count depth.
@@ -137,49 +188,25 @@ fn heading(p: &mut Parser) -> Node {
}
if level > 6 {
- p.diag(warning!(start .. p.end(), "should not exceed depth 6"));
+ p.diag(warning!(start .. p.prev_end(), "should not exceed depth 6"));
level = 6;
}
- // Parse the heading contents.
- let mut tree = vec![];
- while p.check(|t| !matches!(t, Token::Space(n) if n >= 1)) {
- tree.extend(node(p, &mut false));
- }
+ let body = tree_indented(p);
Node::Heading(HeadingNode {
span: p.span(start),
level,
- contents: Rc::new(tree),
+ body: Rc::new(body),
})
}
-/// Handle a raw block.
-fn raw(p: &mut Parser, token: RawToken) -> Node {
- let span = p.peek_span();
- let raw = resolve::resolve_raw(span, token.text, token.backticks);
- if !token.terminated {
- p.diag(error!(p.peek_span().end, "expected backtick(s)"));
- }
- Node::Raw(raw)
-}
-
-/// Handle a unicode escape sequence.
-fn unicode_escape(p: &mut Parser, token: UnicodeEscapeToken) -> String {
- let span = p.peek_span();
- let text = if let Some(c) = resolve::resolve_hex(token.sequence) {
- c.to_string()
- } else {
- // Print out the escape sequence verbatim if it is invalid.
- p.diag(error!(span, "invalid unicode escape sequence"));
- p.get(span).into()
- };
-
- if !token.terminated {
- p.diag(error!(span.end, "expected closing brace"));
- }
-
- text
+/// Parse a single list item.
+fn list(p: &mut Parser) -> Node {
+ let start = p.next_start();
+ p.assert(Token::Hyph);
+ let body = tree_indented(p);
+ Node::List(ListNode { span: p.span(start), body })
}
/// Parse an expression.
@@ -195,7 +222,7 @@ fn expr(p: &mut Parser) -> Option<Expr> {
///
/// Stops parsing at operations with lower precedence than `min_prec`,
fn expr_with(p: &mut Parser, atomic: bool, min_prec: usize) -> Option<Expr> {
- let start = p.start();
+ let start = p.next_start();
let mut lhs = match p.eat_map(UnOp::from_token) {
Some(op) => {
let prec = op.precedence();
@@ -383,7 +410,7 @@ fn collection(p: &mut Parser) -> (Vec<CallArg>, bool) {
break;
}
- let behind = p.end();
+ let behind = p.prev_end();
if p.eat_if(Token::Comma) {
has_comma = true;
} else {
@@ -467,7 +494,7 @@ fn block(p: &mut Parser, scoping: bool) -> Expr {
if let Some(expr) = expr(p) {
exprs.push(expr);
if !p.eof() {
- p.expected_at("semicolon or line break", p.end());
+ p.expected_at("semicolon or line break", p.prev_end());
}
}
p.end_group();
@@ -506,14 +533,14 @@ fn call(p: &mut Parser, callee: Expr) -> Expr {
/// Parse the arguments to a function call.
fn args(p: &mut Parser) -> CallArgs {
- let start = p.start();
+ let start = p.next_start();
let items = collection(p).0;
CallArgs { span: p.span(start), items }
}
/// Parse a let expression.
fn expr_let(p: &mut Parser) -> Option<Expr> {
- let start = p.start();
+ let start = p.next_start();
p.assert(Token::Let);
let mut expr_let = None;
@@ -532,7 +559,7 @@ fn expr_let(p: &mut Parser) -> Option<Expr> {
init = expr(p);
} else if params.is_some() {
// Function definitions must have a body.
- p.expected_at("body", p.end());
+ p.expected_at("body", p.prev_end());
}
// Rewrite into a closure expression if it's a function definition.
@@ -558,7 +585,7 @@ fn expr_let(p: &mut Parser) -> Option<Expr> {
/// Parse an if expresion.
fn expr_if(p: &mut Parser) -> Option<Expr> {
- let start = p.start();
+ let start = p.next_start();
p.assert(Token::If);
let mut expr_if = None;
@@ -589,7 +616,7 @@ fn expr_if(p: &mut Parser) -> Option<Expr> {
/// Parse a while expresion.
fn expr_while(p: &mut Parser) -> Option<Expr> {
- let start = p.start();
+ let start = p.next_start();
p.assert(Token::While);
let mut expr_while = None;
@@ -608,7 +635,7 @@ fn expr_while(p: &mut Parser) -> Option<Expr> {
/// Parse a for expression.
fn expr_for(p: &mut Parser) -> Option<Expr> {
- let start = p.start();
+ let start = p.next_start();
p.assert(Token::For);
let mut expr_for = None;
@@ -643,7 +670,7 @@ fn for_pattern(p: &mut Parser) -> Option<ForPattern> {
/// Parse an import expression.
fn expr_import(p: &mut Parser) -> Option<Expr> {
- let start = p.start();
+ let start = p.next_start();
p.assert(Token::Import);
let mut expr_import = None;
@@ -657,7 +684,7 @@ fn expr_import(p: &mut Parser) -> Option<Expr> {
p.start_group(Group::Expr, TokenMode::Code);
let items = collection(p).0;
if items.is_empty() {
- p.expected_at("import items", p.end());
+ p.expected_at("import items", p.prev_end());
}
let idents = idents(p, items);
@@ -680,7 +707,7 @@ fn expr_import(p: &mut Parser) -> Option<Expr> {
/// Parse an include expression.
fn expr_include(p: &mut Parser) -> Option<Expr> {
- let start = p.start();
+ let start = p.next_start();
p.assert(Token::Include);
expr(p).map(|path| {
@@ -710,7 +737,7 @@ fn body(p: &mut Parser) -> Option<Expr> {
Some(Token::LeftBracket) => Some(template(p)),
Some(Token::LeftBrace) => Some(block(p, true)),
_ => {
- p.expected_at("body", p.end());
+ p.expected_at("body", p.prev_end());
None
}
}
diff --git a/src/parse/parser.rs b/src/parse/parser.rs
index 6269ad73..27346587 100644
--- a/src/parse/parser.rs
+++ b/src/parse/parser.rs
@@ -1,6 +1,7 @@
use std::fmt::{self, Debug, Formatter};
+use std::ops::Range;
-use super::{Scanner, TokenMode, Tokens};
+use super::{search_column, TokenMode, Tokens};
use crate::diag::{Diag, DiagSet};
use crate::syntax::{Pos, Span, Token};
@@ -17,10 +18,10 @@ pub struct Parser<'s> {
/// The peeked token.
/// (Same as `next` except if we are at the end of group, then `None`).
peeked: Option<Token<'s>>,
- /// The start position of the peeked token.
- next_start: Pos,
/// The end position of the last (non-whitespace if in code mode) token.
- last_end: Pos,
+ prev_end: usize,
+ /// The start position of the peeked token.
+ next_start: usize,
}
/// A logical group of tokens, e.g. `[...]`.
@@ -28,7 +29,7 @@ pub struct Parser<'s> {
struct GroupEntry {
/// The start position of the group. Used by `Parser::end_group` to return
/// The group's full span.
- pub start: Pos,
+ pub start: usize,
/// The kind of group this is. This decides which tokens will end the group.
/// For example, a [`Group::Paren`] will be ended by
/// [`Token::RightParen`].
@@ -59,12 +60,12 @@ impl<'s> Parser<'s> {
let next = tokens.next();
Self {
diags: DiagSet::new(),
- next,
tokens,
- last_end: Pos::ZERO,
- peeked: next,
- next_start: Pos::ZERO,
groups: vec![],
+ next,
+ peeked: next,
+ prev_end: 0,
+ next_start: 0,
}
}
@@ -76,9 +77,9 @@ impl<'s> Parser<'s> {
/// Eat the next token and add a diagnostic that it is not the expected
/// `thing`.
pub fn expected(&mut self, what: &str) {
- let before = self.next_start;
+ let before = self.next_start();
if let Some(found) = self.eat() {
- let after = self.last_end;
+ let after = self.prev_end();
self.diag(error!(
before .. after,
"expected {}, found {}",
@@ -86,20 +87,20 @@ impl<'s> Parser<'s> {
found.name(),
));
} else {
- self.expected_at(what, self.next_start);
+ self.expected_at(what, self.next_start());
}
}
/// Add a diagnostic that `what` was expected at the given position.
- pub fn expected_at(&mut self, what: &str, pos: Pos) {
- self.diag(error!(pos, "expected {}", what));
+ pub fn expected_at(&mut self, what: &str, pos: impl Into<Pos>) {
+ self.diag(error!(pos.into(), "expected {}", what));
}
/// Eat the next token and add a diagnostic that it is unexpected.
pub fn unexpected(&mut self) {
- let before = self.next_start;
+ let before = self.next_start();
if let Some(found) = self.eat() {
- let after = self.last_end;
+ let after = self.prev_end();
self.diag(error!(before .. after, "unexpected {}", found.name()));
}
}
@@ -110,11 +111,10 @@ impl<'s> Parser<'s> {
/// `eat()` and `peek()` return `None`. Parsing can only continue with
/// a matching call to `end_group`.
///
- /// # Panics
/// This panics if the next token does not start the given group.
pub fn start_group(&mut self, kind: Group, mode: TokenMode) {
self.groups.push(GroupEntry {
- start: self.next_start,
+ start: self.next_start(),
kind,
outer_mode: self.tokens.mode(),
});
@@ -133,7 +133,6 @@ impl<'s> Parser<'s> {
/// End the parsing of a group.
///
- /// # Panics
/// This panics if no group was started.
pub fn end_group(&mut self) -> Span {
let prev_mode = self.tokens.mode();
@@ -156,17 +155,16 @@ impl<'s> Parser<'s> {
self.bump();
rescan = false;
} else if required {
- self.diag(error!(self.next_start, "expected {}", end.name()));
+ self.diag(error!(self.next_start(), "expected {}", end.name()));
}
}
// Rescan the peeked token if the mode changed.
if rescan {
- self.tokens.jump(self.last_end);
- self.bump();
+ self.jump(self.prev_end());
}
- Span::new(group.start, self.last_end)
+ Span::new(group.start, self.prev_end())
}
/// The tokenization mode outside of the current group.
@@ -193,7 +191,7 @@ impl<'s> Parser<'s> {
/// Peek at the next token if it follows immediately after the last one
/// without any whitespace in between.
pub fn peek_direct(&self) -> Option<Token<'s>> {
- if self.next_start == self.last_end {
+ if self.next_start() == self.prev_end() {
self.peeked
} else {
None
@@ -204,15 +202,17 @@ impl<'s> Parser<'s> {
///
/// Has length zero if `peek()` returns `None`.
pub fn peek_span(&self) -> Span {
- Span::new(
- self.next_start,
- if self.eof() { self.next_start } else { self.tokens.pos() },
- )
+ self.peek_range().into()
}
/// Peek at the source of the next token.
pub fn peek_src(&self) -> &'s str {
- self.get(self.peek_span())
+ self.tokens.scanner().get(self.peek_range())
+ }
+
+ /// Peek at the source range (start and end index) of the next token.
+ pub fn peek_range(&self) -> Range<usize> {
+ self.next_start() .. self.next_end()
}
/// Checks whether the next token fulfills a condition.
@@ -255,11 +255,11 @@ impl<'s> Parser<'s> {
mapped
}
- /// Eat the next token and return its span.
+ /// Eat the next token and return its source range.
pub fn eat_span(&mut self) -> Span {
- let start = self.next_start;
+ let start = self.next_start();
self.eat();
- Span::new(start, self.last_end)
+ Span::new(start, self.prev_end())
}
/// Consume the next token if it is the given one and produce a diagnostic
@@ -267,7 +267,7 @@ impl<'s> Parser<'s> {
pub fn expect(&mut self, t: Token) -> bool {
let eaten = self.eat_if(t);
if !eaten {
- self.expected_at(t.name(), self.last_end);
+ self.expected_at(t.name(), self.prev_end());
}
eaten
}
@@ -290,45 +290,48 @@ impl<'s> Parser<'s> {
}
}
- /// The position at which the next token starts.
- pub fn start(&self) -> Pos {
- self.next_start
- }
-
- /// The position at which the last token ended.
+ /// The index at which the last token ended.
///
/// Refers to the end of the last _non-whitespace_ token in code mode.
- pub fn end(&self) -> Pos {
- self.last_end
+ pub fn prev_end(&self) -> usize {
+ self.prev_end
}
- /// The span from `start` to the end of the last token.
- pub fn span(&self, start: Pos) -> Span {
- Span::new(start, self.last_end)
+ /// The index at which the next token starts.
+ pub fn next_start(&self) -> usize {
+ self.next_start
}
- /// Jump to a position in the source string.
- pub fn jump(&mut self, pos: Pos) {
- self.tokens.jump(pos);
- self.bump();
+ /// The index at which the next token will end.
+ ///
+ /// Is the same as [`next_start()`][Self::next_start] if `peek()` returns
+ /// `None`.
+ pub fn next_end(&self) -> usize {
+ self.tokens.index()
}
- /// Slice a part out of the source string.
- pub fn get(&self, span: impl Into<Span>) -> &'s str {
- self.tokens.scanner().get(span.into().to_range())
+ /// Determine the column for the given index in the source.
+ pub fn column(&self, index: usize) -> usize {
+ search_column(self.tokens.scanner().get(.. index))
}
- /// The underlying scanner.
- pub fn scanner(&self) -> Scanner<'s> {
- let mut scanner = self.tokens.scanner().clone();
- scanner.jump(self.next_start.to_usize());
- scanner
+ /// The span from `start` to [`self.prev_end()`](Self::prev_end).
+ pub fn span(&self, start: impl Into<Pos>) -> Span {
+ Span::new(start, self.prev_end())
+ }
+
+ /// Jump to an index in the string.
+ ///
+ /// You need to know the correct column.
+ fn jump(&mut self, index: usize) {
+ self.tokens.jump(index);
+ self.bump();
}
/// Move to the next token.
fn bump(&mut self) {
- self.last_end = self.tokens.pos();
- self.next_start = self.tokens.pos();
+ self.prev_end = self.tokens.index();
+ self.next_start = self.tokens.index();
self.next = self.tokens.next();
if self.tokens.mode() == TokenMode::Code {
@@ -339,7 +342,7 @@ impl<'s> Parser<'s> {
Some(Token::BlockComment(_)) => true,
_ => false,
} {
- self.next_start = self.tokens.pos();
+ self.next_start = self.tokens.index();
self.next = self.tokens.next();
}
}
@@ -381,7 +384,8 @@ impl<'s> Parser<'s> {
impl Debug for Parser<'_> {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
- let s = self.scanner();
+ let mut s = self.tokens.scanner();
+ s.jump(self.next_start());
write!(f, "Parser({}|{})", s.eaten(), s.rest())
}
}
diff --git a/src/parse/scanner.rs b/src/parse/scanner.rs
index cc23a612..1f262e63 100644
--- a/src/parse/scanner.rs
+++ b/src/parse/scanner.rs
@@ -2,7 +2,7 @@ use std::fmt::{self, Debug, Formatter};
use std::slice::SliceIndex;
/// A featureful char-based scanner.
-#[derive(Clone)]
+#[derive(Copy, Clone)]
pub struct Scanner<'s> {
src: &'s str,
index: usize,
diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs
index fa86d2f1..74051801 100644
--- a/src/parse/tokens.rs
+++ b/src/parse/tokens.rs
@@ -38,20 +38,22 @@ impl<'s> Tokens<'s> {
self.mode = mode;
}
- /// The position in the string at which the last token ends and next token
+ /// The index in the string at which the last token ends and next token
/// will start.
- pub fn pos(&self) -> Pos {
- self.s.index().into()
+ pub fn index(&self) -> usize {
+ self.s.index()
}
- /// Jump to the given position.
- pub fn jump(&mut self, pos: Pos) {
- self.s.jump(pos.to_usize());
+ /// Jump to the given index in the string.
+ ///
+ /// You need to know the correct column.
+ pub fn jump(&mut self, index: usize) {
+ self.s.jump(index);
}
/// The underlying scanner.
- pub fn scanner(&self) -> &Scanner<'s> {
- &self.s
+ pub fn scanner(&self) -> Scanner<'s> {
+ self.s
}
}
@@ -62,126 +64,100 @@ impl<'s> Iterator for Tokens<'s> {
fn next(&mut self) -> Option<Self::Item> {
let start = self.s.index();
let c = self.s.eat()?;
-
- // This never loops. It just exists to allow breaking out of it.
- loop {
- // Common elements.
- return Some(match c {
- // Blocks and templates.
- '[' => Token::LeftBracket,
- ']' => Token::RightBracket,
- '{' => Token::LeftBrace,
- '}' => Token::RightBrace,
-
- // Headings, keywords, identifiers, colors.
- '#' => self.hash(start),
-
- // Whitespace.
- c if c.is_whitespace() => self.whitespace(c),
-
- // Comments.
- '/' if self.s.eat_if('/') => self.line_comment(),
- '/' if self.s.eat_if('*') => self.block_comment(),
- '*' if self.s.eat_if('/') => Token::Invalid(self.s.eaten_from(start)),
-
- _ => break,
- });
- }
-
- Some(match self.mode {
- TokenMode::Markup => match c {
- // Markup.
- '*' => Token::Star,
- '_' => Token::Underscore,
- '~' => Token::Tilde,
- '`' => self.raw(),
- '$' => self.math(),
- '\\' => self.backslash(),
-
- // Plain text.
- _ => self.text(start),
- },
-
- TokenMode::Code => match c {
- // Parens.
- '(' => Token::LeftParen,
- ')' => Token::RightParen,
-
- // Length two.
- '=' if self.s.eat_if('=') => Token::EqEq,
- '!' if self.s.eat_if('=') => Token::BangEq,
- '<' if self.s.eat_if('=') => Token::LtEq,
- '>' if self.s.eat_if('=') => Token::GtEq,
- '+' if self.s.eat_if('=') => Token::PlusEq,
- '-' if self.s.eat_if('=') => Token::HyphEq,
- '*' if self.s.eat_if('=') => Token::StarEq,
- '/' if self.s.eat_if('=') => Token::SlashEq,
- '.' if self.s.eat_if('.') => Token::Dots,
- '=' if self.s.eat_if('>') => Token::Arrow,
-
- // Length one.
- ',' => Token::Comma,
- ';' => Token::Semicolon,
- ':' => Token::Colon,
- '+' => Token::Plus,
- '-' => Token::Hyph,
- '*' => Token::Star,
- '/' => Token::Slash,
- '=' => Token::Eq,
- '<' => Token::Lt,
- '>' => Token::Gt,
-
- // Identifiers.
- c if is_id_start(c) => self.ident(start),
-
- // Numbers.
- c if c.is_ascii_digit()
- || (c == '.' && self.s.check(|n| n.is_ascii_digit())) =>
- {
- self.number(start, c)
- }
-
- // Strings.
- '"' => self.string(),
-
- _ => Token::Invalid(self.s.eaten_from(start)),
+ Some(match c {
+ // Blocks and templates.
+ '[' => Token::LeftBracket,
+ ']' => Token::RightBracket,
+ '{' => Token::LeftBrace,
+ '}' => Token::RightBrace,
+
+ // Headings, keywords, identifiers, colors.
+ '#' => self.hash(start),
+
+ // Whitespace.
+ c if c.is_whitespace() => self.whitespace(c),
+
+ // Comments.
+ '/' if self.s.eat_if('/') => self.line_comment(),
+ '/' if self.s.eat_if('*') => self.block_comment(),
+ '*' if self.s.eat_if('/') => Token::Invalid(self.s.eaten_from(start)),
+
+ // Other things.
+ _ => match self.mode {
+ TokenMode::Markup => self.markup(start, c),
+ TokenMode::Code => self.code(start, c),
},
})
}
}
impl<'s> Tokens<'s> {
- fn hash(&mut self, start: usize) -> Token<'s> {
- let read = self.s.eat_while(is_id_continue);
-
- match self.mode {
- TokenMode::Markup => {
- if read.is_empty() {
- return Token::Hashtag;
- }
-
- if let Some(token) = keyword(read) {
- return token;
- }
+ fn markup(&mut self, start: usize, c: char) -> Token<'s> {
+ match c {
+ // Markup.
+ '~' => Token::Tilde,
+ '*' => Token::Star,
+ '_' => Token::Underscore,
+ '\\' => self.backslash(),
+ '`' => self.raw(),
+ '$' => self.math(),
+ '-' => self.hyph(start),
+
+ // Plain text.
+ _ => self.text(start),
+ }
+ }
- if read.chars().next().map_or(false, is_id_start) {
- return Token::Ident(read);
- }
+ fn code(&mut self, start: usize, c: char) -> Token<'s> {
+ match c {
+ // Parens.
+ '(' => Token::LeftParen,
+ ')' => Token::RightParen,
+
+ // Length two.
+ '=' if self.s.eat_if('=') => Token::EqEq,
+ '!' if self.s.eat_if('=') => Token::BangEq,
+ '<' if self.s.eat_if('=') => Token::LtEq,
+ '>' if self.s.eat_if('=') => Token::GtEq,
+ '+' if self.s.eat_if('=') => Token::PlusEq,
+ '-' if self.s.eat_if('=') => Token::HyphEq,
+ '*' if self.s.eat_if('=') => Token::StarEq,
+ '/' if self.s.eat_if('=') => Token::SlashEq,
+ '.' if self.s.eat_if('.') => Token::Dots,
+ '=' if self.s.eat_if('>') => Token::Arrow,
+
+ // Length one.
+ ',' => Token::Comma,
+ ';' => Token::Semicolon,
+ ':' => Token::Colon,
+ '+' => Token::Plus,
+ '-' => Token::Hyph,
+ '*' => Token::Star,
+ '/' => Token::Slash,
+ '=' => Token::Eq,
+ '<' => Token::Lt,
+ '>' => Token::Gt,
+
+ // Identifiers.
+ c if is_id_start(c) => self.ident(start),
+
+ // Numbers.
+ c if c.is_ascii_digit()
+ || (c == '.' && self.s.check(|n| n.is_ascii_digit())) =>
+ {
+ self.number(start, c)
}
- TokenMode::Code => {
- if let Ok(color) = RgbaColor::from_str(read) {
- return Token::Color(color);
- }
- }
- }
+ // Strings.
+ '"' => self.string(),
- Token::Invalid(self.s.eaten_from(start))
+ _ => Token::Invalid(self.s.eaten_from(start)),
+ }
}
fn whitespace(&mut self, first: char) -> Token<'s> {
// Fast path for just a single space
- if first == ' ' && !self.s.check(|c| c.is_whitespace()) {
+ if first == ' ' && !self.s.check(char::is_whitespace) {
Token::Space(0)
} else {
self.s.uneat();
@@ -210,12 +186,13 @@ impl<'s> Tokens<'s> {
c if c.is_whitespace() => true,
// Comments.
'/' if self.s.check(|c| c == '/' || c == '*') => true,
- // Parenthesis and hashtag.
- '[' | ']' | '{' | '}' | '#' => true,
+ // Parentheses.
+ '[' | ']' | '{' | '}' => true,
// Markup.
- '*' | '_' | '=' | '~' | '`' | '$' => true,
+ '#' | '~' | '*' | '_' | '-' | '`' | '$' => true,
// Escaping.
'\\' => true,
+ // Just text.
_ => false,
} {
self.s.uneat();
@@ -226,6 +203,77 @@ impl<'s> Tokens<'s> {
Token::Text(self.s.eaten_from(start))
}
+ fn backslash(&mut self) -> Token<'s> {
+ if let Some(c) = self.s.peek() {
+ match c {
+ // Backslash and comments.
+ '\\' | '/' |
+ // Parenthesis and hashtag.
+ '[' | ']' | '{' | '}' | '#' |
+ // Markup.
+ '*' | '_' | '=' | '~' | '`' | '$' => {
+ let start = self.s.index();
+ self.s.eat_assert(c);
+ Token::Text(&self.s.eaten_from(start))
+ }
+ 'u' if self.s.peek_nth(1) == Some('{') => {
+ self.s.eat_assert('u');
+ self.s.eat_assert('{');
+ Token::UnicodeEscape(UnicodeEscapeToken {
+ // Allow more than `ascii_hexdigit` for better error recovery.
+ sequence: self.s.eat_while(|c| c.is_ascii_alphanumeric()),
+ terminated: self.s.eat_if('}'),
+ })
+ }
+ c if c.is_whitespace() => Token::Backslash,
+ _ => Token::Text("\\"),
+ }
+ } else {
+ Token::Backslash
+ }
+ }
+
+ fn hash(&mut self, start: usize) -> Token<'s> {
+ match self.mode {
+ TokenMode::Markup => {
+ if self.s.check(is_id_start) {
+ let read = self.s.eat_while(is_id_continue);
+ if let Some(keyword) = keyword(read) {
+ keyword
+ } else {
+ Token::Ident(read)
+ }
+ } else if self.s.check(|c| c != '#' && !c.is_whitespace()) {
+ Token::Text(self.s.eaten_from(start))
+ } else {
+ Token::Hashtag
+ }
+ }
+ TokenMode::Code => {
+ let read = self.s.eat_while(is_id_continue);
+ if let Ok(color) = RgbaColor::from_str(read) {
+ Token::Color(color)
+ } else {
+ Token::Invalid(self.s.eaten_from(start))
+ }
+ }
+ }
+ }
+
+ fn hyph(&mut self, start: usize) -> Token<'s> {
+ if self.s.eat_if('-') {
+ if self.s.eat_if('-') {
+ Token::HyphHyphHyph
+ } else {
+ Token::HyphHyph
+ }
+ } else if self.s.check(|c| !c.is_whitespace()) {
+ Token::Text(self.s.eaten_from(start))
+ } else {
+ Token::Hyph
+ }
+ }
+
fn raw(&mut self) -> Token<'s> {
let mut backticks = 1;
while self.s.eat_if('`') {
@@ -295,36 +343,6 @@ impl<'s> Tokens<'s> {
})
}
- fn backslash(&mut self) -> Token<'s> {
- if let Some(c) = self.s.peek() {
- match c {
- // Backslash and comments.
- '\\' | '/' |
- // Parenthesis and hashtag.
- '[' | ']' | '{' | '}' | '#' |
- // Markup.
- '*' | '_' | '=' | '~' | '`' | '$' => {
- let start = self.s.index();
- self.s.eat_assert(c);
- Token::Text(&self.s.eaten_from(start))
- }
- 'u' if self.s.peek_nth(1) == Some('{') => {
- self.s.eat_assert('u');
- self.s.eat_assert('{');
- Token::UnicodeEscape(UnicodeEscapeToken {
- // Allow more than `ascii_hexdigit` for better error recovery.
- sequence: self.s.eat_while(|c| c.is_ascii_alphanumeric()),
- terminated: self.s.eat_if('}'),
- })
- }
- c if c.is_whitespace() => Token::Backslash,
- _ => Token::Text("\\"),
- }
- } else {
- Token::Backslash
- }
- }
-
fn ident(&mut self, start: usize) -> Token<'s> {
self.s.eat_while(is_id_continue);
match self.s.eaten_from(start) {
@@ -474,6 +492,10 @@ mod tests {
use Token::{Ident, *};
use TokenMode::{Code, Markup};
+ const fn UnicodeEscape(sequence: &str, terminated: bool) -> Token {
+ Token::UnicodeEscape(UnicodeEscapeToken { sequence, terminated })
+ }
+
const fn Raw(text: &str, backticks: usize, terminated: bool) -> Token {
Token::Raw(RawToken { text, backticks, terminated })
}
@@ -482,18 +504,14 @@ mod tests {
Token::Math(MathToken { formula, display, terminated })
}
- const fn UnicodeEscape(sequence: &str, terminated: bool) -> Token {
- Token::UnicodeEscape(UnicodeEscapeToken { sequence, terminated })
+ const fn Color(r: u8, g: u8, b: u8, a: u8) -> Token<'static> {
+ Token::Color(RgbaColor { r, g, b, a })
}
const fn Str(string: &str, terminated: bool) -> Token {
Token::Str(StrToken { string, terminated })
}
- const fn Color(r: u8, g: u8, b: u8, a: u8) -> Token<'static> {
- Token::Color(RgbaColor { r, g, b, a })
- }
-
/// Building blocks for suffix testing.
///
/// We extend each test case with a collection of different suffixes to make
@@ -606,14 +624,91 @@ mod tests {
}
#[test]
+ fn test_tokenize_whitespace() {
+ // Test basic whitespace.
+ t!(Both["a1/"]: "" => );
+ t!(Both["a1/"]: " " => Space(0));
+ t!(Both["a1/"]: " " => Space(0));
+ t!(Both["a1/"]: "\t" => Space(0));
+ t!(Both["a1/"]: " \t" => Space(0));
+ t!(Both["a1/"]: "\u{202F}" => Space(0));
+
+ // Test newline counting.
+ t!(Both["a1/"]: "\n" => Space(1));
+ t!(Both["a1/"]: "\n " => Space(1));
+ t!(Both["a1/"]: " \n" => Space(1));
+ t!(Both["a1/"]: " \n " => Space(1));
+ t!(Both["a1/"]: "\r\n" => Space(1));
+ t!(Both["a1/"]: " \n\t \n " => Space(2));
+ t!(Both["a1/"]: "\n\r" => Space(2));
+ t!(Both["a1/"]: " \r\r\n \x0D" => Space(3));
+ }
+
+ #[test]
+ fn test_tokenize_text() {
+ // Test basic text.
+ t!(Markup[" /"]: "hello" => Text("hello"));
+ t!(Markup[" /"]: "hello-world" => Text("hello"), Text("-"), Text("world"));
+
+ // Test code symbols in text.
+ t!(Markup[" /"]: "a():\"b" => Text("a():\"b"));
+ t!(Markup[" /"]: ";:,|/+" => Text(";:,|/+"));
+ t!(Markup[" /"]: "#-a" => Text("#"), Text("-"), Text("a"));
+ t!(Markup[" "]: "#123" => Text("#"), Text("123"));
+
+ // Test text ends.
+ t!(Markup[""]: "hello " => Text("hello"), Space(0));
+ t!(Markup[""]: "hello~" => Text("hello"), Tilde);
+ }
+
+ #[test]
+ fn test_tokenize_escape_sequences() {
+ // Test escapable symbols.
+ t!(Markup: r"\\" => Text(r"\"));
+ t!(Markup: r"\/" => Text("/"));
+ t!(Markup: r"\[" => Text("["));
+ t!(Markup: r"\]" => Text("]"));
+ t!(Markup: r"\{" => Text("{"));
+ t!(Markup: r"\}" => Text("}"));
+ t!(Markup: r"\*" => Text("*"));
+ t!(Markup: r"\_" => Text("_"));
+ t!(Markup: r"\=" => Text("="));
+ t!(Markup: r"\~" => Text("~"));
+ t!(Markup: r"\`" => Text("`"));
+ t!(Markup: r"\$" => Text("$"));
+ t!(Markup: r"\#" => Text("#"));
+
+ // Test unescapable symbols.
+ t!(Markup[" /"]: r"\a" => Text(r"\"), Text("a"));
+ t!(Markup[" /"]: r"\u" => Text(r"\"), Text("u"));
+ t!(Markup[" /"]: r"\1" => Text(r"\"), Text("1"));
+ t!(Markup[" /"]: r"\:" => Text(r"\"), Text(":"));
+ t!(Markup[" /"]: r#"\""# => Text(r"\"), Text("\""));
+
+ // Test basic unicode escapes.
+ t!(Markup: r"\u{}" => UnicodeEscape("", true));
+ t!(Markup: r"\u{2603}" => UnicodeEscape("2603", true));
+ t!(Markup: r"\u{P}" => UnicodeEscape("P", true));
+
+ // Test unclosed unicode escapes.
+ t!(Markup[" /"]: r"\u{" => UnicodeEscape("", false));
+ t!(Markup[" /"]: r"\u{1" => UnicodeEscape("1", false));
+ t!(Markup[" /"]: r"\u{26A4" => UnicodeEscape("26A4", false));
+ t!(Markup[" /"]: r"\u{1Q3P" => UnicodeEscape("1Q3P", false));
+ t!(Markup: r"\u{1🏕}" => UnicodeEscape("1", false), Text("🏕"), RightBrace);
+ }
+
+ #[test]
fn test_tokenize_markup_symbols() {
// Test markup tokens.
- t!(Markup[" a1"]: "*" => Star);
- t!(Markup: "_" => Underscore);
- t!(Markup[""]: "###" => Hashtag, Hashtag, Hashtag);
- t!(Markup["a1/"]: "# " => Hashtag, Space(0));
- t!(Markup: "~" => Tilde);
- t!(Markup[" "]: r"\" => Backslash);
+ t!(Markup[" a1"]: "*" => Star);
+ t!(Markup: "_" => Underscore);
+ t!(Markup[""]: "###" => Hashtag, Hashtag, Hashtag);
+ t!(Markup["a1/"]: "# " => Hashtag, Space(0));
+ t!(Markup["a1/"]: "- " => Hyph, Space(0));
+ t!(Markup: "~" => Tilde);
+ t!(Markup[" "]: r"\" => Backslash);
+ t!(Markup["a "]: r"a--" => Text("a"), HyphHyph);
}
#[test]
@@ -654,71 +749,32 @@ mod tests {
#[test]
fn test_tokenize_keywords() {
- let keywords = [
+ // A list of a few (not all) keywords.
+ let list = [
("let", Let),
("if", If),
("else", Else),
("for", For),
("in", In),
- ("while", While),
- ("break", Break),
- ("continue", Continue),
- ("return", Return),
+ ("import", Import),
];
- for &(s, t) in &keywords {
+ for &(s, t) in &list {
t!(Markup[" "]: format!("#{}", s) => t);
t!(Markup[" "]: format!("#{0}#{0}", s) => t, t);
t!(Markup[" /"]: format!("# {}", s) => Token::Hashtag, Space(0), Text(s));
}
- for &(s, t) in &keywords {
+ for &(s, t) in &list {
t!(Code[" "]: s => t);
t!(Markup[" /"]: s => Text(s));
}
// Test simple identifier.
t!(Markup[" "]: "#letter" => Ident("letter"));
- t!(Markup[" "]: "#123" => Invalid("#123"));
- t!(Code[" /"]: "falser" => Ident("falser"));
- t!(Code[" /"]: "None" => Ident("None"));
- t!(Code[" /"]: "True" => Ident("True"));
- }
-
- #[test]
- fn test_tokenize_whitespace() {
- // Test basic whitespace.
- t!(Both["a1/"]: "" => );
- t!(Both["a1/"]: " " => Space(0));
- t!(Both["a1/"]: " " => Space(0));
- t!(Both["a1/"]: "\t" => Space(0));
- t!(Both["a1/"]: " \t" => Space(0));
- t!(Both["a1/"]: "\u{202F}" => Space(0));
-
- // Test newline counting.
- t!(Both["a1/"]: "\n" => Space(1));
- t!(Both["a1/"]: "\n " => Space(1));
- t!(Both["a1/"]: " \n" => Space(1));
- t!(Both["a1/"]: " \n " => Space(1));
- t!(Both["a1/"]: "\r\n" => Space(1));
- t!(Both["a1/"]: " \n\t \n " => Space(2));
- t!(Both["a1/"]: "\n\r" => Space(2));
- t!(Both["a1/"]: " \r\r\n \x0D" => Space(3));
- }
-
- #[test]
- fn test_tokenize_text() {
- // Test basic text.
- t!(Markup[" /"]: "hello" => Text("hello"));
- t!(Markup[" /"]: "hello-world" => Text("hello-world"));
-
- // Test code symbols in text.
- t!(Markup[" /"]: "a():\"b" => Text("a():\"b"));
- t!(Markup[" /"]: ";:,|/+-" => Text(";:,|/+-"));
-
- // Test text ends.
- t!(Markup[""]: "hello " => Text("hello"), Space(0));
- t!(Markup[""]: "hello~" => Text("hello"), Tilde);
+ t!(Code[" /"]: "falser" => Ident("falser"));
+ t!(Code[" /"]: "None" => Ident("None"));
+ t!(Code[" /"]: "True" => Ident("True"));
}
#[test]
@@ -765,43 +821,6 @@ mod tests {
}
#[test]
- fn test_tokenize_escape_sequences() {
- // Test escapable symbols.
- t!(Markup: r"\\" => Text(r"\"));
- t!(Markup: r"\/" => Text("/"));
- t!(Markup: r"\[" => Text("["));
- t!(Markup: r"\]" => Text("]"));
- t!(Markup: r"\{" => Text("{"));
- t!(Markup: r"\}" => Text("}"));
- t!(Markup: r"\*" => Text("*"));
- t!(Markup: r"\_" => Text("_"));
- t!(Markup: r"\=" => Text("="));
- t!(Markup: r"\~" => Text("~"));
- t!(Markup: r"\`" => Text("`"));
- t!(Markup: r"\$" => Text("$"));
- t!(Markup: r"\#" => Text("#"));
-
- // Test unescapable symbols.
- t!(Markup[" /"]: r"\a" => Text(r"\"), Text("a"));
- t!(Markup[" /"]: r"\u" => Text(r"\"), Text("u"));
- t!(Markup[" /"]: r"\1" => Text(r"\"), Text("1"));
- t!(Markup[" /"]: r"\:" => Text(r"\"), Text(":"));
- t!(Markup[" /"]: r#"\""# => Text(r"\"), Text("\""));
-
- // Test basic unicode escapes.
- t!(Markup: r"\u{}" => UnicodeEscape("", true));
- t!(Markup: r"\u{2603}" => UnicodeEscape("2603", true));
- t!(Markup: r"\u{P}" => UnicodeEscape("P", true));
-
- // Test unclosed unicode escapes.
- t!(Markup[" /"]: r"\u{" => UnicodeEscape("", false));
- t!(Markup[" /"]: r"\u{1" => UnicodeEscape("1", false));
- t!(Markup[" /"]: r"\u{26A4" => UnicodeEscape("26A4", false));
- t!(Markup[" /"]: r"\u{1Q3P" => UnicodeEscape("1Q3P", false));
- t!(Markup: r"\u{1🏕}" => UnicodeEscape("1", false), Text("🏕"), RightBrace);
- }
-
- #[test]
fn test_tokenize_idents() {
// Test valid identifiers.
t!(Code[" /"]: "x" => Ident("x"));
@@ -956,8 +975,7 @@ mod tests {
t!(Code: "1p%" => Invalid("1p"), Invalid("%"));
t!(Code: "1%%" => Percent(1.0), Invalid("%"));
- // Test invalid keyword.
- t!(Markup[" /"]: "#-" => Invalid("#-"));
+ // Test invalid color.
t!(Code[" /"]: r"#letter" => Invalid(r"#letter"));
}
}
diff --git a/src/pretty.rs b/src/pretty.rs
index 49f6fd82..9987e506 100644
--- a/src/pretty.rs
+++ b/src/pretty.rs
@@ -17,8 +17,8 @@ where
p.finish()
}
-/// Pretty print an item with a node map and return the resulting string.
-pub fn pretty_with_map<T>(item: &T, map: &NodeMap) -> String
+/// Pretty print an item with a expression map and return the resulting string.
+pub fn pretty_with_map<T>(item: &T, map: &ExprMap) -> String
where
T: PrettyWithMap + ?Sized,
{
@@ -33,10 +33,10 @@ pub trait Pretty {
fn pretty(&self, p: &mut Printer);
}
-/// Pretty print an item with a node map that applies to it.
+/// Pretty print an item with an expression map that applies to it.
pub trait PrettyWithMap {
/// Pretty print this item into the given printer.
- fn pretty_with_map(&self, p: &mut Printer, map: Option<&NodeMap>);
+ fn pretty_with_map(&self, p: &mut Printer, map: Option<&ExprMap>);
}
impl<T> Pretty for T
@@ -104,7 +104,7 @@ impl Write for Printer {
}
impl PrettyWithMap for Tree {
- fn pretty_with_map(&self, p: &mut Printer, map: Option<&NodeMap>) {
+ fn pretty_with_map(&self, p: &mut Printer, map: Option<&ExprMap>) {
for node in self {
node.pretty_with_map(p, map);
}
@@ -112,20 +112,21 @@ impl PrettyWithMap for Tree {
}
impl PrettyWithMap for Node {
- fn pretty_with_map(&self, p: &mut Printer, map: Option<&NodeMap>) {
+ fn pretty_with_map(&self, p: &mut Printer, map: Option<&ExprMap>) {
match self {
// TODO: Handle escaping.
Self::Text(text) => p.push_str(text),
Self::Space => p.push(' '),
- Self::Strong(_) => p.push('*'),
- Self::Emph(_) => p.push('_'),
Self::Linebreak(_) => p.push_str(r"\"),
Self::Parbreak(_) => p.push_str("\n\n"),
- Self::Heading(heading) => heading.pretty_with_map(p, map),
+ Self::Strong(_) => p.push('*'),
+ Self::Emph(_) => p.push('_'),
Self::Raw(raw) => raw.pretty(p),
+ Self::Heading(heading) => heading.pretty_with_map(p, map),
+ Self::List(list) => list.pretty_with_map(p, map),
Self::Expr(expr) => {
if let Some(map) = map {
- let value = &map[&(self as *const _)];
+ let value = &map[&(expr as *const _)];
value.pretty(p);
} else {
if expr.has_short_form() {
@@ -138,15 +139,6 @@ impl PrettyWithMap for Node {
}
}
-impl PrettyWithMap for HeadingNode {
- fn pretty_with_map(&self, p: &mut Printer, map: Option<&NodeMap>) {
- for _ in 0 .. self.level {
- p.push('#');
- }
- self.contents.pretty_with_map(p, map);
- }
-}
-
impl Pretty for RawNode {
fn pretty(&self, p: &mut Printer) {
// Find out how many backticks we need.
@@ -203,6 +195,23 @@ impl Pretty for RawNode {
}
}
+impl PrettyWithMap for HeadingNode {
+ fn pretty_with_map(&self, p: &mut Printer, map: Option<&ExprMap>) {
+ for _ in 0 .. self.level {
+ p.push('#');
+ }
+ p.push(' ');
+ self.body.pretty_with_map(p, map);
+ }
+}
+
+impl PrettyWithMap for ListNode {
+ fn pretty_with_map(&self, p: &mut Printer, map: Option<&ExprMap>) {
+ p.push_str("- ");
+ self.body.pretty_with_map(p, map);
+ }
+}
+
impl Pretty for Expr {
fn pretty(&self, p: &mut Printer) {
match self {
@@ -664,9 +673,8 @@ mod tests {
roundtrip("\\ ");
roundtrip("\n\n");
roundtrip("hi");
-
- // Heading.
roundtrip("# *Ok*");
+ roundtrip("- Ok");
// Raw.
roundtrip("``");
diff --git a/src/syntax/node.rs b/src/syntax/node.rs
index 5f76b56a..b4684d0b 100644
--- a/src/syntax/node.rs
+++ b/src/syntax/node.rs
@@ -17,70 +17,16 @@ pub enum Node {
Strong(Span),
/// Emphasized text was enabled / disabled: `_`.
Emph(Span),
- /// A section heading: `= Introduction`.
- Heading(HeadingNode),
/// A raw block with optional syntax highlighting: `` `...` ``.
Raw(RawNode),
+ /// A section heading: `= Introduction`.
+ Heading(HeadingNode),
+ /// A single list item: `- ...`.
+ List(ListNode),
/// An expression.
Expr(Expr),
}
-impl Node {
- // The names of the corresponding library functions.
- pub const LINEBREAK: &'static str = "linebreak";
- pub const PARBREAK: &'static str = "parbreak";
- pub const STRONG: &'static str = "strong";
- pub const EMPH: &'static str = "emph";
- pub const HEADING: &'static str = "heading";
- pub const RAW: &'static str = "raw";
-
- /// Desugar markup into a function call.
- pub fn desugar(&self) -> Option<CallExpr> {
- match *self {
- Self::Text(_) => None,
- Self::Space => None,
- Self::Linebreak(span) => Some(call(span, Self::LINEBREAK)),
- Self::Parbreak(span) => Some(call(span, Self::PARBREAK)),
- Self::Strong(span) => Some(call(span, Self::STRONG)),
- Self::Emph(span) => Some(call(span, Self::EMPH)),
- Self::Heading(ref heading) => Some(heading.desugar()),
- Self::Raw(ref raw) => Some(raw.desugar()),
- Self::Expr(_) => None,
- }
- }
-}
-
-/// A section heading: `= Introduction`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct HeadingNode {
- /// The source code location.
- pub span: Span,
- /// The section depth (numer of equals signs).
- pub level: usize,
- /// The contents of the heading.
- pub contents: Rc<Tree>,
-}
-
-impl HeadingNode {
- pub const LEVEL: &'static str = "level";
- pub const BODY: &'static str = "body";
-
- /// Desugar into a function call.
- pub fn desugar(&self) -> CallExpr {
- let Self { span, level, ref contents } = *self;
- let mut call = call(span, Node::HEADING);
- call.args.items.push(CallArg::Named(Named {
- name: ident(span, Self::LEVEL),
- expr: Expr::Int(span, level as i64),
- }));
- call.args.items.push(CallArg::Pos(Expr::Template(TemplateExpr {
- span,
- tree: Rc::clone(&contents),
- })));
- call
- }
-}
-
/// A raw block with optional syntax highlighting: `` `...` ``.
///
/// Raw blocks start with 1 or 3+ backticks and end with the same number of
@@ -158,38 +104,22 @@ pub struct RawNode {
pub block: bool,
}
-impl RawNode {
- pub const LANG: &'static str = "lang";
- pub const BLOCK: &'static str = "block";
- pub const TEXT: &'static str = "text";
-
- /// Desugar into a function call.
- pub fn desugar(&self) -> CallExpr {
- let Self { span, ref lang, ref text, block } = *self;
- let mut call = call(span, Node::RAW);
- if let Some(lang) = lang {
- call.args.items.push(CallArg::Named(Named {
- name: ident(span, Self::LANG),
- expr: Expr::Str(span, lang.string.clone()),
- }));
- }
- call.args.items.push(CallArg::Named(Named {
- name: ident(span, Self::BLOCK),
- expr: Expr::Bool(span, block),
- }));
- call.args.items.push(CallArg::Pos(Expr::Str(span, text.clone())));
- call
- }
-}
-
-fn call(span: Span, name: &str) -> CallExpr {
- CallExpr {
- span,
- callee: Box::new(Expr::Ident(Ident { span, string: name.into() })),
- args: CallArgs { span, items: vec![] },
- }
+/// A section heading: `= Introduction`.
+#[derive(Debug, Clone, PartialEq)]
+pub struct HeadingNode {
+ /// The source code location.
+ pub span: Span,
+ /// The section depth (numer of equals signs).
+ pub level: usize,
+ /// The contents of the heading.
+ pub body: Rc<Tree>,
}
-fn ident(span: Span, string: &str) -> Ident {
- Ident { span, string: string.into() }
+/// A single list item: `- ...`.
+#[derive(Debug, Clone, PartialEq)]
+pub struct ListNode {
+ /// The source code location.
+ pub span: Span,
+ /// The contents of the list item.
+ pub body: Tree,
}
diff --git a/src/syntax/token.rs b/src/syntax/token.rs
index 538d81b7..9098f176 100644
--- a/src/syntax/token.rs
+++ b/src/syntax/token.rs
@@ -24,6 +24,10 @@ pub enum Token<'s> {
Hashtag,
/// A tilde: `~`.
Tilde,
+ /// Two hyphens: `--`.
+ HyphHyph,
+ /// Three hyphens: `---`.
+ HyphHyphHyph,
/// A backslash followed by nothing or whitespace: `\`.
Backslash,
/// A comma: `,`.
@@ -103,15 +107,15 @@ pub enum Token<'s> {
Space(usize),
/// A consecutive non-markup string.
Text(&'s str),
+ /// A slash and the letter "u" followed by a hexadecimal unicode entity
+ /// enclosed in curly braces: `\u{1F5FA}`.
+ UnicodeEscape(UnicodeEscapeToken<'s>),
/// An arbitrary number of backticks followed by inner contents, terminated
/// with the same number of backticks: `` `...` ``.
Raw(RawToken<'s>),
/// One or two dollar signs followed by inner contents, terminated with the
/// same number of dollar signs.
Math(MathToken<'s>),
- /// A slash and the letter "u" followed by a hexadecimal unicode entity
- /// enclosed in curly braces: `\u{1F5FA}`.
- UnicodeEscape(UnicodeEscapeToken<'s>),
/// An identifier: `center`.
Ident(&'s str),
/// A boolean: `true`, `false`.
@@ -204,6 +208,8 @@ impl<'s> Token<'s> {
Self::Underscore => "underscore",
Self::Hashtag => "hashtag",
Self::Tilde => "tilde",
+ Self::HyphHyph => "en dash",
+ Self::HyphHyphHyph => "em dash",
Self::Backslash => "backslash",
Self::Comma => "comma",
Self::Semicolon => "semicolon",
@@ -242,9 +248,9 @@ impl<'s> Token<'s> {
Self::Using => "keyword `using`",
Self::Space(_) => "space",
Self::Text(_) => "text",
+ Self::UnicodeEscape(_) => "unicode escape sequence",
Self::Raw(_) => "raw block",
Self::Math(_) => "math formula",
- Self::UnicodeEscape(_) => "unicode escape sequence",
Self::Ident(_) => "identifier",
Self::Bool(_) => "boolean",
Self::Int(_) => "integer",
diff --git a/src/syntax/visit.rs b/src/syntax/visit.rs
index 40d8e664..86481d4e 100644
--- a/src/syntax/visit.rs
+++ b/src/syntax/visit.rs
@@ -52,16 +52,25 @@ visit! {
match node {
Node::Text(_) => {}
Node::Space => {}
- Node::Strong(_) => {}
Node::Linebreak(_) => {}
Node::Parbreak(_) => {}
+ Node::Strong(_) => {}
Node::Emph(_) => {}
- Node::Heading(heading) => v.visit_tree(&heading.contents),
Node::Raw(_) => {}
- Node::Expr(expr) => v.visit_expr(expr),
+ Node::Heading(n) => v.visit_heading(n),
+ Node::List(n) => v.visit_list(n),
+ Node::Expr(n) => v.visit_expr(n),
}
}
+ fn visit_heading(v, node: &HeadingNode) {
+ v.visit_tree(&node.body);
+ }
+
+ fn visit_list(v, node: &ListNode) {
+ v.visit_tree(&node.body);
+ }
+
fn visit_expr(v, node: &Expr) {
match node {
Expr::None(_) => {}
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-15 17:13:30 +0000