summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLaurenz <laurmaedje@gmail.com>2021-11-08 13:08:15 +0100
committerGitHub <noreply@github.com>2021-11-08 13:08:15 +0100
commitc6f8ad35f45248f1fd36ee00195966f1629c6ca7 (patch)
tree51faa3f6bbc56f75636823adeea135ed76e1b33b
parentea6ee3f667e922ed2f21b08719a45d2395787932 (diff)
parent38c5c362419c5eee7a4fdc0b43d3a9dfb339a6d2 (diff)
Merge pull request #46 from typst/parser-ng
Next Generation Parser
-rw-r--r--benches/oneshot.rs6
-rw-r--r--src/diag.rs2
-rw-r--r--src/eval/capture.rs161
-rw-r--r--src/eval/mod.rs222
-rw-r--r--src/eval/scope.rs4
-rw-r--r--src/eval/walk.rs41
-rw-r--r--src/geom/relative.rs2
-rw-r--r--src/lib.rs10
-rw-r--r--src/parse/mod.rs963
-rw-r--r--src/parse/parser.rs556
-rw-r--r--src/parse/resolve.rs21
-rw-r--r--src/parse/scanner.rs45
-rw-r--r--src/parse/tokens.rs618
-rw-r--r--src/source.rs18
-rw-r--r--src/syntax/ast.rs988
-rw-r--r--src/syntax/expr.rs584
-rw-r--r--src/syntax/ident.rs85
-rw-r--r--src/syntax/markup.rs78
-rw-r--r--src/syntax/mod.rs749
-rw-r--r--src/syntax/pretty.rs186
-rw-r--r--src/syntax/span.rs128
-rw-r--r--src/syntax/token.rs276
-rw-r--r--src/syntax/visit.rs263
-rw-r--r--tests/ref/markup/heading.pngbin6611 -> 6406 bytes
-rw-r--r--tests/ref/markup/math.pngbin0 -> 2448 bytes
-rw-r--r--tests/typ/code/closure.typ46
-rw-r--r--tests/typ/code/import.typ2
-rw-r--r--tests/typ/code/let.typ2
-rw-r--r--tests/typ/code/spread.typ6
-rw-r--r--tests/typ/markup/heading.typ4
-rw-r--r--tests/typ/markup/math.typ12
-rw-r--r--tests/typ/markup/raw.typ2
-rw-r--r--tests/typeset.rs14
33 files changed, 3391 insertions, 2703 deletions
diff --git a/benches/oneshot.rs b/benches/oneshot.rs
index 6ce81a63..bb385688 100644
--- a/benches/oneshot.rs
+++ b/benches/oneshot.rs
@@ -6,7 +6,7 @@ use typst::eval::eval;
use typst::layout::layout;
use typst::loading::MemLoader;
use typst::parse::{parse, Scanner, TokenMode, Tokens};
-use typst::source::{SourceFile, SourceId};
+use typst::source::SourceId;
use typst::Context;
const SRC: &str = include_str!("bench.typ");
@@ -48,12 +48,12 @@ fn bench_tokenize(iai: &mut Iai) {
}
fn bench_parse(iai: &mut Iai) {
- iai.run(|| parse(&SourceFile::detached(SRC)));
+ iai.run(|| parse(SRC));
}
fn bench_eval(iai: &mut Iai) {
let (mut ctx, id) = context();
- let ast = ctx.parse(id).unwrap();
+ let ast = ctx.sources.get(id).ast().unwrap();
iai.run(|| eval(&mut ctx, id, &ast).unwrap());
}
diff --git a/src/diag.rs b/src/diag.rs
index f0455310..d284687e 100644
--- a/src/diag.rs
+++ b/src/diag.rs
@@ -100,7 +100,7 @@ impl<T> Trace<T> for TypResult<T> {
{
self.map_err(|mut errors| {
for error in errors.iter_mut() {
- if !span.contains(error.span) {
+ if !span.surrounds(error.span) {
error.trace.push(Spanned::new(make_point(), span));
}
}
diff --git a/src/eval/capture.rs b/src/eval/capture.rs
index f0a2b729..e47831df 100644
--- a/src/eval/capture.rs
+++ b/src/eval/capture.rs
@@ -1,8 +1,8 @@
use std::rc::Rc;
use super::{Scope, Scopes, Value};
-use crate::syntax::visit::{immutable::visit_expr, Visit};
-use crate::syntax::{Expr, Ident};
+use crate::syntax::ast::{ClosureParam, Expr, Ident, Imports, TypedNode};
+use crate::syntax::RedRef;
/// A visitor that captures variable slots.
pub struct CapturesVisitor<'a> {
@@ -25,32 +25,153 @@ impl<'a> CapturesVisitor<'a> {
pub fn finish(self) -> Scope {
self.captures
}
-}
-impl<'ast> Visit<'ast> for CapturesVisitor<'_> {
- fn visit_expr(&mut self, node: &'ast Expr) {
- if let Expr::Ident(ident) = node {
- // Find out whether the name is not locally defined and if so if it
- // can be captured.
- if self.internal.get(ident).is_none() {
- if let Some(slot) = self.external.get(ident) {
- self.captures.def_slot(ident.as_str(), Rc::clone(slot));
- }
+ /// Bind a new internal variable.
+ pub fn bind(&mut self, ident: Ident) {
+ self.internal.def_mut(ident.take(), Value::None);
+ }
+
+ /// Capture a variable if it isn't internal.
+ pub fn capture(&mut self, ident: Ident) {
+ if self.internal.get(&ident).is_none() {
+ if let Some(slot) = self.external.get(&ident) {
+ self.captures.def_slot(ident.take(), Rc::clone(slot));
}
- } else {
- visit_expr(self, node);
}
}
- fn visit_binding(&mut self, ident: &'ast Ident) {
- self.internal.def_mut(ident.as_str(), Value::None);
+ /// Visit any node and collect all captured variables.
+ pub fn visit(&mut self, node: RedRef) {
+ match node.cast() {
+ // Every identifier is a potential variable that we need to capture.
+ // Identifiers that shouldn't count as captures because they
+ // actually bind a new name are handled further below (individually
+ // through the expressions that contain them).
+ Some(Expr::Ident(ident)) => self.capture(ident),
+
+ // A closure contains parameter bindings, which are bound before the
+ // body is evaluated. Take must be taken so that the default values
+ // of named parameters cannot access previous parameter bindings.
+ Some(Expr::Closure(expr)) => {
+ for param in expr.params() {
+ if let ClosureParam::Named(named) = param {
+ self.visit(named.expr().as_red());
+ }
+ }
+
+ for param in expr.params() {
+ match param {
+ ClosureParam::Pos(ident) => self.bind(ident),
+ ClosureParam::Named(named) => self.bind(named.name()),
+ ClosureParam::Sink(ident) => self.bind(ident),
+ }
+ }
+
+ self.visit(expr.body().as_red());
+ }
+
+ // A let expression contains a binding, but that binding is only
+ // active after the body is evaluated.
+ Some(Expr::Let(expr)) => {
+ if let Some(init) = expr.init() {
+ self.visit(init.as_red());
+ }
+ self.bind(expr.binding());
+ }
+
+ // A for loop contains one or two bindings in its pattern. These are
+ // active after the iterable is evaluated but before the body is
+ // evaluated.
+ Some(Expr::For(expr)) => {
+ self.visit(expr.iter().as_red());
+ let pattern = expr.pattern();
+ if let Some(key) = pattern.key() {
+ self.bind(key);
+ }
+ self.bind(pattern.value());
+ self.visit(expr.body().as_red());
+ }
+
+ // An import contains items, but these are active only after the
+ // path is evaluated.
+ Some(Expr::Import(expr)) => {
+ self.visit(expr.path().as_red());
+ if let Imports::Items(items) = expr.imports() {
+ for item in items {
+ self.bind(item);
+ }
+ }
+ }
+
+ // Blocks and templates create a scope.
+ Some(Expr::Block(_) | Expr::Template(_)) => {
+ self.internal.enter();
+ for child in node.children() {
+ self.visit(child);
+ }
+ self.internal.exit();
+ }
+
+ // Everything else is traversed from left to right.
+ _ => {
+ for child in node.children() {
+ self.visit(child);
+ }
+ }
+ }
}
+}
- fn visit_enter(&mut self) {
- self.internal.enter();
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::parse::parse;
+ use crate::source::SourceId;
+ use crate::syntax::RedNode;
+
+ #[track_caller]
+ fn test(src: &str, result: &[&str]) {
+ let green = parse(src);
+ let red = RedNode::from_root(green, SourceId::from_raw(0));
+
+ let mut scopes = Scopes::new(None);
+ scopes.def_const("x", 0);
+ scopes.def_const("y", 0);
+ scopes.def_const("z", 0);
+
+ let mut visitor = CapturesVisitor::new(&scopes);
+ visitor.visit(red.as_ref());
+
+ let captures = visitor.finish();
+ let mut names: Vec<_> = captures.iter().map(|(k, _)| k).collect();
+ names.sort();
+
+ assert_eq!(names, result);
}
- fn visit_exit(&mut self) {
- self.internal.exit();
+ #[test]
+ fn test_captures() {
+ // Let binding and function definition.
+ test("#let x = x", &["x"]);
+ test("#let x; {x + y}", &["y"]);
+ test("#let f(x, y) = x + y", &[]);
+
+ // Closure with different kinds of params.
+ test("{(x, y) => x + z}", &["z"]);
+ test("{(x: y, z) => x + z}", &["y"]);
+ test("{(..x) => x + y}", &["y"]);
+ test("{(x, y: x + z) => x + y}", &["x", "z"]);
+
+ // For loop.
+ test("#for x in y { x + z }", &["y", "z"]);
+ test("#for x, y in y { x + y }", &["y"]);
+
+ // Import.
+ test("#import x, y from z", &["z"]);
+ test("#import x, y, z from x + y", &["x", "y"]);
+
+ // Scoping.
+ test("{ let x = 1; { let y = 2; y }; x + y }", &["y"]);
+ test("[#let x = 1]#x", &["x"]);
}
}
diff --git a/src/eval/mod.rs b/src/eval/mod.rs
index 691e3c49..fda2184e 100644
--- a/src/eval/mod.rs
+++ b/src/eval/mod.rs
@@ -30,16 +30,14 @@ use std::collections::HashMap;
use std::io;
use std::mem;
use std::path::PathBuf;
-use std::rc::Rc;
use crate::diag::{At, Error, StrResult, Trace, Tracepoint, TypResult};
use crate::geom::{Angle, Fractional, Length, Relative};
use crate::image::ImageStore;
use crate::loading::Loader;
-use crate::parse::parse;
use crate::source::{SourceId, SourceStore};
-use crate::syntax::visit::Visit;
-use crate::syntax::*;
+use crate::syntax::ast::*;
+use crate::syntax::{Span, Spanned};
use crate::util::RefMutExt;
use crate::Context;
@@ -114,7 +112,7 @@ impl<'a> EvalContext<'a> {
// Parse the file.
let source = self.sources.get(id);
- let ast = parse(&source)?;
+ let ast = source.ast()?;
// Prepare the new context.
let new_scopes = Scopes::new(self.scopes.base);
@@ -122,7 +120,7 @@ impl<'a> EvalContext<'a> {
self.route.push(id);
// Evaluate the module.
- let template = Rc::new(ast).eval(self).trace(|| Tracepoint::Import, span)?;
+ let template = ast.eval(self).trace(|| Tracepoint::Import, span)?;
// Restore the old context.
let new_scopes = mem::replace(&mut self.scopes, old_scopes);
@@ -176,8 +174,8 @@ impl Eval for Expr {
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
match self {
- Self::Ident(v) => v.eval(ctx),
Self::Lit(v) => v.eval(ctx),
+ Self::Ident(v) => v.eval(ctx),
Self::Array(v) => v.eval(ctx).map(Value::Array),
Self::Dict(v) => v.eval(ctx).map(Value::Dict),
Self::Template(v) => v.eval(ctx).map(Value::Template),
@@ -202,17 +200,17 @@ impl Eval for Lit {
type Output = Value;
fn eval(&self, _: &mut EvalContext) -> TypResult<Self::Output> {
- Ok(match *self {
- Self::None(_) => Value::None,
- Self::Auto(_) => Value::Auto,
- Self::Bool(_, v) => Value::Bool(v),
- Self::Int(_, v) => Value::Int(v),
- Self::Float(_, v) => Value::Float(v),
- Self::Length(_, v, unit) => Value::Length(Length::with_unit(v, unit)),
- Self::Angle(_, v, unit) => Value::Angle(Angle::with_unit(v, unit)),
- Self::Percent(_, v) => Value::Relative(Relative::new(v / 100.0)),
- Self::Fractional(_, v) => Value::Fractional(Fractional::new(v)),
- Self::Str(_, ref v) => Value::Str(v.into()),
+ Ok(match self.kind() {
+ LitKind::None => Value::None,
+ LitKind::Auto => Value::Auto,
+ LitKind::Bool(v) => Value::Bool(v),
+ LitKind::Int(v) => Value::Int(v),
+ LitKind::Float(v) => Value::Float(v),
+ LitKind::Length(v, unit) => Value::Length(Length::with_unit(v, unit)),
+ LitKind::Angle(v, unit) => Value::Angle(Angle::with_unit(v, unit)),
+ LitKind::Percent(v) => Value::Relative(Relative::new(v / 100.0)),
+ LitKind::Fractional(v) => Value::Fractional(Fractional::new(v)),
+ LitKind::Str(ref v) => Value::Str(v.into()),
})
}
}
@@ -223,7 +221,7 @@ impl Eval for Ident {
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
match ctx.scopes.get(self) {
Some(slot) => Ok(slot.borrow().clone()),
- None => bail!(self.span, "unknown variable"),
+ None => bail!(self.span(), "unknown variable"),
}
}
}
@@ -232,7 +230,7 @@ impl Eval for ArrayExpr {
type Output = Array;
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
- self.items.iter().map(|expr| expr.eval(ctx)).collect()
+ self.items().map(|expr| expr.eval(ctx)).collect()
}
}
@@ -240,9 +238,8 @@ impl Eval for DictExpr {
type Output = Dict;
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
- self.items
- .iter()
- .map(|Named { name, expr }| Ok(((&name.string).into(), expr.eval(ctx)?)))
+ self.items()
+ .map(|x| Ok((x.name().take().into(), x.expr().eval(ctx)?)))
.collect()
}
}
@@ -251,7 +248,7 @@ impl Eval for TemplateExpr {
type Output = Template;
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
- self.body.eval(ctx)
+ self.body().eval(ctx)
}
}
@@ -259,7 +256,7 @@ impl Eval for GroupExpr {
type Output = Value;
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
- self.expr.eval(ctx)
+ self.expr().eval(ctx)
}
}
@@ -270,7 +267,7 @@ impl Eval for BlockExpr {
ctx.scopes.enter();
let mut output = Value::None;
- for expr in &self.exprs {
+ for expr in self.exprs() {
let value = expr.eval(ctx)?;
output = ops::join(output, value).at(expr.span())?;
}
@@ -285,13 +282,13 @@ impl Eval for UnaryExpr {
type Output = Value;
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
- let value = self.expr.eval(ctx)?;
- let result = match self.op {
+ let value = self.expr().eval(ctx)?;
+ let result = match self.op() {
UnOp::Pos => ops::pos(value),
UnOp::Neg => ops::neg(value),
UnOp::Not => ops::not(value),
};
- result.at(self.span)
+ result.at(self.span())
}
}
@@ -299,7 +296,7 @@ impl Eval for BinaryExpr {
type Output = Value;
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
- match self.op {
+ match self.op() {
BinOp::Add => self.apply(ctx, ops::add),
BinOp::Sub => self.apply(ctx, ops::sub),
BinOp::Mul => self.apply(ctx, ops::mul),
@@ -327,17 +324,17 @@ impl BinaryExpr {
where
F: FnOnce(Value, Value) -> StrResult<Value>,
{
- let lhs = self.lhs.eval(ctx)?;
+ let lhs = self.lhs().eval(ctx)?;
// Short-circuit boolean operations.
- if (self.op == BinOp::And && lhs == Value::Bool(false))
- || (self.op == BinOp::Or && lhs == Value::Bool(true))
+ if (self.op() == BinOp::And && lhs == Value::Bool(false))
+ || (self.op() == BinOp::Or && lhs == Value::Bool(true))
{
return Ok(lhs);
}
- let rhs = self.rhs.eval(ctx)?;
- op(lhs, rhs).at(self.span)
+ let rhs = self.rhs().eval(ctx)?;
+ op(lhs, rhs).at(self.span())
}
/// Apply an assignment operation.
@@ -345,10 +342,10 @@ impl BinaryExpr {
where
F: FnOnce(Value, Value) -> StrResult<Value>,
{
- let rhs = self.rhs.eval(ctx)?;
- let mut target = self.lhs.access(ctx)?;
+ let rhs = self.rhs().eval(ctx)?;
+ let mut target = self.lhs().access(ctx)?;
let lhs = mem::take(&mut *target);
- *target = op(lhs, rhs).at(self.span)?;
+ *target = op(lhs, rhs).at(self.span())?;
Ok(Value::None)
}
}
@@ -357,27 +354,27 @@ impl Eval for CallExpr {
type Output = Value;
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
- let callee = self.callee.eval(ctx)?;
- let mut args = self.args.eval(ctx)?;
+ let callee = self.callee().eval(ctx)?;
+ let mut args = self.args().eval(ctx)?;
match callee {
Value::Array(array) => {
- array.get(args.into_index()?).map(Value::clone).at(self.span)
+ array.get(args.into_index()?).map(Value::clone).at(self.span())
}
Value::Dict(dict) => {
- dict.get(args.into_key()?).map(Value::clone).at(self.span)
+ dict.get(args.into_key()?).map(Value::clone).at(self.span())
}
Value::Func(func) => {
let point = || Tracepoint::Call(func.name().map(ToString::to_string));
- let value = func.call(ctx, &mut args).trace(point, self.span)?;
+ let value = func.call(ctx, &mut args).trace(point, self.span())?;
args.finish()?;
Ok(value)
}
v => bail!(
- self.callee.span(),
+ self.callee().span(),
"expected function or collection, found {}",
v.type_name(),
),
@@ -389,9 +386,9 @@ impl Eval for CallArgs {
type Output = Args;
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
- let mut items = Vec::with_capacity(self.items.len());
+ let mut items = Vec::new();
- for arg in &self.items {
+ for arg in self.items() {
let span = arg.span();
match arg {
CallArg::Pos(expr) => {
@@ -401,11 +398,11 @@ impl Eval for CallArgs {
value: Spanned::new(expr.eval(ctx)?, expr.span()),
});
}
- CallArg::Named(Named { name, expr }) => {
+ CallArg::Named(named) => {
items.push(Arg {
span,
- name: Some((&name.string).into()),
- value: Spanned::new(expr.eval(ctx)?, expr.span()),
+ name: Some(named.name().take().into()),
+ value: Spanned::new(named.expr().eval(ctx)?, named.expr().span()),
});
}
CallArg::Spread(expr) => match expr.eval(ctx)? {
@@ -438,7 +435,7 @@ impl Eval for CallArgs {
}
}
- Ok(Args { span: self.span, items })
+ Ok(Args { span: self.span(), items })
}
}
@@ -446,39 +443,38 @@ impl Eval for ClosureExpr {
type Output = Value;
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
- let name = self.name.as_ref().map(|name| name.string.clone());
-
// Collect captured variables.
let captured = {
let mut visitor = CapturesVisitor::new(&ctx.scopes);
- visitor.visit_closure(self);
+ visitor.visit(self.as_red());
visitor.finish()
};
let mut sink = None;
- let mut params = Vec::with_capacity(self.params.len());
+ let mut params = Vec::new();
// Collect parameters and an optional sink parameter.
- for param in &self.params {
+ for param in self.params() {
match param {
ClosureParam::Pos(name) => {
- params.push((name.string.clone(), None));
+ params.push((name.take(), None));
}
- ClosureParam::Named(Named { name, expr }) => {
- params.push((name.string.clone(), Some(expr.eval(ctx)?)));
+ ClosureParam::Named(named) => {
+ params.push((named.name().take(), Some(named.expr().eval(ctx)?)));
}
ClosureParam::Sink(name) => {
if sink.is_some() {
- bail!(name.span, "only one argument sink is allowed");
+ bail!(name.span(), "only one argument sink is allowed");
}
- sink = Some(name.string.clone());
+ sink = Some(name.take());
}
}
}
// Clone the body expression so that we don't have a lifetime
// dependence on the AST.
- let body = Rc::clone(&self.body);
+ let name = self.name().map(Ident::take);
+ let body = self.body();
// Define the actual function.
let func = Function::new(name, move |ctx, args| {
@@ -515,8 +511,9 @@ impl Eval for WithExpr {
type Output = Value;
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
- let wrapped = self.callee.eval(ctx)?.cast::<Function>().at(self.callee.span())?;
- let applied = self.args.eval(ctx)?;
+ let callee = self.callee();
+ let wrapped = callee.eval(ctx)?.cast::<Function>().at(callee.span())?;
+ let applied = self.args().eval(ctx)?;
let name = wrapped.name().cloned();
let func = Function::new(name, move |ctx, args| {
@@ -532,11 +529,11 @@ impl Eval for LetExpr {
type Output = Value;
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
- let value = match &self.init {
+ let value = match self.init() {
Some(expr) => expr.eval(ctx)?,
None => Value::None,
};
- ctx.scopes.def_mut(self.binding.as_str(), value);
+ ctx.scopes.def_mut(self.binding().take(), value);
Ok(Value::None)
}
}
@@ -545,12 +542,10 @@ impl Eval for IfExpr {
type Output = Value;
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
- let condition =
- self.condition.eval(ctx)?.cast::<bool>().at(self.condition.span())?;
-
- if condition {
- self.if_body.eval(ctx)
- } else if let Some(else_body) = &self.else_body {
+ let condition = self.condition();
+ if condition.eval(ctx)?.cast::<bool>().at(condition.span())? {
+ self.if_body().eval(ctx)
+ } else if let Some(else_body) = self.else_body() {
else_body.eval(ctx)
} else {
Ok(Value::None)
@@ -564,9 +559,11 @@ impl Eval for WhileExpr {
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
let mut output = Value::None;
- while self.condition.eval(ctx)?.cast::<bool>().at(self.condition.span())? {
- let value = self.body.eval(ctx)?;
- output = ops::join(output, value).at(self.body.span())?;
+ let condition = self.condition();
+ while condition.eval(ctx)?.cast::<bool>().at(condition.span())? {
+ let body = self.body();
+ let value = body.eval(ctx)?;
+ output = ops::join(output, value).at(body.span())?;
}
Ok(output)
@@ -584,40 +581,44 @@ impl Eval for ForExpr {
#[allow(unused_parens)]
for ($($value),*) in $iter {
- $(ctx.scopes.def_mut($binding.as_str(), $value);)*
+ $(ctx.scopes.def_mut(&$binding, $value);)*
- let value = self.body.eval(ctx)?;
+ let value = self.body().eval(ctx)?;
output = ops::join(output, value)
- .at(self.body.span())?;
+ .at(self.body().span())?;
}
ctx.scopes.exit();
- Ok(output)
+ return Ok(output);
}};
}
- let iter = self.iter.eval(ctx)?;
- match (&self.pattern, iter) {
- (ForPattern::Value(v), Value::Str(string)) => {
- iter!(for (v => value) in string.iter())
+ let iter = self.iter().eval(ctx)?;
+ let pattern = self.pattern();
+ let key = pattern.key().map(Ident::take);
+ let value = pattern.value().take();
+
+ match (key, value, iter) {
+ (None, v, Value::Str(string)) => {
+ iter!(for (v => value) in string.iter());
}
- (ForPattern::Value(v), Value::Array(array)) => {
- iter!(for (v => value) in array.into_iter())
+ (None, v, Value::Array(array)) => {
+ iter!(for (v => value) in array.into_iter());
}
- (ForPattern::KeyValue(i, v), Value::Array(array)) => {
- iter!(for (i => idx, v => value) in array.into_iter().enumerate())
+ (Some(i), v, Value::Array(array)) => {
+ iter!(for (i => idx, v => value) in array.into_iter().enumerate());
}
- (ForPattern::Value(v), Value::Dict(dict)) => {
- iter!(for (v => value) in dict.into_iter().map(|p| p.1))
+ (None, v, Value::Dict(dict)) => {
+ iter!(for (v => value) in dict.into_iter().map(|p| p.1));
}
- (ForPattern::KeyValue(k, v), Value::Dict(dict)) => {
- iter!(for (k => key, v => value) in dict.into_iter())
+ (Some(k), v, Value::Dict(dict)) => {
+ iter!(for (k => key, v => value) in dict.into_iter());
}
- (ForPattern::KeyValue(_, _), Value::Str(_)) => {
- bail!(self.pattern.span(), "mismatched pattern");
+ (_, _, Value::Str(_)) => {
+ bail!(pattern.span(), "mismatched pattern");
}
- (_, iter) => {
- bail!(self.iter.span(), "cannot loop over {}", iter.type_name());
+ (_, _, iter) => {
+ bail!(self.iter().span(), "cannot loop over {}", iter.type_name());
}
}
}
@@ -627,23 +628,23 @@ impl Eval for ImportExpr {
type Output = Value;
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
- let path = self.path.eval(ctx)?.cast::<Str>().at(self.path.span())?;
-
- let file = ctx.import(&path, self.path.span())?;
+ let path = self.path();
+ let resolved = path.eval(ctx)?.cast::<Str>().at(path.span())?;
+ let file = ctx.import(&resolved, path.span())?;
let module = &ctx.modules[&file];
- match &self.imports {
+ match self.imports() {
Imports::Wildcard => {
for (var, slot) in module.scope.iter() {
ctx.scopes.def_mut(var, slot.borrow().clone());
}
}
- Imports::Idents(idents) => {
+ Imports::Items(idents) => {
for ident in idents {
if let Some(slot) = module.scope.get(&ident) {
- ctx.scopes.def_mut(ident.as_str(), slot.borrow().clone());
+ ctx.scopes.def_mut(ident.take(), slot.borrow().clone());
} else {
- bail!(ident.span, "unresolved import");
+ bail!(ident.span(), "unresolved import");
}
}
}
@@ -657,11 +658,10 @@ impl Eval for IncludeExpr {
type Output = Value;
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
- let path = self.path.eval(ctx)?.cast::<Str>().at(self.path.span())?;
-
- let file = ctx.import(&path, self.path.span())?;
+ let path = self.path();
+ let resolved = path.eval(ctx)?.cast::<Str>().at(path.span())?;
+ let file = ctx.import(&resolved, path.span())?;
let module = &ctx.modules[&file];
-
Ok(Value::Template(module.template.clone()))
}
}
@@ -689,23 +689,23 @@ impl Access for Ident {
match ctx.scopes.get(self) {
Some(slot) => match slot.try_borrow_mut() {
Ok(guard) => Ok(guard),
- Err(_) => bail!(self.span, "cannot mutate a constant"),
+ Err(_) => bail!(self.span(), "cannot mutate a constant"),
},
- None => bail!(self.span, "unknown variable"),
+ None => bail!(self.span(), "unknown variable"),
}
}
}
impl Access for CallExpr {
fn access<'a>(&self, ctx: &'a mut EvalContext) -> TypResult<RefMut<'a, Value>> {
- let args = self.args.eval(ctx)?;
- let guard = self.callee.access(ctx)?;
+ let args = self.args().eval(ctx)?;
+ let guard = self.callee().access(ctx)?;
RefMut::try_map(guard, |value| match value {
- Value::Array(array) => array.get_mut(args.into_index()?).at(self.span),
+ Value::Array(array) => array.get_mut(args.into_index()?).at(self.span()),
Value::Dict(dict) => Ok(dict.get_mut(args.into_key()?)),
v => bail!(
- self.callee.span(),
+ self.callee().span(),
"expected collection, found {}",
v.type_name(),
),
diff --git a/src/eval/scope.rs b/src/eval/scope.rs
index eb057ae3..2290affd 100644
--- a/src/eval/scope.rs
+++ b/src/eval/scope.rs
@@ -120,6 +120,8 @@ impl Scope {
impl Debug for Scope {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
- self.values.fmt(f)
+ f.debug_map()
+ .entries(self.values.iter().map(|(k, v)| (k, v.borrow())))
+ .finish()
}
}
diff --git a/src/eval/walk.rs b/src/eval/walk.rs
index 96138338..aab32f40 100644
--- a/src/eval/walk.rs
+++ b/src/eval/walk.rs
@@ -5,7 +5,7 @@ use crate::diag::TypResult;
use crate::geom::Spec;
use crate::layout::BlockLevel;
use crate::library::{GridNode, ParChild, ParNode, TrackSizing};
-use crate::syntax::*;
+use crate::syntax::ast::*;
use crate::util::BoolExt;
/// Walk markup, filling the currently built template.
@@ -16,7 +16,7 @@ pub trait Walk {
impl Walk for Markup {
fn walk(&self, ctx: &mut EvalContext) -> TypResult<()> {
- for node in self.iter() {
+ for node in self.nodes() {
node.walk(ctx)?;
}
Ok(())
@@ -27,12 +27,13 @@ impl Walk for MarkupNode {
fn walk(&self, ctx: &mut EvalContext) -> TypResult<()> {
match self {
Self::Space => ctx.template.space(),
- Self::Linebreak(_) => ctx.template.linebreak(),
- Self::Parbreak(_) => ctx.template.parbreak(),
- Self::Strong(_) => ctx.template.modify(|s| s.text_mut().strong.flip()),
- Self::Emph(_) => ctx.template.modify(|s| s.text_mut().emph.flip()),
+ Self::Linebreak => ctx.template.linebreak(),
+ Self::Parbreak => ctx.template.parbreak(),
+ Self::Strong => ctx.template.modify(|s| s.text_mut().strong.flip()),
+ Self::Emph => ctx.template.modify(|s| s.text_mut().emph.flip()),
Self::Text(text) => ctx.template.text(text),
Self::Raw(raw) => raw.walk(ctx)?,
+ Self::Math(math) => math.walk(ctx)?,
Self::Heading(heading) => heading.walk(ctx)?,
Self::List(list) => list.walk(ctx)?,
Self::Enum(enum_) => enum_.walk(ctx)?,
@@ -67,16 +68,32 @@ impl Walk for RawNode {
}
}
+impl Walk for MathNode {
+ fn walk(&self, ctx: &mut EvalContext) -> TypResult<()> {
+ if self.display {
+ ctx.template.parbreak();
+ }
+
+ ctx.template.monospace(self.formula.trim());
+
+ if self.display {
+ ctx.template.parbreak();
+ }
+
+ Ok(())
+ }
+}
+
impl Walk for HeadingNode {
fn walk(&self, ctx: &mut EvalContext) -> TypResult<()> {
- let level = self.level;
- let body = self.body.eval(ctx)?;
+ let level = self.level();
+ let body = self.body().eval(ctx)?;
ctx.template.parbreak();
ctx.template.save();
ctx.template.modify(move |style| {
let text = style.text_mut();
- let upscale = 1.6 - 0.1 * level as f64;
+ let upscale = (1.6 - 0.1 * level as f64).max(0.75);
text.size *= upscale;
text.strong = true;
});
@@ -90,7 +107,7 @@ impl Walk for HeadingNode {
impl Walk for ListNode {
fn walk(&self, ctx: &mut EvalContext) -> TypResult<()> {
- let body = self.body.eval(ctx)?;
+ let body = self.body().eval(ctx)?;
walk_item(ctx, Str::from('•'), body);
Ok(())
}
@@ -98,8 +115,8 @@ impl Walk for ListNode {
impl Walk for EnumNode {
fn walk(&self, ctx: &mut EvalContext) -> TypResult<()> {
- let body = self.body.eval(ctx)?;
- let label = format_str!("{}.", self.number.unwrap_or(1));
+ let body = self.body().eval(ctx)?;
+ let label = format_str!("{}.", self.number().unwrap_or(1));
walk_item(ctx, label, body);
Ok(())
}
diff --git a/src/geom/relative.rs b/src/geom/relative.rs
index c2d0a0cb..e46c51de 100644
--- a/src/geom/relative.rs
+++ b/src/geom/relative.rs
@@ -3,7 +3,7 @@ use super::*;
/// A relative length.
///
/// _Note_: `50%` is represented as `0.5` here, but stored as `50.0` in the
-/// corresponding [literal](crate::syntax::Lit::Percent).
+/// corresponding [literal](crate::syntax::ast::LitKind::Percent).
#[derive(Default, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub struct Relative(N64);
diff --git a/src/lib.rs b/src/lib.rs
index 41b2e88b..033230f0 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -20,7 +20,7 @@
//!
//! [tokens]: parse::Tokens
//! [parsed]: parse::parse
-//! [markup]: syntax::Markup
+//! [markup]: syntax::ast::Markup
//! [evaluate]: eval::eval
//! [module]: eval::Module
//! [layout tree]: layout::LayoutTree
@@ -58,7 +58,6 @@ use crate::layout::{EvictionPolicy, LayoutCache};
use crate::loading::Loader;
use crate::source::{SourceId, SourceStore};
use crate::style::Style;
-use crate::syntax::Markup;
/// The core context which holds the loader, configuration and cached artifacts.
pub struct Context {
@@ -100,14 +99,9 @@ impl Context {
&self.style
}
- /// Parse a source file and return the resulting markup.
- pub fn parse(&mut self, id: SourceId) -> TypResult<Markup> {
- parse::parse(self.sources.get(id))
- }
-
/// Evaluate a source file and return the resulting module.
pub fn evaluate(&mut self, id: SourceId) -> TypResult<Module> {
- let ast = self.parse(id)?;
+ let ast = self.sources.get(id).ast()?;
eval::eval(self, id, &ast)
}
diff --git a/src/parse/mod.rs b/src/parse/mod.rs
index 30787423..f9c0049f 100644
--- a/src/parse/mod.rs
+++ b/src/parse/mod.rs
@@ -12,216 +12,162 @@ pub use tokens::*;
use std::rc::Rc;
-use crate::diag::TypResult;
-use crate::source::SourceFile;
-use crate::syntax::*;
-use crate::util::EcoString;
+use crate::syntax::ast::{Associativity, BinOp, UnOp};
+use crate::syntax::{ErrorPos, Green, GreenNode, NodeKind};
/// Parse a source file.
-pub fn parse(source: &SourceFile) -> TypResult<Markup> {
- let mut p = Parser::new(source);
- let markup = markup(&mut p);
- let errors = p.finish();
- if errors.is_empty() {
- Ok(markup)
- } else {
- Err(Box::new(errors))
+pub fn parse(src: &str) -> Rc<GreenNode> {
+ let mut p = Parser::new(src);
+ markup(&mut p);
+ match p.finish().into_iter().next() {
+ Some(Green::Node(node)) => node,
+ _ => unreachable!(),
}
}
/// Parse markup.
-fn markup(p: &mut Parser) -> Markup {
+fn markup(p: &mut Parser) {
markup_while(p, true, &mut |_| true)
}
-/// Parse markup that stays equal or right of the given column.
-fn markup_indented(p: &mut Parser, column: usize) -> Markup {
+/// Parse markup that stays right of the given column.
+fn markup_indented(p: &mut Parser, column: usize) {
p.eat_while(|t| match t {
- Token::Space(n) => n == 0,
- Token::LineComment(_) | Token::BlockComment(_) => true,
+ NodeKind::Space(n) => *n == 0,
+ NodeKind::LineComment | NodeKind::BlockComment => true,
_ => false,
});
markup_while(p, false, &mut |p| match p.peek() {
- Some(Token::Space(n)) if n >= 1 => p.column(p.next_end()) >= column,
+ Some(NodeKind::Space(n)) if *n >= 1 => p.column(p.current_end()) >= column,
_ => true,
})
}
-/// Parse a syntax tree while the peeked token satisifies a condition.
+/// Parse a syntax tree while the peeked NodeKind satisifies a condition.
///
/// If `at_start` is true, things like headings that may only appear at the
/// beginning of a line or template are allowed.
-fn markup_while<F>(p: &mut Parser, mut at_start: bool, f: &mut F) -> Markup
+fn markup_while<F>(p: &mut Parser, mut at_start: bool, f: &mut F)
where
F: FnMut(&mut Parser) -> bool,
{
- let mut tree = vec![];
- while !p.eof() && f(p) {
- if let Some(node) = markup_node(p, &mut at_start) {
- at_start &= matches!(node, MarkupNode::Space | MarkupNode::Parbreak(_));
- tree.push(node);
+ p.perform(NodeKind::Markup, |p| {
+ while !p.eof() && f(p) {
+ markup_node(p, &mut at_start);
}
- }
-
- tree
+ });
}
/// Parse a markup node.
-fn markup_node(p: &mut Parser, at_start: &mut bool) -> Option<MarkupNode> {
- let token = p.peek()?;
- let span = p.peek_span();
- let node = match token {
+fn markup_node(p: &mut Parser, at_start: &mut bool) {
+ let token = match p.peek() {
+ Some(t) => t,
+ None => return,
+ };
+
+ match token {
// Whitespace.
- Token::Space(newlines) => {
- *at_start |= newlines > 0;
- if newlines < 2 {
- MarkupNode::Space
+ NodeKind::Space(newlines) => {
+ *at_start |= *newlines > 0;
+ if *newlines < 2 {
+ p.eat();
} else {
- MarkupNode::Parbreak(span)
+ p.convert(NodeKind::Parbreak);
}
+ return;
}
- // Text.
- Token::Text(text) => MarkupNode::Text(text.into()),
- Token::Tilde => MarkupNode::Text("\u{00A0}".into()),
- Token::HyphHyph => MarkupNode::Text("\u{2013}".into()),
- Token::HyphHyphHyph => MarkupNode::Text("\u{2014}".into()),
- Token::UnicodeEscape(t) => MarkupNode::Text(unicode_escape(p, t)),
+ // Comments.
+ NodeKind::LineComment | NodeKind::BlockComment => {
+ p.eat();
+ return;
+ }
+
+ // Text and markup.
+ NodeKind::Text(_)
+ | NodeKind::EnDash
+ | NodeKind::EmDash
+ | NodeKind::NonBreakingSpace
+ | NodeKind::Emph
+ | NodeKind::Strong
+ | NodeKind::Linebreak
+ | NodeKind::Raw(_)
+ | NodeKind::Math(_)
+ | NodeKind::UnicodeEscape(_) => {
+ p.eat();
+ }
- // Markup.
- Token::Backslash => MarkupNode::Linebreak(span),
- Token::Star => MarkupNode::Strong(span),
- Token::Underscore => MarkupNode::Emph(span),
- Token::Raw(t) => raw(p, t),
- Token::Eq if *at_start => return Some(heading(p)),
- Token::Hyph if *at_start => return Some(list_node(p)),
- Token::Numbering(number) if *at_start => return Some(enum_node(p, number)),
+ NodeKind::Eq if *at_start => heading(p),
+ NodeKind::Minus if *at_start => list_node(p),
+ NodeKind::EnumNumbering(_) if *at_start => enum_node(p),
// Line-based markup that is not currently at the start of the line.
- Token::Eq | Token::Hyph | Token::Numbering(_) => {
- MarkupNode::Text(p.peek_src().into())
+ NodeKind::Eq | NodeKind::Minus | NodeKind::EnumNumbering(_) => {
+ p.convert(NodeKind::Text(p.peek_src().into()));
}
// Hashtag + keyword / identifier.
- Token::Ident(_)
- | Token::Let
- | Token::If
- | Token::While
- | Token::For
- | Token::Import
- | Token::Include => {
- let stmt = matches!(token, Token::Let | Token::Import);
+ NodeKind::Ident(_)
+ | NodeKind::Let
+ | NodeKind::If
+ | NodeKind::While
+ | NodeKind::For
+ | NodeKind::Import
+ | NodeKind::Include => {
+ let stmt = matches!(token, NodeKind::Let | NodeKind::Import);
let group = if stmt { Group::Stmt } else { Group::Expr };
- p.start_group(group, TokenMode::Code);
- let expr = expr_with(p, true, 0);
- if stmt && expr.is_some() && !p.eof() {
- p.expected_at(p.prev_end(), "semicolon or line break");
+ p.start_group(group);
+ let res = expr_prec(p, true, 0);
+ if stmt && res.is_ok() && !p.eof() {
+ p.expected_at("semicolon or line break");
}
p.end_group();
-
- return expr.map(MarkupNode::Expr);
}
// Block and template.
- Token::LeftBrace => return Some(MarkupNode::Expr(block(p))),
- Token::LeftBracket => return Some(MarkupNode::Expr(template(p))),
-
- // Comments.
- Token::LineComment(_) | Token::BlockComment(_) => {
- p.eat();
- return None;
- }
+ NodeKind::LeftBrace => block(p),
+ NodeKind::LeftBracket => template(p),
- _ => {
- *at_start = false;
- p.unexpected();
- return None;
- }
+ NodeKind::Error(_, _) => p.eat(),
+ _ => p.unexpected(),
};
- p.eat();
- Some(node)
-}
-/// Handle a unicode escape sequence.
-fn unicode_escape(p: &mut Parser, token: UnicodeEscapeToken) -> EcoString {
- let span = p.peek_span();
- let text = if let Some(c) = resolve::resolve_hex(token.sequence) {
- c.into()
- } else {
- // Print out the escape sequence verbatim if it is invalid.
- p.error(span, "invalid unicode escape sequence");
- p.peek_src().into()
- };
-
- if !token.terminated {
- p.error(span.end, "expected closing brace");
- }
-
- text
-}
-
-/// Handle a raw block.
-fn raw(p: &mut Parser, token: RawToken) -> MarkupNode {
- let column = p.column(p.next_start());
- let span = p.peek_span();
- let raw = resolve::resolve_raw(span, column, token.backticks, token.text);
- if !token.terminated {
- p.error(span.end, "expected backtick(s)");
- }
- MarkupNode::Raw(Box::new(raw))
+ *at_start = false;
}
/// Parse a heading.
-fn heading(p: &mut Parser) -> MarkupNode {
- let start = p.next_start();
- p.eat_assert(Token::Eq);
-
- // Count depth.
- let mut level: usize = 1;
- while p.eat_if(Token::Eq) {
- level += 1;
- }
-
- if level > 6 {
- return MarkupNode::Text(p.get(start .. p.prev_end()).into());
- }
-
- let column = p.column(p.prev_end());
- let body = markup_indented(p, column);
- MarkupNode::Heading(Box::new(HeadingNode {
- span: p.span_from(start),
- level,
- body,
- }))
+fn heading(p: &mut Parser) {
+ p.perform(NodeKind::Heading, |p| {
+ p.eat_assert(&NodeKind::Eq);
+ while p.eat_if(&NodeKind::Eq) {}
+ let column = p.column(p.prev_end());
+ markup_indented(p, column);
+ });
}
/// Parse a single list item.
-fn list_node(p: &mut Parser) -> MarkupNode {
- let start = p.next_start();
- p.eat_assert(Token::Hyph);
- let column = p.column(p.prev_end());
- let body = markup_indented(p, column);
- MarkupNode::List(Box::new(ListNode { span: p.span_from(start), body }))
+fn list_node(p: &mut Parser) {
+ p.perform(NodeKind::List, |p| {
+ p.eat_assert(&NodeKind::Minus);
+ let column = p.column(p.prev_end());
+ markup_indented(p, column);
+ });
}
/// Parse a single enum item.
-fn enum_node(p: &mut Parser, number: Option<usize>) -> MarkupNode {
- let start = p.next_start();
- p.eat_assert(Token::Numbering(number));
- let column = p.column(p.prev_end());
- let body = markup_indented(p, column);
- MarkupNode::Enum(Box::new(EnumNode {
- span: p.span_from(start),
- number,
- body,
- }))
+fn enum_node(p: &mut Parser) {
+ p.perform(NodeKind::Enum, |p| {
+ p.eat();
+ let column = p.column(p.prev_end());
+ markup_indented(p, column);
+ });
}
/// Parse an expression.
-fn expr(p: &mut Parser) -> Option<Expr> {
- expr_with(p, false, 0)
+fn expr(p: &mut Parser) -> ParseResult {
+ expr_prec(p, false, 0)
}
/// Parse an expression with operators having at least the minimum precedence.
@@ -231,13 +177,16 @@ fn expr(p: &mut Parser) -> Option<Expr> {
/// in markup.
///
/// Stops parsing at operations with lower precedence than `min_prec`,
-fn expr_with(p: &mut Parser, atomic: bool, min_prec: usize) -> Option<Expr> {
- let start = p.next_start();
- let mut lhs = match p.eat_map(UnOp::from_token) {
+fn expr_prec(p: &mut Parser, atomic: bool, min_prec: usize) -> ParseResult {
+ let marker = p.marker();
+
+ // Start the unary expression.
+ match p.peek().and_then(UnOp::from_token) {
Some(op) => {
+ p.eat();
let prec = op.precedence();
- let expr = expr_with(p, atomic, prec)?;
- Expr::Unary(Box::new(UnaryExpr { span: p.span_from(start), op, expr }))
+ expr_prec(p, atomic, prec)?;
+ marker.end(p, NodeKind::Unary);
}
None => primary(p, atomic)?,
};
@@ -245,19 +194,19 @@ fn expr_with(p: &mut Parser, atomic: bool, min_prec: usize) -> Option<Expr> {
loop {
// Exclamation mark, parenthesis or bracket means this is a function
// call.
- if matches!(p.peek_direct(), Some(Token::LeftParen | Token::LeftBracket)) {
- lhs = call(p, lhs)?;
+ if let Some(NodeKind::LeftParen | NodeKind::LeftBracket) = p.peek_direct() {
+ call(p, marker)?;
continue;
}
- if p.eat_if(Token::With) {
- lhs = with_expr(p, lhs)?;
- }
-
if atomic {
break;
}
+ if p.at(&NodeKind::With) {
+ with_expr(p, marker)?;
+ }
+
let op = match p.peek().and_then(BinOp::from_token) {
Some(binop) => binop,
None => break,
@@ -269,96 +218,94 @@ fn expr_with(p: &mut Parser, atomic: bool, min_prec: usize) -> Option<Expr> {
}
p.eat();
+
match op.associativity() {
Associativity::Left => prec += 1,
Associativity::Right => {}
}
- let rhs = match expr_with(p, atomic, prec) {
- Some(rhs) => rhs,
- None => break,
- };
-
- let span = lhs.span().join(rhs.span());
- lhs = Expr::Binary(Box::new(BinaryExpr { span, lhs, op, rhs }));
+ marker.perform(p, NodeKind::Binary, |p| expr_prec(p, atomic, prec))?;
}
- Some(lhs)
+ Ok(())
}
/// Parse a primary expression.
-fn primary(p: &mut Parser, atomic: bool) -> Option<Expr> {
- if let Some(expr) = literal(p) {
- return Some(expr);
+fn primary(p: &mut Parser, atomic: bool) -> ParseResult {
+ if literal(p) {
+ return Ok(());
}
match p.peek() {
// Things that start with an identifier.
- Some(Token::Ident(string)) => {
- let ident = Ident {
- span: p.eat_span(),
- string: string.into(),
- };
+ Some(NodeKind::Ident(_)) => {
+ let marker = p.marker();
+ p.eat();
// Arrow means this is a closure's lone parameter.
- Some(if !atomic && p.eat_if(Token::Arrow) {
- let body = expr(p)?;
- Expr::Closure(Box::new(ClosureExpr {
- span: ident.span.join(body.span()),
- name: None,
- params: vec![ClosureParam::Pos(ident)],
- body: Rc::new(body),
- }))
+ if !atomic && p.at(&NodeKind::Arrow) {
+ marker.end(p, NodeKind::ClosureParams);
+ p.eat_assert(&NodeKind::Arrow);
+ marker.perform(p, NodeKind::Closure, expr)
} else {
- Expr::Ident(Box::new(ident))
- })
+ Ok(())
+ }
}
// Structures.
- Some(Token::LeftParen) => parenthesized(p),
- Some(Token::LeftBracket) => Some(template(p)),
- Some(Token::LeftBrace) => Some(block(p)),
+ Some(NodeKind::LeftParen) => parenthesized(p),
+ Some(NodeKind::LeftBracket) => {
+ template(p);
+ Ok(())
+ }
+ Some(NodeKind::LeftBrace) => {
+ block(p);
+ Ok(())
+ }
// Keywords.
- Some(Token::Let) => let_expr(p),
- Some(Token::If) => if_expr(p),
- Some(Token::While) => while_expr(p),
- Some(Token::For) => for_expr(p),
- Some(Token::Import) => import_expr(p),
- Some(Token::Include) => include_expr(p),
+ Some(NodeKind::Let) => let_expr(p),
+ Some(NodeKind::If) => if_expr(p),
+ Some(NodeKind::While) => while_expr(p),
+ Some(NodeKind::For) => for_expr(p),
+ Some(NodeKind::Import) => import_expr(p),
+ Some(NodeKind::Include) => include_expr(p),
+
+ Some(NodeKind::Error(_, _)) => {
+ p.eat();
+ Err(())
+ }
// Nothing.
_ => {
p.expected("expression");
- None
+ Err(())
}
}
}
/// Parse a literal.
-fn literal(p: &mut Parser) -> Option<Expr> {
- let span = p.peek_span();
- let lit = match p.peek()? {
+fn literal(p: &mut Parser) -> bool {
+ match p.peek() {
// Basic values.
- Token::None => Lit::None(span),
- Token::Auto => Lit::Auto(span),
- Token::Bool(b) => Lit::Bool(span, b),
- Token::Int(i) => Lit::Int(span, i),
- Token::Float(f) => Lit::Float(span, f),
- Token::Length(val, unit) => Lit::Length(span, val, unit),
- Token::Angle(val, unit) => Lit::Angle(span, val, unit),
- Token::Percent(p) => Lit::Percent(span, p),
- Token::Fraction(p) => Lit::Fractional(span, p),
- Token::Str(token) => Lit::Str(span, {
- if !token.terminated {
- p.expected_at(span.end, "quote");
- }
- resolve::resolve_string(token.string)
- }),
- _ => return None,
- };
- p.eat();
- Some(Expr::Lit(Box::new(lit)))
+ Some(
+ NodeKind::None
+ | NodeKind::Auto
+ | NodeKind::Int(_)
+ | NodeKind::Float(_)
+ | NodeKind::Bool(_)
+ | NodeKind::Fraction(_)
+ | NodeKind::Length(_, _)
+ | NodeKind::Angle(_, _)
+ | NodeKind::Percentage(_)
+ | NodeKind::Str(_),
+ ) => {
+ p.eat();
+ true
+ }
+
+ _ => false,
+ }
}
/// Parse something that starts with a parenthesis, which can be either of:
@@ -366,433 +313,395 @@ fn literal(p: &mut Parser) -> Option<Expr> {
/// - Dictionary literal
/// - Parenthesized expression
/// - Parameter list of closure expression
-fn parenthesized(p: &mut Parser) -> Option<Expr> {
- p.start_group(Group::Paren, TokenMode::Code);
- let colon = p.eat_if(Token::Colon);
- let (items, has_comma) = collection(p);
- let span = p.end_group();
+fn parenthesized(p: &mut Parser) -> ParseResult {
+ let marker = p.marker();
- // Leading colon makes this a dictionary.
+ p.start_group(Group::Paren);
+ let colon = p.eat_if(&NodeKind::Colon);
+ let kind = collection(p).0;
+ p.end_group();
+
+ // Leading colon makes this a (empty) dictionary.
if colon {
- return Some(dict(p, items, span));
+ dict(p, marker);
+ return Ok(());
}
// Arrow means this is a closure's parameter list.
- if p.eat_if(Token::Arrow) {
- let params = params(p, items);
- let body = expr(p)?;
- return Some(Expr::Closure(Box::new(ClosureExpr {
- span: span.join(body.span()),
- name: None,
- params,
- body: Rc::new(body),
- })));
+ if p.at(&NodeKind::Arrow) {
+ params(p, marker);
+ p.eat_assert(&NodeKind::Arrow);
+ return marker.perform(p, NodeKind::Closure, expr);
}
- // Find out which kind of collection this is.
- Some(match items.as_slice() {
- [] => array(p, items, span),
- [CallArg::Pos(_)] if !has_comma => match items.into_iter().next() {
- Some(CallArg::Pos(expr)) => Expr::Group(Box::new(GroupExpr { span, expr })),
- _ => unreachable!(),
- },
- [CallArg::Pos(_), ..] => array(p, items, span),
- [CallArg::Named(_), ..] => dict(p, items, span),
- [CallArg::Spread(expr), ..] => {
- p.error(expr.span(), "spreading is not allowed here");
- return None;
- }
- })
+ // Transform into the identified collection.
+ match kind {
+ CollectionKind::Group => marker.end(p, NodeKind::Group),
+ CollectionKind::Positional => array(p, marker),
+ CollectionKind::Named => dict(p, marker),
+ }
+
+ Ok(())
+}
+
+/// The type of a collection.
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+enum CollectionKind {
+ /// The collection is only one item and has no comma.
+ Group,
+ /// The collection starts with a positional and has more items or a trailing
+ /// comma.
+ Positional,
+ /// The collection starts with a named item.
+ Named,
}
/// Parse a collection.
///
-/// Returns whether the literal contained any commas.
-fn collection(p: &mut Parser) -> (Vec<CallArg>, bool) {
- let mut items = vec![];
- let mut has_comma = false;
- let mut missing_coma = None;
+/// Returns the length of the collection and whether the literal contained any
+/// commas.
+fn collection(p: &mut Parser) -> (CollectionKind, usize) {
+ let mut kind = CollectionKind::Positional;
+ let mut items = 0;
+ let mut can_group = true;
+ let mut error = false;
+ let mut missing_coma: Option<Marker> = None;
while !p.eof() {
- if let Some(arg) = item(p) {
- items.push(arg);
+ if let Ok(item_kind) = item(p) {
+ if items == 0 && item_kind == NodeKind::Named {
+ kind = CollectionKind::Named;
+ can_group = false;
+ }
- if let Some(pos) = missing_coma.take() {
- p.expected_at(pos, "comma");
+ if item_kind == NodeKind::Spread {
+ can_group = false;
+ }
+
+ items += 1;
+
+ if let Some(marker) = missing_coma.take() {
+ marker.expected(p, "comma");
}
if p.eof() {
break;
}
- let behind = p.prev_end();
- if p.eat_if(Token::Comma) {
- has_comma = true;
+ if p.eat_if(&NodeKind::Comma) {
+ can_group = false;
} else {
- missing_coma = Some(behind);
+ missing_coma = Some(p.trivia_start());
}
+ } else {
+ error = true;
}
}
- (items, has_comma)
+ if error || (can_group && items == 1) {
+ kind = CollectionKind::Group;
+ }
+
+ (kind, items)
}
-/// Parse an expression or a named pair.
-fn item(p: &mut Parser) -> Option<CallArg> {
- if p.eat_if(Token::Dots) {
- return expr(p).map(CallArg::Spread);
+/// Parse an expression or a named pair, returning whether it's a spread or a
+/// named pair.
+fn item(p: &mut Parser) -> ParseResult<NodeKind> {
+ let marker = p.marker();
+ if p.eat_if(&NodeKind::Dots) {
+ marker.perform(p, NodeKind::Spread, expr)?;
+ return Ok(NodeKind::Spread);
}
- let first = expr(p)?;
- if p.eat_if(Token::Colon) {
- if let Expr::Ident(name) = first {
- Some(CallArg::Named(Named { name: *name, expr: expr(p)? }))
- } else {
- p.error(first.span(), "expected identifier");
- expr(p);
- None
- }
+ expr(p)?;
+
+ if p.at(&NodeKind::Colon) {
+ marker.perform(p, NodeKind::Named, |p| {
+ if let Some(NodeKind::Ident(_)) = marker.peek(p).map(|c| c.kind()) {
+ p.eat();
+ expr(p)
+ } else {
+ let error = NodeKind::Error(ErrorPos::Full, "expected identifier".into());
+ marker.end(p, error);
+ p.eat();
+ expr(p).ok();
+ Err(())
+ }
+ })?;
+
+ Ok(NodeKind::Named)
} else {
- Some(CallArg::Pos(first))
+ Ok(NodeKind::None)
}
}
/// Convert a collection into an array, producing errors for anything other than
/// expressions.
-fn array(p: &mut Parser, items: Vec<CallArg>, span: Span) -> Expr {
- let iter = items.into_iter().filter_map(|item| match item {
- CallArg::Pos(expr) => Some(expr),
- CallArg::Named(_) => {
- p.error(item.span(), "expected expression, found named pair");
- None
- }
- CallArg::Spread(_) => {
- p.error(item.span(), "spreading is not allowed here");
- None
- }
+fn array(p: &mut Parser, marker: Marker) {
+ marker.filter_children(p, |x| match x.kind() {
+ NodeKind::Named => Err("expected expression, found named pair"),
+ NodeKind::Spread => Err("spreading is not allowed here"),
+ _ => Ok(()),
});
- Expr::Array(Box::new(ArrayExpr { span, items: iter.collect() }))
+ marker.end(p, NodeKind::Array);
}
/// Convert a collection into a dictionary, producing errors for anything other
/// than named pairs.
-fn dict(p: &mut Parser, items: Vec<CallArg>, span: Span) -> Expr {
- let iter = items.into_iter().filter_map(|item| match item {
- CallArg::Named(named) => Some(named),
- CallArg::Pos(_) => {
- p.error(item.span(), "expected named pair, found expression");
- None
- }
- CallArg::Spread(_) => {
- p.error(item.span(), "spreading is not allowed here");
- None
- }
+fn dict(p: &mut Parser, marker: Marker) {
+ marker.filter_children(p, |x| match x.kind() {
+ kind if kind.is_paren() => Ok(()),
+ NodeKind::Named | NodeKind::Comma | NodeKind::Colon => Ok(()),
+ NodeKind::Spread => Err("spreading is not allowed here"),
+ _ => Err("expected named pair, found expression"),
});
- Expr::Dict(Box::new(DictExpr { span, items: iter.collect() }))
+ marker.end(p, NodeKind::Dict);
}
/// Convert a collection into a list of parameters, producing errors for
/// anything other than identifiers, spread operations and named pairs.
-fn params(p: &mut Parser, items: Vec<CallArg>) -> Vec<ClosureParam> {
- let iter = items.into_iter().filter_map(|item| match item {
- CallArg::Pos(Expr::Ident(ident)) => Some(ClosureParam::Pos(*ident)),
- CallArg::Named(named) => Some(ClosureParam::Named(named)),
- CallArg::Spread(Expr::Ident(ident)) => Some(ClosureParam::Sink(*ident)),
- _ => {
- p.error(item.span(), "expected identifier");
- None
- }
+fn params(p: &mut Parser, marker: Marker) {
+ marker.filter_children(p, |x| match x.kind() {
+ kind if kind.is_paren() => Ok(()),
+ NodeKind::Named | NodeKind::Comma | NodeKind::Ident(_) => Ok(()),
+ NodeKind::Spread
+ if matches!(
+ x.children().last().map(|child| child.kind()),
+ Some(&NodeKind::Ident(_))
+ ) =>
+ {
+ Ok(())
+ }
+ _ => Err("expected identifier"),
});
- iter.collect()
-}
-
-/// Convert a collection into a list of identifiers, producing errors for
-/// anything other than identifiers.
-fn idents(p: &mut Parser, items: Vec<CallArg>) -> Vec<Ident> {
- let iter = items.into_iter().filter_map(|item| match item {
- CallArg::Pos(Expr::Ident(ident)) => Some(*ident),
- _ => {
- p.error(item.span(), "expected identifier");
- None
- }
- });
- iter.collect()
+ marker.end(p, NodeKind::ClosureParams);
}
// Parse a template block: `[...]`.
-fn template(p: &mut Parser) -> Expr {
- p.start_group(Group::Bracket, TokenMode::Markup);
- let tree = markup(p);
- let span = p.end_group();
- Expr::Template(Box::new(TemplateExpr { span, body: tree }))
+fn template(p: &mut Parser) {
+ p.perform(NodeKind::Template, |p| {
+ p.start_group(Group::Bracket);
+ markup(p);
+ p.end_group();
+ });
}
/// Parse a code block: `{...}`.
-fn block(p: &mut Parser) -> Expr {
- p.start_group(Group::Brace, TokenMode::Code);
- let mut exprs = vec![];
- while !p.eof() {
- p.start_group(Group::Stmt, TokenMode::Code);
- if let Some(expr) = expr(p) {
- exprs.push(expr);
- if !p.eof() {
- p.expected_at(p.prev_end(), "semicolon or line break");
+fn block(p: &mut Parser) {
+ p.perform(NodeKind::Block, |p| {
+ p.start_group(Group::Brace);
+ while !p.eof() {
+ p.start_group(Group::Stmt);
+ if expr(p).is_ok() && !p.eof() {
+ p.expected_at("semicolon or line break");
}
+ p.end_group();
+
+ // Forcefully skip over newlines since the group's contents can't.
+ p.eat_while(|t| matches!(t, NodeKind::Space(_)));
}
p.end_group();
-
- // Forcefully skip over newlines since the group's contents can't.
- p.eat_while(|t| matches!(t, Token::Space(_)));
- }
- let span = p.end_group();
- Expr::Block(Box::new(BlockExpr { span, exprs }))
+ });
}
/// Parse a function call.
-fn call(p: &mut Parser, callee: Expr) -> Option<Expr> {
- let mut args = match p.peek_direct() {
- Some(Token::LeftParen) => args(p),
- Some(Token::LeftBracket) => CallArgs {
- span: Span::at(p.id(), callee.span().end),
- items: vec![],
- },
+fn call(p: &mut Parser, callee: Marker) -> ParseResult {
+ callee.perform(p, NodeKind::Call, |p| match p.peek_direct() {
+ Some(NodeKind::LeftParen | NodeKind::LeftBracket) => {
+ args(p, true);
+ Ok(())
+ }
_ => {
- p.expected_at(p.prev_end(), "argument list");
- return None;
+ p.expected_at("argument list");
+ Err(())
}
- };
-
- while p.peek_direct() == Some(Token::LeftBracket) {
- let body = template(p);
- args.items.push(CallArg::Pos(body));
- }
-
- Some(Expr::Call(Box::new(CallExpr {
- span: p.span_from(callee.span().start),
- callee,
- args,
- })))
+ })
}
/// Parse the arguments to a function call.
-fn args(p: &mut Parser) -> CallArgs {
- p.start_group(Group::Paren, TokenMode::Code);
- let items = collection(p).0;
- let span = p.end_group();
- CallArgs { span, items }
+fn args(p: &mut Parser, allow_template: bool) {
+ p.perform(NodeKind::CallArgs, |p| {
+ if !allow_template || p.peek_direct() == Some(&NodeKind::LeftParen) {
+ p.start_group(Group::Paren);
+ collection(p);
+ p.end_group();
+ }
+
+ while allow_template && p.peek_direct() == Some(&NodeKind::LeftBracket) {
+ template(p);
+ }
+ })
}
/// Parse a with expression.
-fn with_expr(p: &mut Parser, callee: Expr) -> Option<Expr> {
- if p.peek() == Some(Token::LeftParen) {
- Some(Expr::With(Box::new(WithExpr {
- span: p.span_from(callee.span().start),
- callee,
- args: args(p),
- })))
- } else {
- p.expected("argument list");
- None
- }
+fn with_expr(p: &mut Parser, marker: Marker) -> ParseResult {
+ marker.perform(p, NodeKind::WithExpr, |p| {
+ p.eat_assert(&NodeKind::With);
+
+ if p.at(&NodeKind::LeftParen) {
+ args(p, false);
+ Ok(())
+ } else {
+ p.expected("argument list");
+ Err(())
+ }
+ })
}
/// Parse a let expression.
-fn let_expr(p: &mut Parser) -> Option<Expr> {
- let start = p.next_start();
- p.eat_assert(Token::Let);
+fn let_expr(p: &mut Parser) -> ParseResult {
+ p.perform(NodeKind::LetExpr, |p| {
+ p.eat_assert(&NodeKind::Let);
- let mut output = None;
- if let Some(binding) = ident(p) {
- let mut init = None;
+ let marker = p.marker();
+ ident(p)?;
- if p.eat_if(Token::With) {
- init = with_expr(p, Expr::Ident(Box::new(binding.clone())));
+ if p.at(&NodeKind::With) {
+ with_expr(p, marker)?;
} else {
// If a parenthesis follows, this is a function definition.
- let mut maybe_params = None;
- if p.peek_direct() == Some(Token::LeftParen) {
- p.start_group(Group::Paren, TokenMode::Code);
- let items = collection(p).0;
- maybe_params = Some(params(p, items));
+ let has_params = p.peek_direct() == Some(&NodeKind::LeftParen);
+ if has_params {
+ let marker = p.marker();
+ p.start_group(Group::Paren);
+ collection(p);
p.end_group();
+ params(p, marker);
}
- if p.eat_if(Token::Eq) {
- init = expr(p);
- } else if maybe_params.is_some() {
+ if p.eat_if(&NodeKind::Eq) {
+ expr(p)?;
+ } else if has_params {
// Function definitions must have a body.
- p.expected_at(p.prev_end(), "body");
+ p.expected_at("body");
}
// Rewrite into a closure expression if it's a function definition.
- if let Some(params) = maybe_params {
- let body = init?;
- init = Some(Expr::Closure(Box::new(ClosureExpr {
- span: binding.span.join(body.span()),
- name: Some(binding.clone()),
- params,
- body: Rc::new(body),
- })));
+ if has_params {
+ marker.end(p, NodeKind::Closure);
}
}
- output = Some(Expr::Let(Box::new(LetExpr {
- span: p.span_from(start),
- binding,
- init,
- })));
- }
-
- output
+ Ok(())
+ })
}
/// Parse an if expresion.
-fn if_expr(p: &mut Parser) -> Option<Expr> {
- let start = p.next_start();
- p.eat_assert(Token::If);
-
- let mut output = None;
- if let Some(condition) = expr(p) {
- if let Some(if_body) = body(p) {
- let mut else_body = None;
- if p.eat_if(Token::Else) {
- if p.peek() == Some(Token::If) {
- else_body = if_expr(p);
- } else {
- else_body = body(p);
- }
- }
+fn if_expr(p: &mut Parser) -> ParseResult {
+ p.perform(NodeKind::IfExpr, |p| {
+ p.eat_assert(&NodeKind::If);
+
+ expr(p)?;
+ body(p)?;
- output = Some(Expr::If(Box::new(IfExpr {
- span: p.span_from(start),
- condition,
- if_body,
- else_body,
- })));
+ if p.eat_if(&NodeKind::Else) {
+ if p.at(&NodeKind::If) {
+ if_expr(p)?;
+ } else {
+ body(p)?;
+ }
}
- }
- output
+ Ok(())
+ })
}
/// Parse a while expresion.
-fn while_expr(p: &mut Parser) -> Option<Expr> {
- let start = p.next_start();
- p.eat_assert(Token::While);
-
- let mut output = None;
- if let Some(condition) = expr(p) {
- if let Some(body) = body(p) {
- output = Some(Expr::While(Box::new(WhileExpr {
- span: p.span_from(start),
- condition,
- body,
- })));
- }
- }
-
- output
+fn while_expr(p: &mut Parser) -> ParseResult {
+ p.perform(NodeKind::WhileExpr, |p| {
+ p.eat_assert(&NodeKind::While);
+ expr(p)?;
+ body(p)?;
+ Ok(())
+ })
}
/// Parse a for expression.
-fn for_expr(p: &mut Parser) -> Option<Expr> {
- let start = p.next_start();
- p.eat_assert(Token::For);
-
- let mut output = None;
- if let Some(pattern) = for_pattern(p) {
- if p.eat_expect(Token::In) {
- if let Some(iter) = expr(p) {
- if let Some(body) = body(p) {
- output = Some(Expr::For(Box::new(ForExpr {
- span: p.span_from(start),
- pattern,
- iter,
- body,
- })));
- }
- }
- }
- }
-
- output
+fn for_expr(p: &mut Parser) -> ParseResult {
+ p.perform(NodeKind::ForExpr, |p| {
+ p.eat_assert(&NodeKind::For);
+ for_pattern(p)?;
+ p.eat_expect(&NodeKind::In)?;
+ expr(p)?;
+ body(p)?;
+ Ok(())
+ })
}
/// Parse a for loop pattern.
-fn for_pattern(p: &mut Parser) -> Option<ForPattern> {
- let first = ident(p)?;
- if p.eat_if(Token::Comma) {
- if let Some(second) = ident(p) {
- return Some(ForPattern::KeyValue(first, second));
+fn for_pattern(p: &mut Parser) -> ParseResult {
+ p.perform(NodeKind::ForPattern, |p| {
+ ident(p)?;
+ if p.eat_if(&NodeKind::Comma) {
+ ident(p)?;
}
- }
- Some(ForPattern::Value(first))
+ Ok(())
+ })
}
/// Parse an import expression.
-fn import_expr(p: &mut Parser) -> Option<Expr> {
- let start = p.next_start();
- p.eat_assert(Token::Import);
+fn import_expr(p: &mut Parser) -> ParseResult {
+ p.perform(NodeKind::ImportExpr, |p| {
+ p.eat_assert(&NodeKind::Import);
+
+ if !p.eat_if(&NodeKind::Star) {
+ // This is the list of identifiers scenario.
+ p.perform(NodeKind::ImportItems, |p| {
+ p.start_group(Group::Imports);
+ let marker = p.marker();
+ let items = collection(p).1;
+ if items == 0 {
+ p.expected_at("import items");
+ }
+ p.end_group();
- let imports = if p.eat_if(Token::Star) {
- // This is the wildcard scenario.
- Imports::Wildcard
- } else {
- // This is the list of identifiers scenario.
- p.start_group(Group::Imports, TokenMode::Code);
- let items = collection(p).0;
- if items.is_empty() {
- p.expected_at(p.prev_end(), "import items");
- }
- p.end_group();
- Imports::Idents(idents(p, items))
- };
+ marker.filter_children(p, |n| match n.kind() {
+ NodeKind::Ident(_) | NodeKind::Comma => Ok(()),
+ _ => Err("expected identifier"),
+ });
+ });
+ };
- let mut output = None;
- if p.eat_expect(Token::From) {
- if let Some(path) = expr(p) {
- output = Some(Expr::Import(Box::new(ImportExpr {
- span: p.span_from(start),
- imports,
- path,
- })));
- }
- }
+ p.eat_expect(&NodeKind::From)?;
+ expr(p)?;
- output
+ Ok(())
+ })
}
/// Parse an include expression.
-fn include_expr(p: &mut Parser) -> Option<Expr> {
- let start = p.next_start();
- p.eat_assert(Token::Include);
-
- expr(p).map(|path| {
- Expr::Include(Box::new(IncludeExpr { span: p.span_from(start), path }))
+fn include_expr(p: &mut Parser) -> ParseResult {
+ p.perform(NodeKind::IncludeExpr, |p| {
+ p.eat_assert(&NodeKind::Include);
+ expr(p)?;
+ Ok(())
})
}
/// Parse an identifier.
-fn ident(p: &mut Parser) -> Option<Ident> {
- if let Some(Token::Ident(string)) = p.peek() {
- Some(Ident {
- span: p.eat_span(),
- string: string.into(),
- })
- } else {
- p.expected("identifier");
- None
+fn ident(p: &mut Parser) -> ParseResult {
+ match p.peek() {
+ Some(NodeKind::Ident(_)) => {
+ p.eat();
+ Ok(())
+ }
+ _ => {
+ p.expected("identifier");
+ Err(())
+ }
}
}
/// Parse a control flow body.
-fn body(p: &mut Parser) -> Option<Expr> {
+fn body(p: &mut Parser) -> ParseResult {
match p.peek() {
- Some(Token::LeftBracket) => Some(template(p)),
- Some(Token::LeftBrace) => Some(block(p)),
+ Some(NodeKind::LeftBracket) => template(p),
+ Some(NodeKind::LeftBrace) => block(p),
_ => {
- p.expected_at(p.prev_end(), "body");
- None
+ p.expected_at("body");
+ return Err(());
}
}
+ Ok(())
}
diff --git a/src/parse/parser.rs b/src/parse/parser.rs
index 347d6f71..1c4c2a5c 100644
--- a/src/parse/parser.rs
+++ b/src/parse/parser.rs
@@ -1,250 +1,216 @@
-use std::ops::Range;
+use std::mem;
use super::{TokenMode, Tokens};
-use crate::diag::Error;
-use crate::source::{SourceFile, SourceId};
-use crate::syntax::{IntoSpan, Pos, Span, Token};
+use crate::syntax::{ErrorPos, Green, GreenData, GreenNode, NodeKind};
+use crate::util::EcoString;
+
+/// Allows parser methods to use the try operator. Not exposed as the parser
+/// recovers from all errors.
+pub(crate) type ParseResult<T = ()> = Result<T, ()>;
/// A convenient token-based parser.
pub struct Parser<'s> {
- /// The parsed file.
- source: &'s SourceFile,
- /// Parsing errors.
- errors: Vec<Error>,
/// An iterator over the source tokens.
tokens: Tokens<'s>,
+ /// Whether we are at the end of the file or of a group.
+ eof: bool,
+ /// The current token.
+ current: Option<NodeKind>,
+ /// The end byte index of the last non-trivia token.
+ prev_end: usize,
+ /// The start byte index of the peeked token.
+ current_start: usize,
/// The stack of open groups.
groups: Vec<GroupEntry>,
- /// The next token.
- next: Option<Token<'s>>,
- /// The peeked token.
- /// (Same as `next` except if we are at the end of group, then `None`).
- peeked: Option<Token<'s>>,
- /// The end index of the last (non-whitespace if in code mode) token.
- prev_end: usize,
- /// The start index of the peeked token.
- next_start: usize,
-}
-
-/// A logical group of tokens, e.g. `[...]`.
-struct GroupEntry {
- /// The kind of group this is. This decides which tokens will end the group.
- /// For example, a [`Group::Paren`] will be ended by
- /// [`Token::RightParen`].
- pub kind: Group,
- /// The start index of the group. Used by `Parser::end_group` to return the
- /// group's full span.
- pub start: usize,
- /// The mode the parser was in _before_ the group started (to which we go
- /// back once the group ends).
- pub prev_mode: TokenMode,
-}
-
-/// A group, confined by optional start and end delimiters.
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
-pub enum Group {
- /// A parenthesized group: `(...)`.
- Paren,
- /// A bracketed group: `[...]`.
- Bracket,
- /// A curly-braced group: `{...}`.
- Brace,
- /// A group ended by a semicolon or a line break: `;`, `\n`.
- Stmt,
- /// A group for a single expression, ended by a line break.
- Expr,
- /// A group for import items, ended by a semicolon, line break or `from`.
- Imports,
+ /// The children of the currently built node.
+ children: Vec<Green>,
}
impl<'s> Parser<'s> {
/// Create a new parser for the source string.
- pub fn new(source: &'s SourceFile) -> Self {
- let mut tokens = Tokens::new(source.src(), TokenMode::Markup);
- let next = tokens.next();
+ pub fn new(src: &'s str) -> Self {
+ let mut tokens = Tokens::new(src, TokenMode::Markup);
+ let current = tokens.next();
Self {
- source,
- errors: vec![],
tokens,
- groups: vec![],
- next,
- peeked: next,
+ eof: current.is_none(),
+ current,
prev_end: 0,
- next_start: 0,
+ current_start: 0,
+ groups: vec![],
+ children: vec![],
}
}
- /// Finish parsing and return all errors.
- pub fn finish(self) -> Vec<Error> {
- self.errors
+ /// End the parsing process and return the last child.
+ pub fn finish(self) -> Vec<Green> {
+ self.children
}
- /// The id of the parsed source file.
- pub fn id(&self) -> SourceId {
- self.source.id()
+ /// Create a new marker.
+ pub fn marker(&mut self) -> Marker {
+ Marker(self.children.len())
}
- /// Whether the end of the source string or group is reached.
- pub fn eof(&self) -> bool {
- self.peek().is_none()
+ /// Create a markup right before the trailing trivia.
+ pub fn trivia_start(&self) -> Marker {
+ let count = self
+ .children
+ .iter()
+ .rev()
+ .take_while(|node| self.is_trivia(node.kind()))
+ .count();
+ Marker(self.children.len() - count)
}
- /// Consume the next token.
- pub fn eat(&mut self) -> Option<Token<'s>> {
- let token = self.peek()?;
- self.bump();
- Some(token)
+ /// Perform a subparse that wraps its result in a node with the given kind.
+ pub fn perform<F, T>(&mut self, kind: NodeKind, f: F) -> T
+ where
+ F: FnOnce(&mut Self) -> T,
+ {
+ let prev = mem::take(&mut self.children);
+ let output = f(self);
+ let until = self.trivia_start();
+ let mut children = mem::replace(&mut self.children, prev);
+
+ if self.tokens.mode() == TokenMode::Code {
+ // Trailing trivia should not be wrapped into the new node.
+ let idx = self.children.len();
+ self.children.push(Green::default());
+ self.children.extend(children.drain(until.0 ..));
+ self.children[idx] = GreenNode::with_children(kind, children).into();
+ } else {
+ self.children.push(GreenNode::with_children(kind, children).into());
+ }
+
+ output
}
- /// Eat the next token and return its source range.
- pub fn eat_span(&mut self) -> Span {
- let start = self.next_start();
- self.eat();
- Span::new(self.id(), start, self.prev_end())
+ /// Whether the end of the source string or group is reached.
+ pub fn eof(&self) -> bool {
+ self.eof
}
- /// Consume the next token if it is the given one.
- pub fn eat_if(&mut self, t: Token) -> bool {
- if self.peek() == Some(t) {
- self.bump();
- true
- } else {
- false
+ /// Consume the current token and also trailing trivia.
+ pub fn eat(&mut self) {
+ self.prev_end = self.tokens.index();
+ self.bump();
+
+ if self.tokens.mode() == TokenMode::Code {
+ // Skip whitespace and comments.
+ while self.current.as_ref().map_or(false, |x| self.is_trivia(x)) {
+ self.bump();
+ }
}
+
+ self.repeek();
}
- /// Consume the next token if the closure maps it a to `Some`-variant.
- pub fn eat_map<T, F>(&mut self, f: F) -> Option<T>
- where
- F: FnOnce(Token<'s>) -> Option<T>,
- {
- let token = self.peek()?;
- let mapped = f(token);
- if mapped.is_some() {
- self.bump();
+ /// Eat if the current token it is the given one.
+ pub fn eat_if(&mut self, t: &NodeKind) -> bool {
+ let at = self.at(t);
+ if at {
+ self.eat();
}
- mapped
+ at
}
- /// Consume the next token if it is the given one and produce an error if
- /// not.
- pub fn eat_expect(&mut self, t: Token) -> bool {
+ /// Eat if the current token is the given one and produce an error if not.
+ pub fn eat_expect(&mut self, t: &NodeKind) -> ParseResult {
let eaten = self.eat_if(t);
if !eaten {
- self.expected_at(self.prev_end(), t.name());
+ self.expected_at(t.as_str());
}
- eaten
+ if eaten { Ok(()) } else { Err(()) }
}
- /// Consume the next token, debug-asserting that it is one of the given ones.
- pub fn eat_assert(&mut self, t: Token) {
- let next = self.eat();
- debug_assert_eq!(next, Some(t));
+ /// Eat, debug-asserting that the token is the given one.
+ pub fn eat_assert(&mut self, t: &NodeKind) {
+ debug_assert_eq!(self.peek(), Some(t));
+ self.eat();
}
- /// Consume tokens while the condition is true.
+ /// Eat tokens while the condition is true.
pub fn eat_while<F>(&mut self, mut f: F)
where
- F: FnMut(Token<'s>) -> bool,
+ F: FnMut(&NodeKind) -> bool,
{
while self.peek().map_or(false, |t| f(t)) {
self.eat();
}
}
- /// Peek at the next token without consuming it.
- pub fn peek(&self) -> Option<Token<'s>> {
- self.peeked
+ /// Eat the current token, but change its type.
+ pub fn convert(&mut self, kind: NodeKind) {
+ let marker = self.marker();
+ self.eat();
+ marker.convert(self, kind);
}
- /// Peek at the next token if it follows immediately after the last one
- /// without any whitespace in between.
- pub fn peek_direct(&self) -> Option<Token<'s>> {
- if self.next_start() == self.prev_end() {
- self.peeked
- } else {
- None
- }
+ /// Whether the current token is of the given type.
+ pub fn at(&self, kind: &NodeKind) -> bool {
+ self.peek() == Some(kind)
}
- /// Peek at the span of the next token.
- ///
- /// Has length zero if `peek()` returns `None`.
- pub fn peek_span(&self) -> Span {
- Span::new(self.id(), self.next_start(), self.next_end())
+ /// Peek at the current token without consuming it.
+ pub fn peek(&self) -> Option<&NodeKind> {
+ if self.eof { None } else { self.current.as_ref() }
}
- /// Peek at the source of the next token.
- pub fn peek_src(&self) -> &'s str {
- self.get(self.next_start() .. self.next_end())
+ /// Peek at the current token, if it follows immediately after the last one
+ /// without any trivia in between.
+ pub fn peek_direct(&self) -> Option<&NodeKind> {
+ if self.prev_end() == self.current_start() {
+ self.peek()
+ } else {
+ None
+ }
}
- /// Checks whether the next token fulfills a condition.
- ///
- /// Returns `false` if there is no next token.
- pub fn check<F>(&self, f: F) -> bool
- where
- F: FnOnce(Token<'s>) -> bool,
- {
- self.peek().map_or(false, f)
+ /// Peek at the source of the current token.
+ pub fn peek_src(&self) -> &'s str {
+ self.tokens.scanner().get(self.current_start() .. self.current_end())
}
- /// The byte index at which the last token ended.
- ///
- /// Refers to the end of the last _non-whitespace_ token in code mode.
+ /// The byte index at which the last non-trivia token ended.
pub fn prev_end(&self) -> usize {
self.prev_end
}
- /// The byte index at which the next token starts.
- pub fn next_start(&self) -> usize {
- self.next_start
+ /// The byte index at which the current token starts.
+ pub fn current_start(&self) -> usize {
+ self.current_start
}
- /// The byte index at which the next token will end.
- ///
- /// Is the same as [`next_start()`][Self::next_start] if `peek()` returns
- /// `None`.
- pub fn next_end(&self) -> usize {
+ /// The byte index at which the current token ends.
+ pub fn current_end(&self) -> usize {
self.tokens.index()
}
/// Determine the column index for the given byte index.
pub fn column(&self, index: usize) -> usize {
- self.source.byte_to_column(index).unwrap()
- }
-
- /// Slice out part of the source string.
- pub fn get(&self, range: Range<usize>) -> &'s str {
- self.source.get(range).unwrap()
- }
-
- /// The span from `start` to [`self.prev_end()`](Self::prev_end).
- pub fn span_from(&self, start: impl Into<Pos>) -> Span {
- Span::new(self.id(), start, self.prev_end())
+ self.tokens.scanner().column(index)
}
/// Continue parsing in a group.
///
/// When the end delimiter of the group is reached, all subsequent calls to
- /// `eat()` and `peek()` return `None`. Parsing can only continue with
- /// a matching call to `end_group`.
+ /// `peek()` return `None`. Parsing can only continue with a matching call
+ /// to `end_group`.
///
- /// This panics if the next token does not start the given group.
- pub fn start_group(&mut self, kind: Group, mode: TokenMode) {
- self.groups.push(GroupEntry {
- kind,
- start: self.next_start(),
- prev_mode: self.tokens.mode(),
+ /// This panics if the current token does not start the given group.
+ pub fn start_group(&mut self, kind: Group) {
+ self.groups.push(GroupEntry { kind, prev_mode: self.tokens.mode() });
+ self.tokens.set_mode(match kind {
+ Group::Bracket => TokenMode::Markup,
+ _ => TokenMode::Code,
});
- self.tokens.set_mode(mode);
self.repeek();
-
match kind {
- Group::Paren => self.eat_assert(Token::LeftParen),
- Group::Bracket => self.eat_assert(Token::LeftBracket),
- Group::Brace => self.eat_assert(Token::LeftBrace),
+ Group::Paren => self.eat_assert(&NodeKind::LeftParen),
+ Group::Bracket => self.eat_assert(&NodeKind::LeftBracket),
+ Group::Brace => self.eat_assert(&NodeKind::LeftBrace),
Group::Stmt => {}
Group::Expr => {}
Group::Imports => {}
@@ -254,130 +220,228 @@ impl<'s> Parser<'s> {
/// End the parsing of a group.
///
/// This panics if no group was started.
- pub fn end_group(&mut self) -> Span {
- let prev_mode = self.tokens.mode();
+ pub fn end_group(&mut self) {
+ let group_mode = self.tokens.mode();
let group = self.groups.pop().expect("no started group");
self.tokens.set_mode(group.prev_mode);
self.repeek();
- let mut rescan = self.tokens.mode() != prev_mode;
+ let mut rescan = self.tokens.mode() != group_mode;
// Eat the end delimiter if there is one.
if let Some((end, required)) = match group.kind {
- Group::Paren => Some((Token::RightParen, true)),
- Group::Bracket => Some((Token::RightBracket, true)),
- Group::Brace => Some((Token::RightBrace, true)),
- Group::Stmt => Some((Token::Semicolon, false)),
+ Group::Paren => Some((NodeKind::RightParen, true)),
+ Group::Bracket => Some((NodeKind::RightBracket, true)),
+ Group::Brace => Some((NodeKind::RightBrace, true)),
+ Group::Stmt => Some((NodeKind::Semicolon, false)),
Group::Expr => None,
Group::Imports => None,
} {
- if self.next == Some(end) {
+ if self.current.as_ref() == Some(&end) {
// Bump the delimeter and return. No need to rescan in this case.
- self.bump();
+ self.eat();
rescan = false;
} else if required {
- self.error(
- self.next_start() .. self.next_start(),
- format!("expected {}", end.name()),
- );
+ self.push_error(format!("expected {}", end));
}
}
// Rescan the peeked token if the mode changed.
if rescan {
+ if group_mode == TokenMode::Code {
+ self.children.truncate(self.trivia_start().0);
+ }
+
self.tokens.jump(self.prev_end());
- self.bump();
+ self.prev_end = self.tokens.index();
+ self.current_start = self.tokens.index();
+ self.current = self.tokens.next();
+ self.repeek();
}
+ }
- Span::new(self.id(), group.start, self.prev_end())
+ /// Low-level bump that consumes exactly one token without special trivia
+ /// handling.
+ fn bump(&mut self) {
+ let kind = self.current.take().unwrap();
+ let len = self.tokens.index() - self.current_start;
+ self.children.push(GreenData::new(kind, len).into());
+ self.current_start = self.tokens.index();
+ self.current = self.tokens.next();
}
- /// Add an error with location and message.
- pub fn error(&mut self, span: impl IntoSpan, message: impl Into<String>) {
- self.errors.push(Error::new(span.into_span(self.id()), message));
+ /// Take another look at the current token to recheck whether it ends a
+ /// group.
+ fn repeek(&mut self) {
+ self.eof = match &self.current {
+ Some(NodeKind::RightParen) => self.inside(Group::Paren),
+ Some(NodeKind::RightBracket) => self.inside(Group::Bracket),
+ Some(NodeKind::RightBrace) => self.inside(Group::Brace),
+ Some(NodeKind::Semicolon) => self.inside(Group::Stmt),
+ Some(NodeKind::From) => self.inside(Group::Imports),
+ Some(NodeKind::Space(n)) => *n >= 1 && self.stop_at_newline(),
+ Some(_) => false,
+ None => true,
+ };
}
- /// Add an error that `what` was expected at the given span.
- pub fn expected_at(&mut self, span: impl IntoSpan, what: &str) {
- self.error(span, format!("expected {}", what));
+ /// Returns whether the given type can be skipped over.
+ fn is_trivia(&self, token: &NodeKind) -> bool {
+ Self::is_trivia_ext(token, self.stop_at_newline())
}
- /// Eat the next token and add an error that it is not the expected `thing`.
- pub fn expected(&mut self, what: &str) {
- let before = self.next_start();
- if let Some(found) = self.eat() {
- let after = self.prev_end();
- self.error(
- before .. after,
- format!("expected {}, found {}", what, found.name()),
- );
- } else {
- self.expected_at(self.next_start(), what);
+ /// Returns whether the given type can be skipped over given the current
+ /// newline mode.
+ fn is_trivia_ext(token: &NodeKind, stop_at_newline: bool) -> bool {
+ match token {
+ NodeKind::Space(n) => *n == 0 || !stop_at_newline,
+ NodeKind::LineComment => true,
+ NodeKind::BlockComment => true,
+ _ => false,
}
}
- /// Eat the next token and add an error that it is unexpected.
+ /// Whether the active group must end at a newline.
+ fn stop_at_newline(&self) -> bool {
+ matches!(
+ self.groups.last().map(|group| group.kind),
+ Some(Group::Stmt | Group::Expr | Group::Imports)
+ )
+ }
+
+ /// Whether we are inside the given group.
+ fn inside(&self, kind: Group) -> bool {
+ self.groups.iter().any(|g| g.kind == kind)
+ }
+}
+
+/// Error handling.
+impl Parser<'_> {
+ /// Push an error into the children list.
+ pub fn push_error(&mut self, msg: impl Into<EcoString>) {
+ let error = NodeKind::Error(ErrorPos::Full, msg.into());
+ self.children.push(GreenData::new(error, 0).into());
+ }
+
+ /// Eat the current token and add an error that it is unexpected.
pub fn unexpected(&mut self) {
- let before = self.next_start();
- if let Some(found) = self.eat() {
- let after = self.prev_end();
- self.error(before .. after, format!("unexpected {}", found.name()));
+ match self.peek() {
+ Some(found) => {
+ let msg = format!("unexpected {}", found);
+ let error = NodeKind::Error(ErrorPos::Full, msg.into());
+ self.perform(error, Self::eat);
+ }
+ None => self.push_error("unexpected end of file"),
}
}
- /// Move to the next token.
- fn bump(&mut self) {
- self.prev_end = self.tokens.index().into();
- self.next_start = self.tokens.index().into();
- self.next = self.tokens.next();
-
- if self.tokens.mode() == TokenMode::Code {
- // Skip whitespace and comments.
- while match self.next {
- Some(Token::Space(n)) => n < 1 || !self.stop_at_newline(),
- Some(Token::LineComment(_)) => true,
- Some(Token::BlockComment(_)) => true,
- _ => false,
- } {
- self.next_start = self.tokens.index().into();
- self.next = self.tokens.next();
+ /// Eat the current token and add an error that it is not the expected `thing`.
+ pub fn expected(&mut self, thing: &str) {
+ match self.peek() {
+ Some(found) => {
+ let msg = format!("expected {}, found {}", thing, found);
+ let error = NodeKind::Error(ErrorPos::Full, msg.into());
+ self.perform(error, Self::eat);
}
+ None => self.expected_at(thing),
}
+ }
- self.repeek();
+ /// Add an error that the `thing` was expected at the end of the last
+ /// non-trivia token.
+ pub fn expected_at(&mut self, thing: &str) {
+ self.trivia_start().expected(self, thing);
}
+}
- /// Take another look at the next token to recheck whether it ends a group.
- fn repeek(&mut self) {
- self.peeked = self.next;
- let token = match self.next {
- Some(token) => token,
- None => return,
- };
+/// A marker that indicates where a node may start.
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+pub struct Marker(usize);
- if match token {
- Token::RightParen => self.inside(Group::Paren),
- Token::RightBracket => self.inside(Group::Bracket),
- Token::RightBrace => self.inside(Group::Brace),
- Token::Semicolon => self.inside(Group::Stmt),
- Token::From => self.inside(Group::Imports),
- Token::Space(n) => n >= 1 && self.stop_at_newline(),
- _ => false,
- } {
- self.peeked = None;
+impl Marker {
+ /// Perform a subparse that wraps all children after the marker in a node
+ /// with the given kind.
+ pub fn perform<T, F>(self, p: &mut Parser, kind: NodeKind, f: F) -> T
+ where
+ F: FnOnce(&mut Parser) -> T,
+ {
+ let success = f(p);
+ self.end(p, kind);
+ success
+ }
+
+ /// Wrap all children after the marker (excluding trailing trivia) in a node
+ /// with the given `kind`.
+ pub fn end(self, p: &mut Parser, kind: NodeKind) {
+ let until = p.trivia_start();
+ let children = p.children.drain(self.0 .. until.0).collect();
+ p.children
+ .insert(self.0, GreenNode::with_children(kind, children).into());
+ }
+
+ /// Wrap all children that do not fulfill the predicate in error nodes.
+ pub fn filter_children<F>(self, p: &mut Parser, f: F)
+ where
+ F: Fn(&Green) -> Result<(), &'static str>,
+ {
+ for child in &mut p.children[self.0 ..] {
+ if (p.tokens.mode() == TokenMode::Markup
+ || !Parser::is_trivia_ext(child.kind(), false))
+ && !child.kind().is_error()
+ {
+ if let Err(msg) = f(child) {
+ let error = NodeKind::Error(ErrorPos::Full, msg.into());
+ let inner = mem::take(child);
+ *child = GreenNode::with_child(error, inner).into();
+ }
+ }
}
}
- /// Whether the active group ends at a newline.
- fn stop_at_newline(&self) -> bool {
- matches!(
- self.groups.last().map(|group| group.kind),
- Some(Group::Stmt | Group::Expr | Group::Imports)
- )
+ /// Insert an error message that `what` was expected at the marker position.
+ pub fn expected(self, p: &mut Parser, what: &str) {
+ let msg = format!("expected {}", what);
+ let error = NodeKind::Error(ErrorPos::Full, msg.into());
+ p.children.insert(self.0, GreenData::new(error, 0).into());
}
- /// Whether we are inside the given group.
- fn inside(&self, kind: Group) -> bool {
- self.groups.iter().any(|g| g.kind == kind)
+ /// Peek at the child directly after the marker.
+ pub fn peek<'a>(self, p: &'a Parser) -> Option<&'a Green> {
+ p.children.get(self.0)
}
+
+ /// Convert the child directly after marker.
+ pub fn convert(self, p: &mut Parser, kind: NodeKind) {
+ if let Some(child) = p.children.get_mut(self.0) {
+ child.convert(kind);
+ }
+ }
+}
+
+/// A logical group of tokens, e.g. `[...]`.
+struct GroupEntry {
+ /// The kind of group this is. This decides which tokens will end the group.
+ /// For example, a [`Group::Paren`] will be ended by
+ /// [`Token::RightParen`].
+ pub kind: Group,
+ /// The mode the parser was in _before_ the group started (to which we go
+ /// back once the group ends).
+ pub prev_mode: TokenMode,
+}
+
+/// A group, confined by optional start and end delimiters.
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+pub enum Group {
+ /// A bracketed group: `[...]`.
+ Bracket,
+ /// A curly-braced group: `{...}`.
+ Brace,
+ /// A parenthesized group: `(...)`.
+ Paren,
+ /// A group ended by a semicolon or a line break: `;`, `\n`.
+ Stmt,
+ /// A group for a single expression, ended by a line break.
+ Expr,
+ /// A group for import items, ended by a semicolon, line break or `from`.
+ Imports,
}
diff --git a/src/parse/resolve.rs b/src/parse/resolve.rs
index 1b323847..e15ae339 100644
--- a/src/parse/resolve.rs
+++ b/src/parse/resolve.rs
@@ -1,5 +1,5 @@
-use super::{is_newline, Scanner};
-use crate::syntax::{Ident, RawNode, Span};
+use super::{is_ident, is_newline, Scanner};
+use crate::syntax::ast::RawNode;
use crate::util::EcoString;
/// Resolve all escape sequences in a string.
@@ -25,11 +25,9 @@ pub fn resolve_string(string: &str) -> EcoString {
let sequence = s.eat_while(|c| c.is_ascii_hexdigit());
let _terminated = s.eat_if('}');
- if let Some(c) = resolve_hex(sequence) {
- out.push(c);
- } else {
- // TODO: Feedback that unicode escape sequence is wrong.
- out.push_str(s.eaten_from(start));
+ match resolve_hex(sequence) {
+ Some(c) => out.push(c),
+ None => out.push_str(s.eaten_from(start)),
}
}
@@ -48,19 +46,17 @@ pub fn resolve_hex(sequence: &str) -> Option<char> {
}
/// Resolve the language tag and trims the raw text.
-pub fn resolve_raw(span: Span, column: usize, backticks: usize, text: &str) -> RawNode {
+pub fn resolve_raw(column: usize, backticks: usize, text: &str) -> RawNode {
if backticks > 1 {
let (tag, inner) = split_at_lang_tag(text);
let (text, block) = trim_and_split_raw(column, inner);
RawNode {
- span,
- lang: Ident::new(tag, span.with_end(span.start + tag.len())),
+ lang: is_ident(tag).then(|| tag.into()),
text: text.into(),
block,
}
} else {
RawNode {
- span,
lang: None,
text: split_lines(text).join("\n").into(),
block: false,
@@ -140,7 +136,6 @@ fn split_lines(text: &str) -> Vec<&str> {
#[cfg(test)]
#[rustfmt::skip]
mod tests {
- use crate::syntax::Span;
use super::*;
#[test]
@@ -190,7 +185,7 @@ mod tests {
text: &str,
block: bool,
) {
- let node = resolve_raw(Span::detached(), column, backticks, raw);
+ let node = resolve_raw(column, backticks, raw);
assert_eq!(node.lang.as_deref(), lang);
assert_eq!(node.text, text);
assert_eq!(node.block, block);
diff --git a/src/parse/scanner.rs b/src/parse/scanner.rs
index 8e3e4278..ea06a2e0 100644
--- a/src/parse/scanner.rs
+++ b/src/parse/scanner.rs
@@ -1,5 +1,7 @@
use std::slice::SliceIndex;
+use unicode_xid::UnicodeXID;
+
/// A featureful char-based scanner.
#[derive(Copy, Clone)]
pub struct Scanner<'s> {
@@ -114,6 +116,12 @@ impl<'s> Scanner<'s> {
self.index = index;
}
+ /// The full source string.
+ #[inline]
+ pub fn src(&self) -> &'s str {
+ &self.src
+ }
+
/// Slice out part of the source string.
#[inline]
pub fn get<I>(&self, index: I) -> &'s str
@@ -150,6 +158,16 @@ impl<'s> Scanner<'s> {
// optimized away in some cases.
self.src.get(start .. self.index).unwrap_or_default()
}
+
+ /// The column index of a given index in the source string.
+ #[inline]
+ pub fn column(&self, index: usize) -> usize {
+ self.src[.. index]
+ .chars()
+ .rev()
+ .take_while(|&c| !is_newline(c))
+ .count()
+ }
}
/// Whether this character denotes a newline.
@@ -163,3 +181,30 @@ pub fn is_newline(character: char) -> bool {
'\u{0085}' | '\u{2028}' | '\u{2029}'
)
}
+
+/// Whether a string is a valid unicode identifier.
+///
+/// In addition to what is specified in the [Unicode Standard][uax31], we allow:
+/// - `_` as a starting character,
+/// - `_` and `-` as continuing characters.
+///
+/// [uax31]: http://www.unicode.org/reports/tr31/
+#[inline]
+pub fn is_ident(string: &str) -> bool {
+ let mut chars = string.chars();
+ chars
+ .next()
+ .map_or(false, |c| is_id_start(c) && chars.all(is_id_continue))
+}
+
+/// Whether a character can start an identifier.
+#[inline]
+pub fn is_id_start(c: char) -> bool {
+ c.is_xid_start() || c == '_'
+}
+
+/// Whether a character can continue an identifier.
+#[inline]
+pub fn is_id_continue(c: char) -> bool {
+ c.is_xid_continue() || c == '_' || c == '-'
+}
diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs
index 5f969452..96dfd9d1 100644
--- a/src/parse/tokens.rs
+++ b/src/parse/tokens.rs
@@ -1,6 +1,13 @@
-use super::{is_newline, Scanner};
+use std::rc::Rc;
+
+use super::{
+ is_id_continue, is_id_start, is_newline, resolve_hex, resolve_raw, resolve_string,
+ Scanner,
+};
use crate::geom::{AngularUnit, LengthUnit};
-use crate::syntax::*;
+use crate::syntax::ast::{MathNode, RawNode};
+use crate::syntax::{ErrorPos, NodeKind};
+use crate::util::EcoString;
/// An iterator over the tokens of a string of source code.
pub struct Tokens<'s> {
@@ -59,7 +66,7 @@ impl<'s> Tokens<'s> {
}
impl<'s> Iterator for Tokens<'s> {
- type Item = Token<'s>;
+ type Item = NodeKind;
/// Parse the next token in the source code.
#[inline]
@@ -68,19 +75,21 @@ impl<'s> Iterator for Tokens<'s> {
let c = self.s.eat()?;
Some(match c {
// Blocks and templates.
- '[' => Token::LeftBracket,
- ']' => Token::RightBracket,
- '{' => Token::LeftBrace,
- '}' => Token::RightBrace,
+ '[' => NodeKind::LeftBracket,
+ ']' => NodeKind::RightBracket,
+ '{' => NodeKind::LeftBrace,
+ '}' => NodeKind::RightBrace,
// Whitespace.
- ' ' if self.s.check_or(true, |c| !c.is_whitespace()) => Token::Space(0),
+ ' ' if self.s.check_or(true, |c| !c.is_whitespace()) => NodeKind::Space(0),
c if c.is_whitespace() => self.whitespace(),
// Comments with special case for URLs.
'/' if self.s.eat_if('*') => self.block_comment(),
'/' if !self.maybe_in_url() && self.s.eat_if('/') => self.line_comment(),
- '*' if self.s.eat_if('/') => Token::Invalid(self.s.eaten_from(start)),
+ '*' if self.s.eat_if('/') => {
+ NodeKind::Unknown(self.s.eaten_from(start).into())
+ }
// Other things.
_ => match self.mode {
@@ -93,7 +102,7 @@ impl<'s> Iterator for Tokens<'s> {
impl<'s> Tokens<'s> {
#[inline]
- fn markup(&mut self, start: usize, c: char) -> Token<'s> {
+ fn markup(&mut self, start: usize, c: char) -> NodeKind {
match c {
// Escape sequences.
'\\' => self.backslash(),
@@ -102,13 +111,15 @@ impl<'s> Tokens<'s> {
'#' => self.hash(),
// Markup.
- '~' => Token::Tilde,
- '*' => Token::Star,
- '_' => Token::Underscore,
+ '~' => NodeKind::NonBreakingSpace,
+ '*' => NodeKind::Strong,
+ '_' => NodeKind::Emph,
'`' => self.raw(),
'$' => self.math(),
- '-' => self.hyph(start),
- '=' if self.s.check_or(true, |c| c == '=' || c.is_whitespace()) => Token::Eq,
+ '-' => self.hyph(),
+ '=' if self.s.check_or(true, |c| c == '=' || c.is_whitespace()) => {
+ NodeKind::Eq
+ }
c if c == '.' || c.is_ascii_digit() => self.numbering(start, c),
// Plain text.
@@ -116,35 +127,35 @@ impl<'s> Tokens<'s> {
}
}
- fn code(&mut self, start: usize, c: char) -> Token<'s> {
+ fn code(&mut self, start: usize, c: char) -> NodeKind {
match c {
// Parens.
- '(' => Token::LeftParen,
- ')' => Token::RightParen,
+ '(' => NodeKind::LeftParen,
+ ')' => NodeKind::RightParen,
// Length two.
- '=' if self.s.eat_if('=') => Token::EqEq,
- '!' if self.s.eat_if('=') => Token::ExclEq,
- '<' if self.s.eat_if('=') => Token::LtEq,
- '>' if self.s.eat_if('=') => Token::GtEq,
- '+' if self.s.eat_if('=') => Token::PlusEq,
- '-' if self.s.eat_if('=') => Token::HyphEq,
- '*' if self.s.eat_if('=') => Token::StarEq,
- '/' if self.s.eat_if('=') => Token::SlashEq,
- '.' if self.s.eat_if('.') => Token::Dots,
- '=' if self.s.eat_if('>') => Token::Arrow,
+ '=' if self.s.eat_if('=') => NodeKind::EqEq,
+ '!' if self.s.eat_if('=') => NodeKind::ExclEq,
+ '<' if self.s.eat_if('=') => NodeKind::LtEq,
+ '>' if self.s.eat_if('=') => NodeKind::GtEq,
+ '+' if self.s.eat_if('=') => NodeKind::PlusEq,
+ '-' if self.s.eat_if('=') => NodeKind::HyphEq,
+ '*' if self.s.eat_if('=') => NodeKind::StarEq,
+ '/' if self.s.eat_if('=') => NodeKind::SlashEq,
+ '.' if self.s.eat_if('.') => NodeKind::Dots,
+ '=' if self.s.eat_if('>') => NodeKind::Arrow,
// Length one.
- ',' => Token::Comma,
- ';' => Token::Semicolon,
- ':' => Token::Colon,
- '+' => Token::Plus,
- '-' => Token::Hyph,
- '*' => Token::Star,
- '/' => Token::Slash,
- '=' => Token::Eq,
- '<' => Token::Lt,
- '>' => Token::Gt,
+ ',' => NodeKind::Comma,
+ ';' => NodeKind::Semicolon,
+ ':' => NodeKind::Colon,
+ '+' => NodeKind::Plus,
+ '-' => NodeKind::Minus,
+ '*' => NodeKind::Star,
+ '/' => NodeKind::Slash,
+ '=' => NodeKind::Eq,
+ '<' => NodeKind::Lt,
+ '>' => NodeKind::Gt,
// Identifiers.
c if is_id_start(c) => self.ident(start),
@@ -159,12 +170,12 @@ impl<'s> Tokens<'s> {
// Strings.
'"' => self.string(),
- _ => Token::Invalid(self.s.eaten_from(start)),
+ _ => NodeKind::Unknown(self.s.eaten_from(start).into()),
}
}
#[inline]
- fn text(&mut self, start: usize) -> Token<'s> {
+ fn text(&mut self, start: usize) -> NodeKind {
macro_rules! table {
($($c:literal)|*) => {{
let mut t = [false; 128];
@@ -186,10 +197,10 @@ impl<'s> Tokens<'s> {
TABLE.get(c as usize).copied().unwrap_or_else(|| c.is_whitespace())
});
- Token::Text(self.s.eaten_from(start))
+ NodeKind::Text(self.s.eaten_from(start).into())
}
- fn whitespace(&mut self) -> Token<'s> {
+ fn whitespace(&mut self) -> NodeKind {
self.s.uneat();
// Count the number of newlines.
@@ -208,73 +219,81 @@ impl<'s> Tokens<'s> {
}
}
- Token::Space(newlines)
+ NodeKind::Space(newlines)
}
- fn backslash(&mut self) -> Token<'s> {
- if let Some(c) = self.s.peek() {
- match c {
+ fn backslash(&mut self) -> NodeKind {
+ match self.s.peek() {
+ Some(c) => match c {
// Backslash and comments.
'\\' | '/' |
// Parenthesis and hashtag.
'[' | ']' | '{' | '}' | '#' |
// Markup.
'*' | '_' | '=' | '~' | '`' | '$' => {
- let start = self.s.index();
self.s.eat_assert(c);
- Token::Text(&self.s.eaten_from(start))
+ NodeKind::Text(c.into())
}
'u' if self.s.rest().starts_with("u{") => {
self.s.eat_assert('u');
self.s.eat_assert('{');
- Token::UnicodeEscape(UnicodeEscapeToken {
- // Allow more than `ascii_hexdigit` for better error recovery.
- sequence: self.s.eat_while(|c| c.is_ascii_alphanumeric()),
- terminated: self.s.eat_if('}'),
- })
+ let sequence = self.s.eat_while(|c| c.is_ascii_alphanumeric());
+ if self.s.eat_if('}') {
+ if let Some(c) = resolve_hex(&sequence) {
+ NodeKind::UnicodeEscape(c)
+ } else {
+ NodeKind::Error(
+ ErrorPos::Full,
+ "invalid unicode escape sequence".into(),
+ )
+ }
+ } else {
+ NodeKind::Error(
+ ErrorPos::End,
+ "expected closing brace".into(),
+ )
+ }
}
- c if c.is_whitespace() => Token::Backslash,
- _ => Token::Text("\\"),
- }
- } else {
- Token::Backslash
+ c if c.is_whitespace() => NodeKind::Linebreak,
+ _ => NodeKind::Text('\\'.into()),
+ },
+ None => NodeKind::Linebreak,
}
}
#[inline]
- fn hash(&mut self) -> Token<'s> {
+ fn hash(&mut self) -> NodeKind {
if self.s.check_or(false, is_id_start) {
let read = self.s.eat_while(is_id_continue);
- if let Some(keyword) = keyword(read) {
- keyword
- } else {
- Token::Ident(read)
+ match keyword(read) {
+ Some(keyword) => keyword,
+ None => NodeKind::Ident(read.into()),
}
} else {
- Token::Text("#")
+ NodeKind::Text("#".into())
}
}
- fn hyph(&mut self, start: usize) -> Token<'s> {
+ fn hyph(&mut self) -> NodeKind {
if self.s.eat_if('-') {
if self.s.eat_if('-') {
- Token::HyphHyphHyph
+ NodeKind::EmDash
} else {
- Token::HyphHyph
+ NodeKind::EnDash
}
} else if self.s.check_or(true, char::is_whitespace) {
- Token::Hyph
+ NodeKind::Minus
} else {
- Token::Text(self.s.eaten_from(start))
+ NodeKind::Text("-".into())
}
}
- fn numbering(&mut self, start: usize, c: char) -> Token<'s> {
+ fn numbering(&mut self, start: usize, c: char) -> NodeKind {
let number = if c != '.' {
self.s.eat_while(|c| c.is_ascii_digit());
let read = self.s.eaten_from(start);
if !self.s.eat_if('.') {
- return Token::Text(read);
+ return NodeKind::Text(self.s.eaten_from(start).into());
}
read.parse().ok()
} else {
@@ -282,13 +301,15 @@ impl<'s> Tokens<'s> {
};
if self.s.check_or(true, char::is_whitespace) {
- Token::Numbering(number)
+ NodeKind::EnumNumbering(number)
} else {
- Token::Text(self.s.eaten_from(start))
+ NodeKind::Text(self.s.eaten_from(start).into())
}
}
- fn raw(&mut self) -> Token<'s> {
+ fn raw(&mut self) -> NodeKind {
+ let column = self.s.column(self.s.index() - 1);
+
let mut backticks = 1;
while self.s.eat_if('`') {
backticks += 1;
@@ -296,7 +317,11 @@ impl<'s> Tokens<'s> {
// Special case for empty inline block.
if backticks == 2 {
- return Token::Raw(RawToken { text: "", backticks: 1, terminated: true });
+ return NodeKind::Raw(Rc::new(RawNode {
+ text: EcoString::new(),
+ lang: None,
+ block: false,
+ }));
}
let start = self.s.index();
@@ -310,17 +335,30 @@ impl<'s> Tokens<'s> {
}
}
- let terminated = found == backticks;
- let end = self.s.index() - if terminated { found } else { 0 };
+ if found == backticks {
+ let end = self.s.index() - found as usize;
+ NodeKind::Raw(Rc::new(resolve_raw(
+ column,
+ backticks,
+ self.s.get(start .. end).into(),
+ )))
+ } else {
+ let remaining = backticks - found;
+ let noun = if remaining == 1 { "backtick" } else { "backticks" };
- Token::Raw(RawToken {
- text: self.s.get(start .. end),
- backticks,
- terminated,
- })
+ NodeKind::Error(
+ ErrorPos::End,
+ if found == 0 {
+ format!("expected {} {}", remaining, noun)
+ } else {
+ format!("expected {} more {}", remaining, noun)
+ }
+ .into(),
+ )
+ }
}
- fn math(&mut self) -> Token<'s> {
+ fn math(&mut self) -> NodeKind {
let mut display = false;
if self.s.eat_if('[') {
display = true;
@@ -350,25 +388,36 @@ impl<'s> Tokens<'s> {
(true, true) => 2,
};
- Token::Math(MathToken {
- formula: self.s.get(start .. end),
- display,
- terminated,
- })
+ if terminated {
+ NodeKind::Math(Rc::new(MathNode {
+ formula: self.s.get(start .. end).into(),
+ display,
+ }))
+ } else {
+ NodeKind::Error(
+ ErrorPos::End,
+ if !display || (!escaped && dollar) {
+ "expected closing dollar sign"
+ } else {
+ "expected closing bracket and dollar sign"
+ }
+ .into(),
+ )
+ }
}
- fn ident(&mut self, start: usize) -> Token<'s> {
+ fn ident(&mut self, start: usize) -> NodeKind {
self.s.eat_while(is_id_continue);
match self.s.eaten_from(start) {
- "none" => Token::None,
- "auto" => Token::Auto,
- "true" => Token::Bool(true),
- "false" => Token::Bool(false),
- id => keyword(id).unwrap_or(Token::Ident(id)),
+ "none" => NodeKind::None,
+ "auto" => NodeKind::Auto,
+ "true" => NodeKind::Bool(true),
+ "false" => NodeKind::Bool(false),
+ id => keyword(id).unwrap_or(NodeKind::Ident(id.into())),
}
}
- fn number(&mut self, start: usize, c: char) -> Token<'s> {
+ fn number(&mut self, start: usize, c: char) -> NodeKind {
// Read the first part (integer or fractional depending on `first`).
self.s.eat_while(|c| c.is_ascii_digit());
@@ -396,55 +445,56 @@ impl<'s> Tokens<'s> {
// Find out whether it is a simple number.
if suffix.is_empty() {
- if let Ok(int) = number.parse::<i64>() {
- return Token::Int(int);
- } else if let Ok(float) = number.parse::<f64>() {
- return Token::Float(float);
+ if let Ok(i) = number.parse::<i64>() {
+ return NodeKind::Int(i);
}
}
- // Otherwise parse into the fitting numeric type.
- let build = match suffix {
- "%" => Token::Percent,
- "fr" => Token::Fraction,
- "pt" => |x| Token::Length(x, LengthUnit::Pt),
- "mm" => |x| Token::Length(x, LengthUnit::Mm),
- "cm" => |x| Token::Length(x, LengthUnit::Cm),
- "in" => |x| Token::Length(x, LengthUnit::In),
- "rad" => |x| Token::Angle(x, AngularUnit::Rad),
- "deg" => |x| Token::Angle(x, AngularUnit::Deg),
- _ => return Token::Invalid(all),
- };
-
- if let Ok(float) = number.parse::<f64>() {
- build(float)
+ if let Ok(f) = number.parse::<f64>() {
+ match suffix {
+ "" => NodeKind::Float(f),
+ "%" => NodeKind::Percentage(f),
+ "fr" => NodeKind::Fraction(f),
+ "pt" => NodeKind::Length(f, LengthUnit::Pt),
+ "mm" => NodeKind::Length(f, LengthUnit::Mm),
+ "cm" => NodeKind::Length(f, LengthUnit::Cm),
+ "in" => NodeKind::Length(f, LengthUnit::In),
+ "deg" => NodeKind::Angle(f, AngularUnit::Deg),
+ "rad" => NodeKind::Angle(f, AngularUnit::Rad),
+ _ => {
+ return NodeKind::Unknown(all.into());
+ }
+ }
} else {
- Token::Invalid(all)
+ NodeKind::Unknown(all.into())
}
}
- fn string(&mut self) -> Token<'s> {
+
+ fn string(&mut self) -> NodeKind {
let mut escaped = false;
- Token::Str(StrToken {
- string: self.s.eat_until(|c| {
- if c == '"' && !escaped {
- true
- } else {
- escaped = c == '\\' && !escaped;
- false
- }
- }),
- terminated: self.s.eat_if('"'),
- })
- }
+ let string = resolve_string(self.s.eat_until(|c| {
+ if c == '"' && !escaped {
+ true
+ } else {
+ escaped = c == '\\' && !escaped;
+ false
+ }
+ }));
- fn line_comment(&mut self) -> Token<'s> {
- Token::LineComment(self.s.eat_until(is_newline))
+ if self.s.eat_if('"') {
+ NodeKind::Str(string)
+ } else {
+ NodeKind::Error(ErrorPos::End, "expected quote".into())
+ }
}
- fn block_comment(&mut self) -> Token<'s> {
- let start = self.s.index();
+ fn line_comment(&mut self) -> NodeKind {
+ self.s.eat_until(is_newline);
+ NodeKind::LineComment
+ }
+ fn block_comment(&mut self) -> NodeKind {
let mut state = '_';
let mut depth = 1;
@@ -466,10 +516,7 @@ impl<'s> Tokens<'s> {
}
}
- let terminated = depth == 0;
- let end = self.s.index() - if terminated { 2 } else { 0 };
-
- Token::BlockComment(self.s.get(start .. end))
+ NodeKind::BlockComment
}
fn maybe_in_url(&self) -> bool {
@@ -477,24 +524,24 @@ impl<'s> Tokens<'s> {
}
}
-fn keyword(ident: &str) -> Option<Token<'static>> {
+fn keyword(ident: &str) -> Option<NodeKind> {
Some(match ident {
- "not" => Token::Not,
- "and" => Token::And,
- "or" => Token::Or,
- "with" => Token::With,
- "let" => Token::Let,
- "if" => Token::If,
- "else" => Token::Else,
- "for" => Token::For,
- "in" => Token::In,
- "while" => Token::While,
- "break" => Token::Break,
- "continue" => Token::Continue,
- "return" => Token::Return,
- "import" => Token::Import,
- "include" => Token::Include,
- "from" => Token::From,
+ "not" => NodeKind::Not,
+ "and" => NodeKind::And,
+ "or" => NodeKind::Or,
+ "with" => NodeKind::With,
+ "let" => NodeKind::Let,
+ "if" => NodeKind::If,
+ "else" => NodeKind::Else,
+ "for" => NodeKind::For,
+ "in" => NodeKind::In,
+ "while" => NodeKind::While,
+ "break" => NodeKind::Break,
+ "continue" => NodeKind::Continue,
+ "return" => NodeKind::Return,
+ "import" => NodeKind::Import,
+ "include" => NodeKind::Include,
+ "from" => NodeKind::From,
_ => return None,
})
}
@@ -506,24 +553,45 @@ mod tests {
use super::*;
+ use ErrorPos::*;
+ use NodeKind::*;
use Option::None;
- use Token::{Ident, *};
use TokenMode::{Code, Markup};
- const fn UnicodeEscape(sequence: &str, terminated: bool) -> Token {
- Token::UnicodeEscape(UnicodeEscapeToken { sequence, terminated })
+ fn UnicodeEscape(c: char) -> NodeKind {
+ NodeKind::UnicodeEscape(c)
}
- const fn Raw(text: &str, backticks: usize, terminated: bool) -> Token {
- Token::Raw(RawToken { text, backticks, terminated })
+ fn Error(pos: ErrorPos, message: &str) -> NodeKind {
+ NodeKind::Error(pos, message.into())
}
- const fn Math(formula: &str, display: bool, terminated: bool) -> Token {
- Token::Math(MathToken { formula, display, terminated })
+ fn Raw(text: &str, lang: Option<&str>, block: bool) -> NodeKind {
+ NodeKind::Raw(Rc::new(RawNode {
+ text: text.into(),
+ lang: lang.map(Into::into),
+ block,
+ }))
}
- const fn Str(string: &str, terminated: bool) -> Token {
- Token::Str(StrToken { string, terminated })
+ fn Math(formula: &str, display: bool) -> NodeKind {
+ NodeKind::Math(Rc::new(MathNode { formula: formula.into(), display }))
+ }
+
+ fn Str(string: &str) -> NodeKind {
+ NodeKind::Str(string.into())
+ }
+
+ fn Text(string: &str) -> NodeKind {
+ NodeKind::Text(string.into())
+ }
+
+ fn Ident(ident: &str) -> NodeKind {
+ NodeKind::Ident(ident.into())
+ }
+
+ fn Invalid(invalid: &str) -> NodeKind {
+ NodeKind::Unknown(invalid.into())
}
/// Building blocks for suffix testing.
@@ -541,40 +609,6 @@ mod tests {
/// - '/': symbols
const BLOCKS: &str = " a1/";
- /// Suffixes described by four-tuples of:
- ///
- /// - block the suffix is part of
- /// - mode in which the suffix is applicable
- /// - the suffix string
- /// - the resulting suffix token
- const SUFFIXES: &[(char, Option<TokenMode>, &str, Token)] = &[
- // Whitespace suffixes.
- (' ', None, " ", Space(0)),
- (' ', None, "\n", Space(1)),
- (' ', None, "\r", Space(1)),
- (' ', None, "\r\n", Space(1)),
- // Letter suffixes.
- ('a', Some(Markup), "hello", Text("hello")),
- ('a', Some(Markup), "💚", Text("💚")),
- ('a', Some(Code), "val", Ident("val")),
- ('a', Some(Code), "α", Ident("α")),
- ('a', Some(Code), "_", Ident("_")),
- // Number suffixes.
- ('1', Some(Code), "2", Int(2)),
- ('1', Some(Code), ".2", Float(0.2)),
- // Symbol suffixes.
- ('/', None, "[", LeftBracket),
- ('/', None, "//", LineComment("")),
- ('/', None, "/**/", BlockComment("")),
- ('/', Some(Markup), "*", Star),
- ('/', Some(Markup), "$ $", Math(" ", false, true)),
- ('/', Some(Markup), r"\\", Text(r"\")),
- ('/', Some(Markup), "#let", Let),
- ('/', Some(Code), "(", LeftParen),
- ('/', Some(Code), ":", Colon),
- ('/', Some(Code), "+=", PlusEq),
- ];
-
macro_rules! t {
(Both $($tts:tt)*) => {
t!(Markup $($tts)*);
@@ -584,8 +618,42 @@ mod tests {
// Test without suffix.
t!(@$mode: $src => $($token),*);
+ // Suffixes described by four-tuples of:
+ //
+ // - block the suffix is part of
+ // - mode in which the suffix is applicable
+ // - the suffix string
+ // - the resulting suffix NodeKind
+ let suffixes: &[(char, Option<TokenMode>, &str, NodeKind)] = &[
+ // Whitespace suffixes.
+ (' ', None, " ", Space(0)),
+ (' ', None, "\n", Space(1)),
+ (' ', None, "\r", Space(1)),
+ (' ', None, "\r\n", Space(1)),
+ // Letter suffixes.
+ ('a', Some(Markup), "hello", Text("hello")),
+ ('a', Some(Markup), "💚", Text("💚")),
+ ('a', Some(Code), "val", Ident("val")),
+ ('a', Some(Code), "α", Ident("α")),
+ ('a', Some(Code), "_", Ident("_")),
+ // Number suffixes.
+ ('1', Some(Code), "2", Int(2)),
+ ('1', Some(Code), ".2", Float(0.2)),
+ // Symbol suffixes.
+ ('/', None, "[", LeftBracket),
+ ('/', None, "//", LineComment),
+ ('/', None, "/**/", BlockComment),
+ ('/', Some(Markup), "*", Strong),
+ ('/', Some(Markup), "$ $", Math(" ", false)),
+ ('/', Some(Markup), r"\\", Text("\\")),
+ ('/', Some(Markup), "#let", Let),
+ ('/', Some(Code), "(", LeftParen),
+ ('/', Some(Code), ":", Colon),
+ ('/', Some(Code), "+=", PlusEq),
+ ];
+
// Test with each applicable suffix.
- for &(block, mode, suffix, token) in SUFFIXES {
+ for &(block, mode, suffix, ref token) in suffixes {
let src = $src;
#[allow(unused_variables)]
let blocks = BLOCKS;
@@ -599,7 +667,7 @@ mod tests {
(@$mode:ident: $src:expr => $($token:expr),*) => {{
let src = $src;
let found = Tokens::new(&src, $mode).collect::<Vec<_>>();
- let expected = vec![$($token),*];
+ let expected = vec![$($token.clone()),*];
check(&src, found, expected);
}};
}
@@ -671,7 +739,7 @@ mod tests {
// Test text ends.
t!(Markup[""]: "hello " => Text("hello"), Space(0));
- t!(Markup[""]: "hello~" => Text("hello"), Tilde);
+ t!(Markup[""]: "hello~" => Text("hello"), NonBreakingSpace);
}
#[test]
@@ -698,31 +766,31 @@ mod tests {
t!(Markup[" /"]: r#"\""# => Text(r"\"), Text("\""));
// Test basic unicode escapes.
- t!(Markup: r"\u{}" => UnicodeEscape("", true));
- t!(Markup: r"\u{2603}" => UnicodeEscape("2603", true));
- t!(Markup: r"\u{P}" => UnicodeEscape("P", true));
+ t!(Markup: r"\u{}" => Error(Full, "invalid unicode escape sequence"));
+ t!(Markup: r"\u{2603}" => UnicodeEscape('☃'));
+ t!(Markup: r"\u{P}" => Error(Full, "invalid unicode escape sequence"));
// Test unclosed unicode escapes.
- t!(Markup[" /"]: r"\u{" => UnicodeEscape("", false));
- t!(Markup[" /"]: r"\u{1" => UnicodeEscape("1", false));
- t!(Markup[" /"]: r"\u{26A4" => UnicodeEscape("26A4", false));
- t!(Markup[" /"]: r"\u{1Q3P" => UnicodeEscape("1Q3P", false));
- t!(Markup: r"\u{1🏕}" => UnicodeEscape("1", false), Text("🏕"), RightBrace);
+ t!(Markup[" /"]: r"\u{" => Error(End, "expected closing brace"));
+ t!(Markup[" /"]: r"\u{1" => Error(End, "expected closing brace"));
+ t!(Markup[" /"]: r"\u{26A4" => Error(End, "expected closing brace"));
+ t!(Markup[" /"]: r"\u{1Q3P" => Error(End, "expected closing brace"));
+ t!(Markup: r"\u{1🏕}" => Error(End, "expected closing brace"), Text("🏕"), RightBrace);
}
#[test]
fn test_tokenize_markup_symbols() {
// Test markup tokens.
- t!(Markup[" a1"]: "*" => Star);
- t!(Markup: "_" => Underscore);
+ t!(Markup[" a1"]: "*" => Strong);
+ t!(Markup: "_" => Emph);
t!(Markup[""]: "===" => Eq, Eq, Eq);
t!(Markup["a1/"]: "= " => Eq, Space(0));
- t!(Markup: "~" => Tilde);
- t!(Markup[" "]: r"\" => Backslash);
- t!(Markup["a "]: r"a--" => Text("a"), HyphHyph);
- t!(Markup["a1/"]: "- " => Hyph, Space(0));
- t!(Markup[" "]: "." => Numbering(None));
- t!(Markup[" "]: "1." => Numbering(Some(1)));
+ t!(Markup: "~" => NonBreakingSpace);
+ t!(Markup[" "]: r"\" => Linebreak);
+ t!(Markup["a "]: r"a--" => Text("a"), EnDash);
+ t!(Markup["a1/"]: "- " => Minus, Space(0));
+ t!(Markup[" "]: "." => EnumNumbering(None));
+ t!(Markup[" "]: "1." => EnumNumbering(Some(1)));
t!(Markup[" "]: "1.a" => Text("1."), Text("a"));
t!(Markup[" /"]: "a1." => Text("a1."));
}
@@ -734,7 +802,7 @@ mod tests {
t!(Code: ";" => Semicolon);
t!(Code: ":" => Colon);
t!(Code: "+" => Plus);
- t!(Code: "-" => Hyph);
+ t!(Code: "-" => Minus);
t!(Code[" a1"]: "*" => Star);
t!(Code[" a1"]: "/" => Slash);
t!(Code: "=" => Eq);
@@ -756,10 +824,10 @@ mod tests {
t!(Code[" a/"]: "..." => Dots, Invalid("."));
// Test hyphen as symbol vs part of identifier.
- t!(Code[" /"]: "-1" => Hyph, Int(1));
- t!(Code[" /"]: "-a" => Hyph, Ident("a"));
- t!(Code[" /"]: "--1" => Hyph, Hyph, Int(1));
- t!(Code[" /"]: "--_a" => Hyph, Hyph, Ident("_a"));
+ t!(Code[" /"]: "-1" => Minus, Int(1));
+ t!(Code[" /"]: "-a" => Minus, Ident("a"));
+ t!(Code[" /"]: "--1" => Minus, Minus, Int(1));
+ t!(Code[" /"]: "--_a" => Minus, Minus, Ident("_a"));
t!(Code[" /"]: "a-b" => Ident("a-b"));
}
@@ -776,13 +844,13 @@ mod tests {
("import", Import),
];
- for &(s, t) in &list {
+ for (s, t) in list.clone() {
t!(Markup[" "]: format!("#{}", s) => t);
t!(Markup[" "]: format!("#{0}#{0}", s) => t, t);
- t!(Markup[" /"]: format!("# {}", s) => Token::Text("#"), Space(0), Text(s));
+ t!(Markup[" /"]: format!("# {}", s) => Text("#"), Space(0), Text(s));
}
- for &(s, t) in &list {
+ for (s, t) in list {
t!(Code[" "]: s => t);
t!(Markup[" /"]: s => Text(s));
}
@@ -796,45 +864,43 @@ mod tests {
#[test]
fn test_tokenize_raw_blocks() {
- let empty = Raw("", 1, true);
-
// Test basic raw block.
- t!(Markup: "``" => empty);
- t!(Markup: "`raw`" => Raw("raw", 1, true));
- t!(Markup[""]: "`]" => Raw("]", 1, false));
+ t!(Markup: "``" => Raw("", None, false));
+ t!(Markup: "`raw`" => Raw("raw", None, false));
+ t!(Markup[""]: "`]" => Error(End, "expected 1 backtick"));
// Test special symbols in raw block.
- t!(Markup: "`[brackets]`" => Raw("[brackets]", 1, true));
- t!(Markup[""]: r"`\`` " => Raw(r"\", 1, true), Raw(" ", 1, false));
+ t!(Markup: "`[brackets]`" => Raw("[brackets]", None, false));
+ t!(Markup[""]: r"`\`` " => Raw(r"\", None, false), Error(End, "expected 1 backtick"));
// Test separated closing backticks.
- t!(Markup: "```not `y`e`t```" => Raw("not `y`e`t", 3, true));
+ t!(Markup: "```not `y`e`t```" => Raw("`y`e`t", Some("not"), false));
// Test more backticks.
- t!(Markup: "``nope``" => empty, Text("nope"), empty);
- t!(Markup: "````🚀````" => Raw("🚀", 4, true));
- t!(Markup[""]: "`````👩‍🚀````noend" => Raw("👩‍🚀````noend", 5, false));
- t!(Markup[""]: "````raw``````" => Raw("raw", 4, true), empty);
+ t!(Markup: "``nope``" => Raw("", None, false), Text("nope"), Raw("", None, false));
+ t!(Markup: "````🚀````" => Raw("", None, false));
+ t!(Markup[""]: "`````👩‍🚀````noend" => Error(End, "expected 5 backticks"));
+ t!(Markup[""]: "````raw``````" => Raw("", Some("raw"), false), Raw("", None, false));
}
#[test]
fn test_tokenize_math_formulas() {
// Test basic formula.
- t!(Markup: "$$" => Math("", false, true));
- t!(Markup: "$x$" => Math("x", false, true));
- t!(Markup: r"$\\$" => Math(r"\\", false, true));
- t!(Markup: "$[x + y]$" => Math("x + y", true, true));
- t!(Markup: r"$[\\]$" => Math(r"\\", true, true));
+ t!(Markup: "$$" => Math("", false));
+ t!(Markup: "$x$" => Math("x", false));
+ t!(Markup: r"$\\$" => Math(r"\\", false));
+ t!(Markup: "$[x + y]$" => Math("x + y", true));
+ t!(Markup: r"$[\\]$" => Math(r"\\", true));
// Test unterminated.
- t!(Markup[""]: "$x" => Math("x", false, false));
- t!(Markup[""]: "$[x" => Math("x", true, false));
- t!(Markup[""]: "$[x]\n$" => Math("x]\n$", true, false));
+ t!(Markup[""]: "$x" => Error(End, "expected closing dollar sign"));
+ t!(Markup[""]: "$[x" => Error(End, "expected closing bracket and dollar sign"));
+ t!(Markup[""]: "$[x]\n$" => Error(End, "expected closing bracket and dollar sign"));
// Test escape sequences.
- t!(Markup: r"$\$x$" => Math(r"\$x", false, true));
- t!(Markup: r"$[\\\]$]$" => Math(r"\\\]$", true, true));
- t!(Markup[""]: r"$[ ]\\$" => Math(r" ]\\$", true, false));
+ t!(Markup: r"$\$x$" => Math(r"\$x", false));
+ t!(Markup: r"$[\\\]$]$" => Math(r"\\\]$", true));
+ t!(Markup[""]: r"$[ ]\\$" => Error(End, "expected closing bracket and dollar sign"));
}
#[test]
@@ -896,8 +962,8 @@ mod tests {
let nums = ints.iter().map(|&(k, v)| (k, v as f64)).chain(floats);
let suffixes = [
- ("%", Percent as fn(f64) -> Token<'static>),
- ("fr", Fraction as fn(f64) -> Token<'static>),
+ ("%", Percentage as fn(f64) -> NodeKind),
+ ("fr", Fraction as fn(f64) -> NodeKind),
("mm", |x| Length(x, LengthUnit::Mm)),
("pt", |x| Length(x, LengthUnit::Pt)),
("cm", |x| Length(x, LengthUnit::Cm)),
@@ -922,62 +988,62 @@ mod tests {
#[test]
fn test_tokenize_strings() {
// Test basic strings.
- t!(Code: "\"hi\"" => Str("hi", true));
- t!(Code: "\"hi\nthere\"" => Str("hi\nthere", true));
- t!(Code: "\"🌎\"" => Str("🌎", true));
+ t!(Code: "\"hi\"" => Str("hi"));
+ t!(Code: "\"hi\nthere\"" => Str("hi\nthere"));
+ t!(Code: "\"🌎\"" => Str("🌎"));
// Test unterminated.
- t!(Code[""]: "\"hi" => Str("hi", false));
+ t!(Code[""]: "\"hi" => Error(End, "expected quote"));
// Test escaped quote.
- t!(Code: r#""a\"bc""# => Str(r#"a\"bc"#, true));
- t!(Code[""]: r#""\""# => Str(r#"\""#, false));
+ t!(Code: r#""a\"bc""# => Str("a\"bc"));
+ t!(Code[""]: r#""\""# => Error(End, "expected quote"));
}
#[test]
fn test_tokenize_line_comments() {
// Test line comment with no trailing newline.
- t!(Both[""]: "//" => LineComment(""));
+ t!(Both[""]: "//" => LineComment);
// Test line comment ends at newline.
- t!(Both["a1/"]: "//bc\n" => LineComment("bc"), Space(1));
- t!(Both["a1/"]: "// bc \n" => LineComment(" bc "), Space(1));
- t!(Both["a1/"]: "//bc\r\n" => LineComment("bc"), Space(1));
+ t!(Both["a1/"]: "//bc\n" => LineComment, Space(1));
+ t!(Both["a1/"]: "// bc \n" => LineComment, Space(1));
+ t!(Both["a1/"]: "//bc\r\n" => LineComment, Space(1));
// Test nested line comments.
- t!(Both["a1/"]: "//a//b\n" => LineComment("a//b"), Space(1));
+ t!(Both["a1/"]: "//a//b\n" => LineComment, Space(1));
}
#[test]
fn test_tokenize_block_comments() {
// Test basic block comments.
- t!(Both[""]: "/*" => BlockComment(""));
- t!(Both: "/**/" => BlockComment(""));
- t!(Both: "/*🏞*/" => BlockComment("🏞"));
- t!(Both: "/*\n*/" => BlockComment("\n"));
+ t!(Both[""]: "/*" => BlockComment);
+ t!(Both: "/**/" => BlockComment);
+ t!(Both: "/*🏞*/" => BlockComment);
+ t!(Both: "/*\n*/" => BlockComment);
// Test depth 1 and 2 nested block comments.
- t!(Both: "/* /* */ */" => BlockComment(" /* */ "));
- t!(Both: "/*/*/**/*/*/" => BlockComment("/*/**/*/"));
+ t!(Both: "/* /* */ */" => BlockComment);
+ t!(Both: "/*/*/**/*/*/" => BlockComment);
// Test two nested, one unclosed block comments.
- t!(Both[""]: "/*/*/**/*/" => BlockComment("/*/**/*/"));
+ t!(Both[""]: "/*/*/**/*/" => BlockComment);
// Test all combinations of up to two following slashes and stars.
- t!(Both[""]: "/*" => BlockComment(""));
- t!(Both[""]: "/*/" => BlockComment("/"));
- t!(Both[""]: "/**" => BlockComment("*"));
- t!(Both[""]: "/*//" => BlockComment("//"));
- t!(Both[""]: "/*/*" => BlockComment("/*"));
- t!(Both[""]: "/**/" => BlockComment(""));
- t!(Both[""]: "/***" => BlockComment("**"));
+ t!(Both[""]: "/*" => BlockComment);
+ t!(Both[""]: "/*/" => BlockComment);
+ t!(Both[""]: "/**" => BlockComment);
+ t!(Both[""]: "/*//" => BlockComment);
+ t!(Both[""]: "/*/*" => BlockComment);
+ t!(Both[""]: "/**/" => BlockComment);
+ t!(Both[""]: "/***" => BlockComment);
}
#[test]
fn test_tokenize_invalid() {
// Test invalidly closed block comments.
- t!(Both: "*/" => Token::Invalid("*/"));
- t!(Both: "/**/*/" => BlockComment(""), Token::Invalid("*/"));
+ t!(Both: "*/" => Invalid("*/"));
+ t!(Both: "/**/*/" => BlockComment, Invalid("*/"));
// Test invalid expressions.
t!(Code: r"\" => Invalid(r"\"));
@@ -990,6 +1056,6 @@ mod tests {
// Test invalid number suffixes.
t!(Code[" /"]: "1foo" => Invalid("1foo"));
t!(Code: "1p%" => Invalid("1p"), Invalid("%"));
- t!(Code: "1%%" => Percent(1.0), Invalid("%"));
+ t!(Code: "1%%" => Percentage(1.0), Invalid("%"));
}
}
diff --git a/src/source.rs b/src/source.rs
index c9164f90..85f48491 100644
--- a/src/source.rs
+++ b/src/source.rs
@@ -8,8 +8,11 @@ use std::rc::Rc;
use serde::{Deserialize, Serialize};
+use crate::diag::TypResult;
use crate::loading::{FileHash, Loader};
-use crate::parse::{is_newline, Scanner};
+use crate::parse::{is_newline, parse, Scanner};
+use crate::syntax::ast::Markup;
+use crate::syntax::{GreenNode, RedNode};
use crate::util::PathExt;
#[cfg(feature = "codespan-reporting")]
@@ -124,6 +127,7 @@ pub struct SourceFile {
path: PathBuf,
src: String,
line_starts: Vec<usize>,
+ root: Rc<GreenNode>,
}
impl SourceFile {
@@ -134,11 +138,23 @@ impl SourceFile {
Self {
id,
path: path.normalize(),
+ root: parse(&src),
src,
line_starts,
}
}
+ /// The file's abstract syntax tree.
+ pub fn ast(&self) -> TypResult<Markup> {
+ let red = RedNode::from_root(self.root.clone(), self.id);
+ let errors = red.errors();
+ if errors.is_empty() {
+ Ok(red.cast().unwrap())
+ } else {
+ Err(Box::new(errors))
+ }
+ }
+
/// Create a source file without a real id and path, usually for testing.
pub fn detached(src: impl Into<String>) -> Self {
Self::new(SourceId(0), Path::new(""), src.into())
diff --git a/src/syntax/ast.rs b/src/syntax/ast.rs
new file mode 100644
index 00000000..288c749a
--- /dev/null
+++ b/src/syntax/ast.rs
@@ -0,0 +1,988 @@
+//! A typed layer over the red-green tree.
+
+use std::ops::Deref;
+
+use super::{Green, GreenData, NodeKind, RedNode, RedRef, Span};
+use crate::geom::{AngularUnit, LengthUnit};
+use crate::util::EcoString;
+
+/// A typed AST node.
+pub trait TypedNode: Sized {
+ /// Convert from a red node to a typed node.
+ fn from_red(value: RedRef) -> Option<Self>;
+
+ /// A reference to the underlying red node.
+ fn as_red(&self) -> RedRef<'_>;
+
+ /// The source code location.
+ fn span(&self) -> Span {
+ self.as_red().span()
+ }
+}
+
+macro_rules! node {
+ ($(#[$attr:meta])* $name:ident) => {
+ node!{$(#[$attr])* $name: $name}
+ };
+ ($(#[$attr:meta])* $name:ident: $variant:ident) => {
+ node!{$(#[$attr])* $name: NodeKind::$variant}
+ };
+ ($(#[$attr:meta])* $name:ident: $($variant:pat)|*) => {
+ #[derive(Debug, Clone, PartialEq)]
+ #[repr(transparent)]
+ $(#[$attr])*
+ pub struct $name(RedNode);
+
+ impl TypedNode for $name {
+ fn from_red(node: RedRef) -> Option<Self> {
+ if matches!(node.kind(), $($variant)|*) {
+ Some(Self(node.own()))
+ } else {
+ None
+ }
+ }
+
+ fn as_red(&self) -> RedRef<'_> {
+ self.0.as_ref()
+ }
+ }
+ };
+}
+
+node! {
+ /// The syntactical root capable of representing a full parsed document.
+ Markup
+}
+
+impl Markup {
+ /// The markup nodes.
+ pub fn nodes(&self) -> impl Iterator<Item = MarkupNode> + '_ {
+ self.0.children().filter_map(|node| match node.kind() {
+ NodeKind::Space(_) => Some(MarkupNode::Space),
+ NodeKind::Linebreak => Some(MarkupNode::Linebreak),
+ NodeKind::Parbreak => Some(MarkupNode::Parbreak),
+ NodeKind::Strong => Some(MarkupNode::Strong),
+ NodeKind::Emph => Some(MarkupNode::Emph),
+ NodeKind::Text(s) => Some(MarkupNode::Text(s.clone())),
+ NodeKind::UnicodeEscape(c) => Some(MarkupNode::Text((*c).into())),
+ NodeKind::EnDash => Some(MarkupNode::Text("\u{2013}".into())),
+ NodeKind::EmDash => Some(MarkupNode::Text("\u{2014}".into())),
+ NodeKind::NonBreakingSpace => Some(MarkupNode::Text("\u{00A0}".into())),
+ NodeKind::Math(math) => Some(MarkupNode::Math(math.as_ref().clone())),
+ NodeKind::Raw(raw) => Some(MarkupNode::Raw(raw.as_ref().clone())),
+ NodeKind::Heading => node.cast().map(MarkupNode::Heading),
+ NodeKind::List => node.cast().map(MarkupNode::List),
+ NodeKind::Enum => node.cast().map(MarkupNode::Enum),
+ _ => node.cast().map(MarkupNode::Expr),
+ })
+ }
+}
+
+/// A single piece of markup.
+#[derive(Debug, Clone, PartialEq)]
+pub enum MarkupNode {
+ /// Whitespace containing less than two newlines.
+ Space,
+ /// A forced line break: `\`.
+ Linebreak,
+ /// A paragraph break: Two or more newlines.
+ Parbreak,
+ /// Strong text was enabled / disabled: `*`.
+ Strong,
+ /// Emphasized text was enabled / disabled: `_`.
+ Emph,
+ /// Plain text.
+ Text(EcoString),
+ /// A raw block with optional syntax highlighting: `` `...` ``.
+ Raw(RawNode),
+ /// A math formula: `$a^2 = b^2 + c^2$`.
+ Math(MathNode),
+ /// A section heading: `= Introduction`.
+ Heading(HeadingNode),
+ /// An item in an unordered list: `- ...`.
+ List(ListNode),
+ /// An item in an enumeration (ordered list): `1. ...`.
+ Enum(EnumNode),
+ /// An expression.
+ Expr(Expr),
+}
+
+/// A raw block with optional syntax highlighting: `` `...` ``.
+#[derive(Debug, Clone, PartialEq)]
+pub struct RawNode {
+ /// An optional identifier specifying the language to syntax-highlight in.
+ pub lang: Option<EcoString>,
+ /// The raw text, determined as the raw string between the backticks trimmed
+ /// according to the above rules.
+ pub text: EcoString,
+ /// Whether the element is block-level, that is, it has 3+ backticks
+ /// and contains at least one newline.
+ pub block: bool,
+}
+
+/// A math formula: `$a^2 + b^2 = c^2$`.
+#[derive(Debug, Clone, PartialEq)]
+pub struct MathNode {
+ /// The formula between the dollars / brackets.
+ pub formula: EcoString,
+ /// Whether the formula is display-level, that is, it is surrounded by
+ /// `$[..]$`.
+ pub display: bool,
+}
+
+node! {
+ /// A section heading: `= Introduction`.
+ HeadingNode: Heading
+}
+
+impl HeadingNode {
+ /// The contents of the heading.
+ pub fn body(&self) -> Markup {
+ self.0.cast_first_child().expect("heading is missing markup body")
+ }
+
+ /// The section depth (numer of equals signs).
+ pub fn level(&self) -> usize {
+ self.0.children().filter(|n| n.kind() == &NodeKind::Eq).count()
+ }
+}
+
+node! {
+ /// An item in an unordered list: `- ...`.
+ ListNode: List
+}
+
+impl ListNode {
+ /// The contents of the list item.
+ pub fn body(&self) -> Markup {
+ self.0.cast_first_child().expect("list node is missing body")
+ }
+}
+
+node! {
+ /// An item in an enumeration (ordered list): `1. ...`.
+ EnumNode: Enum
+}
+
+impl EnumNode {
+ /// The contents of the list item.
+ pub fn body(&self) -> Markup {
+ self.0.cast_first_child().expect("enum node is missing body")
+ }
+
+ /// The number, if any.
+ pub fn number(&self) -> Option<usize> {
+ self.0
+ .children()
+ .find_map(|node| match node.kind() {
+ NodeKind::EnumNumbering(num) => Some(num.clone()),
+ _ => None,
+ })
+ .expect("enum node is missing number")
+ }
+}
+
+/// An expression.
+#[derive(Debug, Clone, PartialEq)]
+pub enum Expr {
+ /// A literal: `1`, `true`, ...
+ Lit(Lit),
+ /// An identifier: `left`.
+ Ident(Ident),
+ /// An array expression: `(1, "hi", 12cm)`.
+ Array(ArrayExpr),
+ /// A dictionary expression: `(thickness: 3pt, pattern: dashed)`.
+ Dict(DictExpr),
+ /// A template expression: `[*Hi* there!]`.
+ Template(TemplateExpr),
+ /// A grouped expression: `(1 + 2)`.
+ Group(GroupExpr),
+ /// A block expression: `{ let x = 1; x + 2 }`.
+ Block(BlockExpr),
+ /// A unary operation: `-x`.
+ Unary(UnaryExpr),
+ /// A binary operation: `a + b`.
+ Binary(BinaryExpr),
+ /// An invocation of a function: `f(x, y)`.
+ Call(CallExpr),
+ /// A closure expression: `(x, y) => z`.
+ Closure(ClosureExpr),
+ /// A with expression: `f with (x, y: 1)`.
+ With(WithExpr),
+ /// A let expression: `let x = 1`.
+ Let(LetExpr),
+ /// An if-else expression: `if x { y } else { z }`.
+ If(IfExpr),
+ /// A while loop expression: `while x { y }`.
+ While(WhileExpr),
+ /// A for loop expression: `for x in y { z }`.
+ For(ForExpr),
+ /// An import expression: `import a, b, c from "utils.typ"`.
+ Import(ImportExpr),
+ /// An include expression: `include "chapter1.typ"`.
+ Include(IncludeExpr),
+}
+
+impl TypedNode for Expr {
+ fn from_red(node: RedRef) -> Option<Self> {
+ match node.kind() {
+ NodeKind::Ident(_) => node.cast().map(Self::Ident),
+ NodeKind::Array => node.cast().map(Self::Array),
+ NodeKind::Dict => node.cast().map(Self::Dict),
+ NodeKind::Template => node.cast().map(Self::Template),
+ NodeKind::Group => node.cast().map(Self::Group),
+ NodeKind::Block => node.cast().map(Self::Block),
+ NodeKind::Unary => node.cast().map(Self::Unary),
+ NodeKind::Binary => node.cast().map(Self::Binary),
+ NodeKind::Call => node.cast().map(Self::Call),
+ NodeKind::Closure => node.cast().map(Self::Closure),
+ NodeKind::WithExpr => node.cast().map(Self::With),
+ NodeKind::LetExpr => node.cast().map(Self::Let),
+ NodeKind::IfExpr => node.cast().map(Self::If),
+ NodeKind::WhileExpr => node.cast().map(Self::While),
+ NodeKind::ForExpr => node.cast().map(Self::For),
+ NodeKind::ImportExpr => node.cast().map(Self::Import),
+ NodeKind::IncludeExpr => node.cast().map(Self::Include),
+ _ => node.cast().map(Self::Lit),
+ }
+ }
+
+ fn as_red(&self) -> RedRef<'_> {
+ match self {
+ Self::Lit(v) => v.as_red(),
+ Self::Ident(v) => v.as_red(),
+ Self::Array(v) => v.as_red(),
+ Self::Dict(v) => v.as_red(),
+ Self::Template(v) => v.as_red(),
+ Self::Group(v) => v.as_red(),
+ Self::Block(v) => v.as_red(),
+ Self::Unary(v) => v.as_red(),
+ Self::Binary(v) => v.as_red(),
+ Self::Call(v) => v.as_red(),
+ Self::Closure(v) => v.as_red(),
+ Self::With(v) => v.as_red(),
+ Self::Let(v) => v.as_red(),
+ Self::If(v) => v.as_red(),
+ Self::While(v) => v.as_red(),
+ Self::For(v) => v.as_red(),
+ Self::Import(v) => v.as_red(),
+ Self::Include(v) => v.as_red(),
+ }
+ }
+}
+
+impl Expr {
+ /// Whether the expression can be shortened in markup with a hashtag.
+ pub fn has_short_form(&self) -> bool {
+ matches!(self,
+ Self::Ident(_)
+ | Self::Call(_)
+ | Self::Let(_)
+ | Self::If(_)
+ | Self::While(_)
+ | Self::For(_)
+ | Self::Import(_)
+ | Self::Include(_)
+ )
+ }
+}
+
+node! {
+ /// A literal: `1`, `true`, ...
+ Lit: NodeKind::None
+ | NodeKind::Auto
+ | NodeKind::Bool(_)
+ | NodeKind::Int(_)
+ | NodeKind::Float(_)
+ | NodeKind::Length(_, _)
+ | NodeKind::Angle(_, _)
+ | NodeKind::Percentage(_)
+ | NodeKind::Fraction(_)
+ | NodeKind::Str(_)
+}
+
+impl Lit {
+ /// The kind of literal.
+ pub fn kind(&self) -> LitKind {
+ match *self.0.kind() {
+ NodeKind::None => LitKind::None,
+ NodeKind::Auto => LitKind::Auto,
+ NodeKind::Bool(v) => LitKind::Bool(v),
+ NodeKind::Int(v) => LitKind::Int(v),
+ NodeKind::Float(v) => LitKind::Float(v),
+ NodeKind::Length(v, unit) => LitKind::Length(v, unit),
+ NodeKind::Angle(v, unit) => LitKind::Angle(v, unit),
+ NodeKind::Percentage(v) => LitKind::Percent(v),
+ NodeKind::Fraction(v) => LitKind::Fractional(v),
+ NodeKind::Str(ref v) => LitKind::Str(v.clone()),
+ _ => panic!("literal is of wrong kind"),
+ }
+ }
+}
+
+/// The kind of a literal.
+#[derive(Debug, Clone, PartialEq)]
+pub enum LitKind {
+ /// The none literal: `none`.
+ None,
+ /// The auto literal: `auto`.
+ Auto,
+ /// A boolean literal: `true`, `false`.
+ Bool(bool),
+ /// An integer literal: `120`.
+ Int(i64),
+ /// A floating-point literal: `1.2`, `10e-4`.
+ Float(f64),
+ /// A length literal: `12pt`, `3cm`.
+ Length(f64, LengthUnit),
+ /// An angle literal: `1.5rad`, `90deg`.
+ Angle(f64, AngularUnit),
+ /// A percent literal: `50%`.
+ ///
+ /// _Note_: `50%` is stored as `50.0` here, but as `0.5` in the
+ /// corresponding [value](crate::geom::Relative).
+ Percent(f64),
+ /// A fraction unit literal: `1fr`.
+ Fractional(f64),
+ /// A string literal: `"hello!"`.
+ Str(EcoString),
+}
+
+node! {
+ /// An array expression: `(1, "hi", 12cm)`.
+ ArrayExpr: Array
+}
+
+impl ArrayExpr {
+ /// The array items.
+ pub fn items(&self) -> impl Iterator<Item = Expr> + '_ {
+ self.0.children().filter_map(RedRef::cast)
+ }
+}
+
+node! {
+ /// A dictionary expression: `(thickness: 3pt, pattern: dashed)`.
+ DictExpr: Dict
+}
+
+impl DictExpr {
+ /// The named dictionary items.
+ pub fn items(&self) -> impl Iterator<Item = Named> + '_ {
+ self.0.children().filter_map(RedRef::cast)
+ }
+}
+
+node! {
+ /// A pair of a name and an expression: `pattern: dashed`.
+ Named
+}
+
+impl Named {
+ /// The name: `pattern`.
+ pub fn name(&self) -> Ident {
+ self.0.cast_first_child().expect("named pair is missing name")
+ }
+
+ /// The right-hand side of the pair: `dashed`.
+ pub fn expr(&self) -> Expr {
+ self.0.cast_last_child().expect("named pair is missing expression")
+ }
+}
+
+node! {
+ /// A template expression: `[*Hi* there!]`.
+ TemplateExpr: Template
+}
+
+impl TemplateExpr {
+ /// The contents of the template.
+ pub fn body(&self) -> Markup {
+ self.0.cast_first_child().expect("template is missing body")
+ }
+}
+
+node! {
+ /// A grouped expression: `(1 + 2)`.
+ GroupExpr: Group
+}
+
+impl GroupExpr {
+ /// The wrapped expression.
+ pub fn expr(&self) -> Expr {
+ self.0.cast_first_child().expect("group is missing expression")
+ }
+}
+
+node! {
+ /// A block expression: `{ let x = 1; x + 2 }`.
+ BlockExpr: Block
+}
+
+impl BlockExpr {
+ /// The list of expressions contained in the block.
+ pub fn exprs(&self) -> impl Iterator<Item = Expr> + '_ {
+ self.0.children().filter_map(RedRef::cast)
+ }
+}
+
+node! {
+ /// A unary operation: `-x`.
+ UnaryExpr: Unary
+}
+
+impl UnaryExpr {
+ /// The operator: `-`.
+ pub fn op(&self) -> UnOp {
+ self.0
+ .children()
+ .find_map(|node| UnOp::from_token(node.kind()))
+ .expect("unary expression is missing operator")
+ }
+
+ /// The expression to operator on: `x`.
+ pub fn expr(&self) -> Expr {
+ self.0.cast_last_child().expect("unary expression is missing child")
+ }
+}
+
+/// A unary operator.
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+pub enum UnOp {
+ /// The plus operator: `+`.
+ Pos,
+ /// The negation operator: `-`.
+ Neg,
+ /// The boolean `not`.
+ Not,
+}
+
+impl UnOp {
+ /// Try to convert the token into a unary operation.
+ pub fn from_token(token: &NodeKind) -> Option<Self> {
+ Some(match token {
+ NodeKind::Plus => Self::Pos,
+ NodeKind::Minus => Self::Neg,
+ NodeKind::Not => Self::Not,
+ _ => return None,
+ })
+ }
+
+ /// The precedence of this operator.
+ pub fn precedence(self) -> usize {
+ match self {
+ Self::Pos | Self::Neg => 7,
+ Self::Not => 4,
+ }
+ }
+
+ /// The string representation of this operation.
+ pub fn as_str(self) -> &'static str {
+ match self {
+ Self::Pos => "+",
+ Self::Neg => "-",
+ Self::Not => "not",
+ }
+ }
+}
+
+node! {
+ /// A binary operation: `a + b`.
+ BinaryExpr: Binary
+}
+
+impl BinaryExpr {
+ /// The binary operator: `+`.
+ pub fn op(&self) -> BinOp {
+ self.0
+ .children()
+ .find_map(|node| BinOp::from_token(node.kind()))
+ .expect("binary expression is missing operator")
+ }
+
+ /// The left-hand side of the operation: `a`.
+ pub fn lhs(&self) -> Expr {
+ self.0
+ .cast_first_child()
+ .expect("binary expression is missing left-hand side")
+ }
+
+ /// The right-hand side of the operation: `b`.
+ pub fn rhs(&self) -> Expr {
+ self.0
+ .cast_last_child()
+ .expect("binary expression is missing right-hand side")
+ }
+}
+
+/// A binary operator.
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+pub enum BinOp {
+ /// The addition operator: `+`.
+ Add,
+ /// The subtraction operator: `-`.
+ Sub,
+ /// The multiplication operator: `*`.
+ Mul,
+ /// The division operator: `/`.
+ Div,
+ /// The short-circuiting boolean `and`.
+ And,
+ /// The short-circuiting boolean `or`.
+ Or,
+ /// The equality operator: `==`.
+ Eq,
+ /// The inequality operator: `!=`.
+ Neq,
+ /// The less-than operator: `<`.
+ Lt,
+ /// The less-than or equal operator: `<=`.
+ Leq,
+ /// The greater-than operator: `>`.
+ Gt,
+ /// The greater-than or equal operator: `>=`.
+ Geq,
+ /// The assignment operator: `=`.
+ Assign,
+ /// The add-assign operator: `+=`.
+ AddAssign,
+ /// The subtract-assign oeprator: `-=`.
+ SubAssign,
+ /// The multiply-assign operator: `*=`.
+ MulAssign,
+ /// The divide-assign operator: `/=`.
+ DivAssign,
+}
+
+impl BinOp {
+ /// Try to convert the token into a binary operation.
+ pub fn from_token(token: &NodeKind) -> Option<Self> {
+ Some(match token {
+ NodeKind::Plus => Self::Add,
+ NodeKind::Minus => Self::Sub,
+ NodeKind::Star => Self::Mul,
+ NodeKind::Slash => Self::Div,
+ NodeKind::And => Self::And,
+ NodeKind::Or => Self::Or,
+ NodeKind::EqEq => Self::Eq,
+ NodeKind::ExclEq => Self::Neq,
+ NodeKind::Lt => Self::Lt,
+ NodeKind::LtEq => Self::Leq,
+ NodeKind::Gt => Self::Gt,
+ NodeKind::GtEq => Self::Geq,
+ NodeKind::Eq => Self::Assign,
+ NodeKind::PlusEq => Self::AddAssign,
+ NodeKind::HyphEq => Self::SubAssign,
+ NodeKind::StarEq => Self::MulAssign,
+ NodeKind::SlashEq => Self::DivAssign,
+ _ => return None,
+ })
+ }
+
+ /// The precedence of this operator.
+ pub fn precedence(self) -> usize {
+ match self {
+ Self::Mul | Self::Div => 6,
+ Self::Add | Self::Sub => 5,
+ Self::Eq | Self::Neq | Self::Lt | Self::Leq | Self::Gt | Self::Geq => 4,
+ Self::And => 3,
+ Self::Or => 2,
+ Self::Assign
+ | Self::AddAssign
+ | Self::SubAssign
+ | Self::MulAssign
+ | Self::DivAssign => 1,
+ }
+ }
+
+ /// The associativity of this operator.
+ pub fn associativity(self) -> Associativity {
+ match self {
+ Self::Add
+ | Self::Sub
+ | Self::Mul
+ | Self::Div
+ | Self::And
+ | Self::Or
+ | Self::Eq
+ | Self::Neq
+ | Self::Lt
+ | Self::Leq
+ | Self::Gt
+ | Self::Geq => Associativity::Left,
+ Self::Assign
+ | Self::AddAssign
+ | Self::SubAssign
+ | Self::MulAssign
+ | Self::DivAssign => Associativity::Right,
+ }
+ }
+
+ /// The string representation of this operation.
+ pub fn as_str(self) -> &'static str {
+ match self {
+ Self::Add => "+",
+ Self::Sub => "-",
+ Self::Mul => "*",
+ Self::Div => "/",
+ Self::And => "and",
+ Self::Or => "or",
+ Self::Eq => "==",
+ Self::Neq => "!=",
+ Self::Lt => "<",
+ Self::Leq => "<=",
+ Self::Gt => ">",
+ Self::Geq => ">=",
+ Self::Assign => "=",
+ Self::AddAssign => "+=",
+ Self::SubAssign => "-=",
+ Self::MulAssign => "*=",
+ Self::DivAssign => "/=",
+ }
+ }
+}
+
+/// The associativity of a binary operator.
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+pub enum Associativity {
+ /// Left-associative: `a + b + c` is equivalent to `(a + b) + c`.
+ Left,
+ /// Right-associative: `a = b = c` is equivalent to `a = (b = c)`.
+ Right,
+}
+
+node! {
+ /// An invocation of a function: `foo(...)`.
+ CallExpr: Call
+}
+
+impl CallExpr {
+ /// The function to call.
+ pub fn callee(&self) -> Expr {
+ self.0.cast_first_child().expect("call is missing callee")
+ }
+
+ /// The arguments to the function.
+ pub fn args(&self) -> CallArgs {
+ self.0.cast_last_child().expect("call is missing argument list")
+ }
+}
+
+node! {
+ /// The arguments to a function: `12, draw: false`.
+ CallArgs
+}
+
+impl CallArgs {
+ /// The positional and named arguments.
+ pub fn items(&self) -> impl Iterator<Item = CallArg> + '_ {
+ self.0.children().filter_map(RedRef::cast)
+ }
+}
+
+/// An argument to a function call.
+#[derive(Debug, Clone, PartialEq)]
+pub enum CallArg {
+ /// A positional argument: `12`.
+ Pos(Expr),
+ /// A named argument: `draw: false`.
+ Named(Named),
+ /// A spreaded argument: `..things`.
+ Spread(Expr),
+}
+
+impl TypedNode for CallArg {
+ fn from_red(node: RedRef) -> Option<Self> {
+ match node.kind() {
+ NodeKind::Named => node.cast().map(CallArg::Named),
+ NodeKind::Spread => node.cast_first_child().map(CallArg::Spread),
+ _ => node.cast().map(CallArg::Pos),
+ }
+ }
+
+ fn as_red(&self) -> RedRef<'_> {
+ match self {
+ Self::Pos(v) => v.as_red(),
+ Self::Named(v) => v.as_red(),
+ Self::Spread(v) => v.as_red(),
+ }
+ }
+}
+
+impl CallArg {
+ /// The name of this argument.
+ pub fn span(&self) -> Span {
+ match self {
+ Self::Pos(expr) => expr.span(),
+ Self::Named(named) => named.span(),
+ Self::Spread(expr) => expr.span(),
+ }
+ }
+}
+
+node! {
+ /// A closure expression: `(x, y) => z`.
+ ClosureExpr: Closure
+}
+
+impl ClosureExpr {
+ /// The name of the closure.
+ ///
+ /// This only exists if you use the function syntax sugar: `let f(x) = y`.
+ pub fn name(&self) -> Option<Ident> {
+ self.0.cast_first_child()
+ }
+
+ /// The parameter bindings.
+ pub fn params(&self) -> impl Iterator<Item = ClosureParam> + '_ {
+ self.0
+ .children()
+ .find(|x| x.kind() == &NodeKind::ClosureParams)
+ .expect("closure is missing parameter list")
+ .children()
+ .filter_map(RedRef::cast)
+ }
+
+ /// The body of the closure.
+ pub fn body(&self) -> Expr {
+ self.0.cast_last_child().expect("closure is missing body")
+ }
+}
+
+/// A parameter to a closure.
+#[derive(Debug, Clone, PartialEq)]
+pub enum ClosureParam {
+ /// A positional parameter: `x`.
+ Pos(Ident),
+ /// A named parameter with a default value: `draw: false`.
+ Named(Named),
+ /// A parameter sink: `..args`.
+ Sink(Ident),
+}
+
+impl TypedNode for ClosureParam {
+ fn from_red(node: RedRef) -> Option<Self> {
+ match node.kind() {
+ NodeKind::Ident(_) => node.cast().map(ClosureParam::Pos),
+ NodeKind::Named => node.cast().map(ClosureParam::Named),
+ NodeKind::Spread => node.cast_first_child().map(ClosureParam::Sink),
+ _ => None,
+ }
+ }
+
+ fn as_red(&self) -> RedRef<'_> {
+ match self {
+ Self::Pos(v) => v.as_red(),
+ Self::Named(v) => v.as_red(),
+ Self::Sink(v) => v.as_red(),
+ }
+ }
+}
+
+node! {
+ /// A with expression: `f with (x, y: 1)`.
+ WithExpr
+}
+
+impl WithExpr {
+ /// The function to apply the arguments to.
+ pub fn callee(&self) -> Expr {
+ self.0.cast_first_child().expect("with expression is missing callee")
+ }
+
+ /// The arguments to apply to the function.
+ pub fn args(&self) -> CallArgs {
+ self.0
+ .cast_first_child()
+ .expect("with expression is missing argument list")
+ }
+}
+
+node! {
+ /// A let expression: `let x = 1`.
+ LetExpr
+}
+
+impl LetExpr {
+ /// The binding to assign to.
+ pub fn binding(&self) -> Ident {
+ match self.0.cast_first_child() {
+ Some(Expr::Ident(binding)) => binding,
+ Some(Expr::With(with)) => match with.callee() {
+ Expr::Ident(binding) => binding,
+ _ => panic!("let .. with callee must be identifier"),
+ },
+ Some(Expr::Closure(closure)) => {
+ closure.name().expect("let-bound closure is missing name")
+ }
+ _ => panic!("let expression is missing binding"),
+ }
+ }
+
+ /// The expression the binding is initialized with.
+ pub fn init(&self) -> Option<Expr> {
+ if self.0.cast_first_child::<Ident>().is_some() {
+ self.0.children().filter_map(RedRef::cast).nth(1)
+ } else {
+ // This is a let .. with expression.
+ self.0.cast_first_child()
+ }
+ }
+}
+
+node! {
+ /// An import expression: `import a, b, c from "utils.typ"`.
+ ImportExpr
+}
+
+impl ImportExpr {
+ /// The items to be imported.
+ pub fn imports(&self) -> Imports {
+ self.0
+ .children()
+ .find_map(|node| match node.kind() {
+ NodeKind::Star => Some(Imports::Wildcard),
+ NodeKind::ImportItems => {
+ let items = node.children().filter_map(RedRef::cast).collect();
+ Some(Imports::Items(items))
+ }
+ _ => None,
+ })
+ .expect("import is missing items")
+ }
+
+ /// The location of the importable file.
+ pub fn path(&self) -> Expr {
+ self.0.cast_last_child().expect("import is missing path")
+ }
+}
+
+/// The items that ought to be imported from a file.
+#[derive(Debug, Clone, PartialEq)]
+pub enum Imports {
+ /// All items in the scope of the file should be imported.
+ Wildcard,
+ /// The specified items from the file should be imported.
+ Items(Vec<Ident>),
+}
+
+node! {
+ /// An include expression: `include "chapter1.typ"`.
+ IncludeExpr
+}
+
+impl IncludeExpr {
+ /// The location of the file to be included.
+ pub fn path(&self) -> Expr {
+ self.0.cast_last_child().expect("include is missing path")
+ }
+}
+
+node! {
+ /// An if-else expression: `if x { y } else { z }`.
+ IfExpr
+}
+
+impl IfExpr {
+ /// The condition which selects the body to evaluate.
+ pub fn condition(&self) -> Expr {
+ self.0.cast_first_child().expect("if expression is missing condition")
+ }
+
+ /// The expression to evaluate if the condition is true.
+ pub fn if_body(&self) -> Expr {
+ self.0
+ .children()
+ .filter_map(RedRef::cast)
+ .nth(1)
+ .expect("if expression is missing body")
+ }
+
+ /// The expression to evaluate if the condition is false.
+ pub fn else_body(&self) -> Option<Expr> {
+ self.0.children().filter_map(RedRef::cast).nth(2)
+ }
+}
+
+node! {
+ /// A while loop expression: `while x { y }`.
+ WhileExpr
+}
+
+impl WhileExpr {
+ /// The condition which selects whether to evaluate the body.
+ pub fn condition(&self) -> Expr {
+ self.0.cast_first_child().expect("while loop is missing condition")
+ }
+
+ /// The expression to evaluate while the condition is true.
+ pub fn body(&self) -> Expr {
+ self.0.cast_last_child().expect("while loop is missing body")
+ }
+}
+
+node! {
+ /// A for loop expression: `for x in y { z }`.
+ ForExpr
+}
+
+impl ForExpr {
+ /// The pattern to assign to.
+ pub fn pattern(&self) -> ForPattern {
+ self.0.cast_first_child().expect("for loop is missing pattern")
+ }
+
+ /// The expression to iterate over.
+ pub fn iter(&self) -> Expr {
+ self.0.cast_first_child().expect("for loop is missing iterable")
+ }
+
+ /// The expression to evaluate for each iteration.
+ pub fn body(&self) -> Expr {
+ self.0.cast_last_child().expect("for loop is missing body")
+ }
+}
+
+node! {
+ /// A for-in loop expression: `for x in y { z }`.
+ ForPattern
+}
+
+impl ForPattern {
+ /// The key part of the pattern: index for arrays, name for dictionaries.
+ pub fn key(&self) -> Option<Ident> {
+ let mut children = self.0.children().filter_map(RedRef::cast);
+ let key = children.next();
+ if children.next().is_some() { key } else { None }
+ }
+
+ /// The value part of the pattern.
+ pub fn value(&self) -> Ident {
+ self.0.cast_last_child().expect("for loop pattern is missing value")
+ }
+}
+
+node! {
+ /// An identifier.
+ Ident: NodeKind::Ident(_)
+}
+
+impl Ident {
+ /// Take out the contained [`EcoString`].
+ pub fn take(self) -> EcoString {
+ match self.0.green {
+ Green::Token(GreenData { kind: NodeKind::Ident(id), .. }) => id,
+ _ => panic!("identifier is of wrong kind"),
+ }
+ }
+}
+
+impl Deref for Ident {
+ type Target = str;
+
+ fn deref(&self) -> &Self::Target {
+ match &self.0.green {
+ Green::Token(GreenData { kind: NodeKind::Ident(id), .. }) => id,
+ _ => panic!("identifier is of wrong kind"),
+ }
+ }
+}
diff --git a/src/syntax/expr.rs b/src/syntax/expr.rs
deleted file mode 100644
index 904515ba..00000000
--- a/src/syntax/expr.rs
+++ /dev/null
@@ -1,584 +0,0 @@
-use std::rc::Rc;
-
-use super::{Ident, Markup, Span, Token};
-use crate::geom::{AngularUnit, LengthUnit};
-use crate::util::EcoString;
-
-/// An expression.
-#[derive(Debug, Clone, PartialEq)]
-pub enum Expr {
- /// An identifier: `left`.
- Ident(Box<Ident>),
- /// A literal: `1`, `true`, ...
- Lit(Box<Lit>),
- /// An array expression: `(1, "hi", 12cm)`.
- Array(Box<ArrayExpr>),
- /// A dictionary expression: `(thickness: 3pt, pattern: dashed)`.
- Dict(Box<DictExpr>),
- /// A template expression: `[*Hi* there!]`.
- Template(Box<TemplateExpr>),
- /// A grouped expression: `(1 + 2)`.
- Group(Box<GroupExpr>),
- /// A block expression: `{ let x = 1; x + 2 }`.
- Block(Box<BlockExpr>),
- /// A unary operation: `-x`.
- Unary(Box<UnaryExpr>),
- /// A binary operation: `a + b`.
- Binary(Box<BinaryExpr>),
- /// An invocation of a function: `f(x, y)`.
- Call(Box<CallExpr>),
- /// A closure expression: `(x, y) => z`.
- Closure(Box<ClosureExpr>),
- /// A with expression: `f with (x, y: 1)`.
- With(Box<WithExpr>),
- /// A let expression: `let x = 1`.
- Let(Box<LetExpr>),
- /// An if-else expression: `if x { y } else { z }`.
- If(Box<IfExpr>),
- /// A while loop expression: `while x { y }`.
- While(Box<WhileExpr>),
- /// A for loop expression: `for x in y { z }`.
- For(Box<ForExpr>),
- /// An import expression: `import a, b, c from "utils.typ"`.
- Import(Box<ImportExpr>),
- /// An include expression: `include "chapter1.typ"`.
- Include(Box<IncludeExpr>),
-}
-
-impl Expr {
- /// The source code location.
- pub fn span(&self) -> Span {
- match self {
- Self::Ident(v) => v.span,
- Self::Lit(v) => v.span(),
- Self::Array(v) => v.span,
- Self::Dict(v) => v.span,
- Self::Template(v) => v.span,
- Self::Group(v) => v.span,
- Self::Block(v) => v.span,
- Self::Unary(v) => v.span,
- Self::Binary(v) => v.span,
- Self::Call(v) => v.span,
- Self::Closure(v) => v.span,
- Self::With(v) => v.span,
- Self::Let(v) => v.span,
- Self::If(v) => v.span,
- Self::While(v) => v.span,
- Self::For(v) => v.span,
- Self::Import(v) => v.span,
- Self::Include(v) => v.span,
- }
- }
-
- /// Whether the expression can be shortened in markup with a hashtag.
- pub fn has_short_form(&self) -> bool {
- matches!(self,
- Self::Ident(_)
- | Self::Call(_)
- | Self::Let(_)
- | Self::If(_)
- | Self::While(_)
- | Self::For(_)
- | Self::Import(_)
- | Self::Include(_)
- )
- }
-}
-
-/// A literal: `1`, `true`, ...
-#[derive(Debug, Clone, PartialEq)]
-pub enum Lit {
- /// The none literal: `none`.
- None(Span),
- /// The auto literal: `auto`.
- Auto(Span),
- /// A boolean literal: `true`, `false`.
- Bool(Span, bool),
- /// An integer literal: `120`.
- Int(Span, i64),
- /// A floating-point literal: `1.2`, `10e-4`.
- Float(Span, f64),
- /// A length literal: `12pt`, `3cm`.
- Length(Span, f64, LengthUnit),
- /// An angle literal: `1.5rad`, `90deg`.
- Angle(Span, f64, AngularUnit),
- /// A percent literal: `50%`.
- ///
- /// _Note_: `50%` is stored as `50.0` here, but as `0.5` in the
- /// corresponding [value](crate::geom::Relative).
- Percent(Span, f64),
- /// A fraction unit literal: `1fr`.
- Fractional(Span, f64),
- /// A string literal: `"hello!"`.
- Str(Span, EcoString),
-}
-
-impl Lit {
- /// The source code location.
- pub fn span(&self) -> Span {
- match *self {
- Self::None(span) => span,
- Self::Auto(span) => span,
- Self::Bool(span, _) => span,
- Self::Int(span, _) => span,
- Self::Float(span, _) => span,
- Self::Length(span, _, _) => span,
- Self::Angle(span, _, _) => span,
- Self::Percent(span, _) => span,
- Self::Fractional(span, _) => span,
- Self::Str(span, _) => span,
- }
- }
-}
-
-/// An array expression: `(1, "hi", 12cm)`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct ArrayExpr {
- /// The source code location.
- pub span: Span,
- /// The entries of the array.
- pub items: Vec<Expr>,
-}
-
-/// A dictionary expression: `(thickness: 3pt, pattern: dashed)`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct DictExpr {
- /// The source code location.
- pub span: Span,
- /// The named dictionary entries.
- pub items: Vec<Named>,
-}
-
-/// A pair of a name and an expression: `pattern: dashed`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct Named {
- /// The name: `pattern`.
- pub name: Ident,
- /// The right-hand side of the pair: `dashed`.
- pub expr: Expr,
-}
-
-impl Named {
- /// The source code location.
- pub fn span(&self) -> Span {
- self.name.span.join(self.expr.span())
- }
-}
-
-/// A template expression: `[*Hi* there!]`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct TemplateExpr {
- /// The source code location.
- pub span: Span,
- /// The contents of the template.
- pub body: Markup,
-}
-
-/// A grouped expression: `(1 + 2)`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct GroupExpr {
- /// The source code location.
- pub span: Span,
- /// The wrapped expression.
- pub expr: Expr,
-}
-
-/// A block expression: `{ let x = 1; x + 2 }`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct BlockExpr {
- /// The source code location.
- pub span: Span,
- /// The list of expressions contained in the block.
- pub exprs: Vec<Expr>,
-}
-
-/// A unary operation: `-x`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct UnaryExpr {
- /// The source code location.
- pub span: Span,
- /// The operator: `-`.
- pub op: UnOp,
- /// The expression to operator on: `x`.
- pub expr: Expr,
-}
-
-/// A unary operator.
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
-pub enum UnOp {
- /// The plus operator: `+`.
- Pos,
- /// The negation operator: `-`.
- Neg,
- /// The boolean `not`.
- Not,
-}
-
-impl UnOp {
- /// Try to convert the token into a unary operation.
- pub fn from_token(token: Token) -> Option<Self> {
- Some(match token {
- Token::Plus => Self::Pos,
- Token::Hyph => Self::Neg,
- Token::Not => Self::Not,
- _ => return None,
- })
- }
-
- /// The precedence of this operator.
- pub fn precedence(self) -> usize {
- match self {
- Self::Pos | Self::Neg => 8,
- Self::Not => 3,
- }
- }
-
- /// The string representation of this operation.
- pub fn as_str(self) -> &'static str {
- match self {
- Self::Pos => "+",
- Self::Neg => "-",
- Self::Not => "not",
- }
- }
-}
-
-/// A binary operation: `a + b`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct BinaryExpr {
- /// The source code location.
- pub span: Span,
- /// The left-hand side of the operation: `a`.
- pub lhs: Expr,
- /// The operator: `+`.
- pub op: BinOp,
- /// The right-hand side of the operation: `b`.
- pub rhs: Expr,
-}
-
-/// A binary operator.
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
-pub enum BinOp {
- /// The addition operator: `+`.
- Add,
- /// The subtraction operator: `-`.
- Sub,
- /// The multiplication operator: `*`.
- Mul,
- /// The division operator: `/`.
- Div,
- /// The short-circuiting boolean `and`.
- And,
- /// The short-circuiting boolean `or`.
- Or,
- /// The equality operator: `==`.
- Eq,
- /// The inequality operator: `!=`.
- Neq,
- /// The less-than operator: `<`.
- Lt,
- /// The less-than or equal operator: `<=`.
- Leq,
- /// The greater-than operator: `>`.
- Gt,
- /// The greater-than or equal operator: `>=`.
- Geq,
- /// The assignment operator: `=`.
- Assign,
- /// The add-assign operator: `+=`.
- AddAssign,
- /// The subtract-assign oeprator: `-=`.
- SubAssign,
- /// The multiply-assign operator: `*=`.
- MulAssign,
- /// The divide-assign operator: `/=`.
- DivAssign,
-}
-
-impl BinOp {
- /// Try to convert the token into a binary operation.
- pub fn from_token(token: Token) -> Option<Self> {
- Some(match token {
- Token::Plus => Self::Add,
- Token::Hyph => Self::Sub,
- Token::Star => Self::Mul,
- Token::Slash => Self::Div,
- Token::And => Self::And,
- Token::Or => Self::Or,
- Token::EqEq => Self::Eq,
- Token::ExclEq => Self::Neq,
- Token::Lt => Self::Lt,
- Token::LtEq => Self::Leq,
- Token::Gt => Self::Gt,
- Token::GtEq => Self::Geq,
- Token::Eq => Self::Assign,
- Token::PlusEq => Self::AddAssign,
- Token::HyphEq => Self::SubAssign,
- Token::StarEq => Self::MulAssign,
- Token::SlashEq => Self::DivAssign,
- _ => return None,
- })
- }
-
- /// The precedence of this operator.
- pub fn precedence(self) -> usize {
- match self {
- Self::Mul | Self::Div => 6,
- Self::Add | Self::Sub => 5,
- Self::Eq | Self::Neq | Self::Lt | Self::Leq | Self::Gt | Self::Geq => 4,
- Self::And => 3,
- Self::Or => 2,
- Self::Assign
- | Self::AddAssign
- | Self::SubAssign
- | Self::MulAssign
- | Self::DivAssign => 1,
- }
- }
-
- /// The associativity of this operator.
- pub fn associativity(self) -> Associativity {
- match self {
- Self::Add
- | Self::Sub
- | Self::Mul
- | Self::Div
- | Self::And
- | Self::Or
- | Self::Eq
- | Self::Neq
- | Self::Lt
- | Self::Leq
- | Self::Gt
- | Self::Geq => Associativity::Left,
- Self::Assign
- | Self::AddAssign
- | Self::SubAssign
- | Self::MulAssign
- | Self::DivAssign => Associativity::Right,
- }
- }
-
- /// The string representation of this operation.
- pub fn as_str(self) -> &'static str {
- match self {
- Self::Add => "+",
- Self::Sub => "-",
- Self::Mul => "*",
- Self::Div => "/",
- Self::And => "and",
- Self::Or => "or",
- Self::Eq => "==",
- Self::Neq => "!=",
- Self::Lt => "<",
- Self::Leq => "<=",
- Self::Gt => ">",
- Self::Geq => ">=",
- Self::Assign => "=",
- Self::AddAssign => "+=",
- Self::SubAssign => "-=",
- Self::MulAssign => "*=",
- Self::DivAssign => "/=",
- }
- }
-}
-
-/// The associativity of a binary operator.
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
-pub enum Associativity {
- /// Left-associative: `a + b + c` is equivalent to `(a + b) + c`.
- Left,
- /// Right-associative: `a = b = c` is equivalent to `a = (b = c)`.
- Right,
-}
-
-/// An invocation of a function: `foo(...)`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct CallExpr {
- /// The source code location.
- pub span: Span,
- /// The function to call.
- pub callee: Expr,
- /// The arguments to the function.
- pub args: CallArgs,
-}
-
-/// The arguments to a function: `12, draw: false`.
-///
-/// In case of a bracketed invocation with a body, the body is _not_
-/// included in the span for the sake of clearer error messages.
-#[derive(Debug, Clone, PartialEq)]
-pub struct CallArgs {
- /// The source code location.
- pub span: Span,
- /// The positional and named arguments.
- pub items: Vec<CallArg>,
-}
-
-/// An argument to a function call.
-#[derive(Debug, Clone, PartialEq)]
-pub enum CallArg {
- /// A positional argument: `12`.
- Pos(Expr),
- /// A named argument: `draw: false`.
- Named(Named),
- /// A spreaded argument: `..things`.
- Spread(Expr),
-}
-
-impl CallArg {
- /// The source code location.
- pub fn span(&self) -> Span {
- match self {
- Self::Pos(expr) => expr.span(),
- Self::Named(named) => named.span(),
- Self::Spread(expr) => expr.span(),
- }
- }
-}
-
-/// A closure expression: `(x, y) => z`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct ClosureExpr {
- /// The source code location.
- pub span: Span,
- /// The name of the closure.
- ///
- /// This only exists if you use the function syntax sugar: `let f(x) = y`.
- pub name: Option<Ident>,
- /// The parameter bindings.
- pub params: Vec<ClosureParam>,
- /// The body of the closure.
- pub body: Rc<Expr>,
-}
-
-/// An parameter to a closure.
-#[derive(Debug, Clone, PartialEq)]
-pub enum ClosureParam {
- /// A positional parameter: `x`.
- Pos(Ident),
- /// A named parameter with a default value: `draw: false`.
- Named(Named),
- /// A parameter sink: `..args`.
- Sink(Ident),
-}
-
-impl ClosureParam {
- /// The source code location.
- pub fn span(&self) -> Span {
- match self {
- Self::Pos(ident) => ident.span,
- Self::Named(named) => named.span(),
- Self::Sink(ident) => ident.span,
- }
- }
-}
-
-/// A with expression: `f with (x, y: 1)`.
-///
-/// Applies arguments to a function.
-#[derive(Debug, Clone, PartialEq)]
-pub struct WithExpr {
- /// The source code location.
- pub span: Span,
- /// The function to apply the arguments to.
- pub callee: Expr,
- /// The arguments to apply to the function.
- pub args: CallArgs,
-}
-
-/// A let expression: `let x = 1`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct LetExpr {
- /// The source code location.
- pub span: Span,
- /// The binding to assign to.
- pub binding: Ident,
- /// The expression the binding is initialized with.
- pub init: Option<Expr>,
-}
-
-/// An import expression: `import a, b, c from "utils.typ"`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct ImportExpr {
- /// The source code location.
- pub span: Span,
- /// The items to be imported.
- pub imports: Imports,
- /// The location of the importable file.
- pub path: Expr,
-}
-
-/// The items that ought to be imported from a file.
-#[derive(Debug, Clone, PartialEq)]
-pub enum Imports {
- /// All items in the scope of the file should be imported.
- Wildcard,
- /// The specified identifiers from the file should be imported.
- Idents(Vec<Ident>),
-}
-
-/// An include expression: `include "chapter1.typ"`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct IncludeExpr {
- /// The source code location.
- pub span: Span,
- /// The location of the file to be included.
- pub path: Expr,
-}
-
-/// An if-else expression: `if x { y } else { z }`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct IfExpr {
- /// The source code location.
- pub span: Span,
- /// The condition which selects the body to evaluate.
- pub condition: Expr,
- /// The expression to evaluate if the condition is true.
- pub if_body: Expr,
- /// The expression to evaluate if the condition is false.
- pub else_body: Option<Expr>,
-}
-
-/// A while loop expression: `while x { y }`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct WhileExpr {
- /// The source code location.
- pub span: Span,
- /// The condition which selects whether to evaluate the body.
- pub condition: Expr,
- /// The expression to evaluate while the condition is true.
- pub body: Expr,
-}
-
-/// A for loop expression: `for x in y { z }`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct ForExpr {
- /// The source code location.
- pub span: Span,
- /// The pattern to assign to.
- pub pattern: ForPattern,
- /// The expression to iterate over.
- pub iter: Expr,
- /// The expression to evaluate for each iteration.
- pub body: Expr,
-}
-
-/// A pattern in a for loop.
-#[derive(Debug, Clone, PartialEq)]
-pub enum ForPattern {
- /// A value pattern: `for v in array`.
- Value(Ident),
- /// A key-value pattern: `for k, v in dict`.
- KeyValue(Ident, Ident),
-}
-
-impl ForPattern {
- /// The source code location.
- pub fn span(&self) -> Span {
- match self {
- Self::Value(v) => v.span,
- Self::KeyValue(k, v) => k.span.join(v.span),
- }
- }
-}
diff --git a/src/syntax/ident.rs b/src/syntax/ident.rs
deleted file mode 100644
index 398e2ff9..00000000
--- a/src/syntax/ident.rs
+++ /dev/null
@@ -1,85 +0,0 @@
-use std::borrow::Borrow;
-use std::ops::Deref;
-
-use unicode_xid::UnicodeXID;
-
-use super::Span;
-use crate::util::EcoString;
-
-/// An unicode identifier with a few extra permissible characters.
-///
-/// In addition to what is specified in the [Unicode Standard][uax31], we allow:
-/// - `_` as a starting character,
-/// - `_` and `-` as continuing characters.
-///
-/// [uax31]: http://www.unicode.org/reports/tr31/
-#[derive(Debug, Clone, PartialEq)]
-pub struct Ident {
- /// The source code location.
- pub span: Span,
- /// The identifier string.
- pub string: EcoString,
-}
-
-impl Ident {
- /// Create a new identifier from a string checking that it is a valid.
- pub fn new(
- string: impl AsRef<str> + Into<EcoString>,
- span: impl Into<Span>,
- ) -> Option<Self> {
- if is_ident(string.as_ref()) {
- Some(Self { span: span.into(), string: string.into() })
- } else {
- None
- }
- }
-
- /// Return a reference to the underlying string.
- pub fn as_str(&self) -> &str {
- self
- }
-}
-
-impl Deref for Ident {
- type Target = str;
-
- fn deref(&self) -> &Self::Target {
- self.string.as_str()
- }
-}
-
-impl AsRef<str> for Ident {
- fn as_ref(&self) -> &str {
- self
- }
-}
-
-impl Borrow<str> for Ident {
- fn borrow(&self) -> &str {
- self
- }
-}
-
-impl From<&Ident> for EcoString {
- fn from(ident: &Ident) -> Self {
- ident.string.clone()
- }
-}
-
-/// Whether a string is a valid identifier.
-pub fn is_ident(string: &str) -> bool {
- let mut chars = string.chars();
- chars
- .next()
- .map_or(false, |c| is_id_start(c) && chars.all(is_id_continue))
-}
-
-/// Whether a character can start an identifier.
-pub fn is_id_start(c: char) -> bool {
- c.is_xid_start() || c == '_'
-}
-
-/// Whether a character can continue an identifier.
-pub fn is_id_continue(c: char) -> bool {
- c.is_xid_continue() || c == '_' || c == '-'
-}
diff --git a/src/syntax/markup.rs b/src/syntax/markup.rs
deleted file mode 100644
index 09a37116..00000000
--- a/src/syntax/markup.rs
+++ /dev/null
@@ -1,78 +0,0 @@
-use super::{Expr, Ident, Span};
-use crate::util::EcoString;
-
-/// The syntactical root capable of representing a full parsed document.
-pub type Markup = Vec<MarkupNode>;
-
-/// A single piece of markup.
-#[derive(Debug, Clone, PartialEq)]
-pub enum MarkupNode {
- /// Whitespace containing less than two newlines.
- Space,
- /// A forced line break: `\`.
- Linebreak(Span),
- /// A paragraph break: Two or more newlines.
- Parbreak(Span),
- /// Strong text was enabled / disabled: `*`.
- Strong(Span),
- /// Emphasized text was enabled / disabled: `_`.
- Emph(Span),
- /// Plain text.
- Text(EcoString),
- /// A raw block with optional syntax highlighting: `` `...` ``.
- Raw(Box<RawNode>),
- /// A section heading: `= Introduction`.
- Heading(Box<HeadingNode>),
- /// An item in an unordered list: `- ...`.
- List(Box<ListNode>),
- /// An item in an enumeration (ordered list): `1. ...`.
- Enum(Box<EnumNode>),
- /// An expression.
- Expr(Expr),
-}
-
-/// A raw block with optional syntax highlighting: `` `...` ``.
-#[derive(Debug, Clone, PartialEq)]
-pub struct RawNode {
- /// The source code location.
- pub span: Span,
- /// An optional identifier specifying the language to syntax-highlight in.
- pub lang: Option<Ident>,
- /// The raw text, determined as the raw string between the backticks trimmed
- /// according to the above rules.
- pub text: EcoString,
- /// Whether the element is block-level, that is, it has 3+ backticks
- /// and contains at least one newline.
- pub block: bool,
-}
-
-/// A section heading: `= Introduction`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct HeadingNode {
- /// The source code location.
- pub span: Span,
- /// The section depth (numer of equals signs).
- pub level: usize,
- /// The contents of the heading.
- pub body: Markup,
-}
-
-/// An item in an unordered list: `- ...`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct ListNode {
- /// The source code location.
- pub span: Span,
- /// The contents of the list item.
- pub body: Markup,
-}
-
-/// An item in an enumeration (ordered list): `1. ...`.
-#[derive(Debug, Clone, PartialEq)]
-pub struct EnumNode {
- /// The source code location.
- pub span: Span,
- /// The number, if any.
- pub number: Option<usize>,
- /// The contents of the list item.
- pub body: Markup,
-}
diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs
index 8dbb108d..ca6ed243 100644
--- a/src/syntax/mod.rs
+++ b/src/syntax/mod.rs
@@ -1,16 +1,747 @@
//! Syntax types.
-mod expr;
-mod ident;
-mod markup;
+pub mod ast;
mod pretty;
mod span;
-mod token;
-pub mod visit;
-pub use expr::*;
-pub use ident::*;
-pub use markup::*;
+use std::fmt::{self, Debug, Display, Formatter};
+use std::rc::Rc;
+
pub use pretty::*;
pub use span::*;
-pub use token::*;
+
+use self::ast::{MathNode, RawNode, TypedNode};
+use crate::diag::Error;
+use crate::geom::{AngularUnit, LengthUnit};
+use crate::source::SourceId;
+use crate::util::EcoString;
+
+/// An inner of leaf node in the untyped green tree.
+#[derive(Clone, PartialEq)]
+pub enum Green {
+ /// A reference-counted inner node.
+ Node(Rc<GreenNode>),
+ /// A terminal, owned token.
+ Token(GreenData),
+}
+
+impl Green {
+ /// Returns the metadata of the node.
+ fn data(&self) -> &GreenData {
+ match self {
+ Green::Node(n) => &n.data,
+ Green::Token(t) => &t,
+ }
+ }
+
+ /// The type of the node.
+ pub fn kind(&self) -> &NodeKind {
+ self.data().kind()
+ }
+
+ /// The length of the node.
+ pub fn len(&self) -> usize {
+ self.data().len()
+ }
+
+ /// Whether the node or its children contain an error.
+ pub fn erroneous(&self) -> bool {
+ match self {
+ Self::Node(node) => node.erroneous,
+ Self::Token(data) => data.kind.is_error(),
+ }
+ }
+
+ /// The node's children.
+ pub fn children(&self) -> &[Green] {
+ match self {
+ Green::Node(n) => &n.children(),
+ Green::Token(_) => &[],
+ }
+ }
+
+ /// Change the type of the node.
+ pub fn convert(&mut self, kind: NodeKind) {
+ match self {
+ Self::Node(node) => {
+ let node = Rc::make_mut(node);
+ node.erroneous |= kind.is_error();
+ node.data.kind = kind;
+ }
+ Self::Token(data) => data.kind = kind,
+ }
+ }
+}
+
+impl Default for Green {
+ fn default() -> Self {
+ Self::Token(GreenData::new(NodeKind::None, 0))
+ }
+}
+
+impl Debug for Green {
+ fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+ write!(f, "{:?}: {}", self.kind(), self.len())?;
+ if let Self::Node(n) = self {
+ if !n.children.is_empty() {
+ f.write_str(" ")?;
+ f.debug_list().entries(&n.children).finish()?;
+ }
+ }
+ Ok(())
+ }
+}
+
+/// An inner node in the untyped green tree.
+#[derive(Debug, Clone, PartialEq)]
+pub struct GreenNode {
+ /// Node metadata.
+ data: GreenData,
+ /// This node's children, losslessly make up this node.
+ children: Vec<Green>,
+ /// Whether this node or any of its children are erroneous.
+ erroneous: bool,
+}
+
+impl GreenNode {
+ /// Creates a new node with the given kind and a single child.
+ pub fn with_child(kind: NodeKind, child: impl Into<Green>) -> Self {
+ Self::with_children(kind, vec![child.into()])
+ }
+
+ /// Creates a new node with the given kind and children.
+ pub fn with_children(kind: NodeKind, children: Vec<Green>) -> Self {
+ let mut erroneous = kind.is_error();
+ let len = children
+ .iter()
+ .inspect(|c| erroneous |= c.erroneous())
+ .map(Green::len)
+ .sum();
+
+ Self {
+ data: GreenData::new(kind, len),
+ children,
+ erroneous,
+ }
+ }
+
+ /// The node's children.
+ pub fn children(&self) -> &[Green] {
+ &self.children
+ }
+}
+
+impl From<GreenNode> for Green {
+ fn from(node: GreenNode) -> Self {
+ Rc::new(node).into()
+ }
+}
+
+impl From<Rc<GreenNode>> for Green {
+ fn from(node: Rc<GreenNode>) -> Self {
+ Self::Node(node)
+ }
+}
+
+/// Data shared between inner and leaf nodes.
+#[derive(Debug, Clone, PartialEq)]
+pub struct GreenData {
+ /// What kind of node this is (each kind would have its own struct in a
+ /// strongly typed AST).
+ kind: NodeKind,
+ /// The byte length of the node in the source.
+ len: usize,
+}
+
+impl GreenData {
+ /// Create new node metadata.
+ pub fn new(kind: NodeKind, len: usize) -> Self {
+ Self { len, kind }
+ }
+
+ /// The type of the node.
+ pub fn kind(&self) -> &NodeKind {
+ &self.kind
+ }
+
+ /// The length of the node.
+ pub fn len(&self) -> usize {
+ self.len
+ }
+}
+
+impl From<GreenData> for Green {
+ fn from(token: GreenData) -> Self {
+ Self::Token(token)
+ }
+}
+
+/// A owned wrapper for a green node with span information.
+///
+/// Owned variant of [`RedRef`]. Can be [cast](Self::cast) to an AST node.
+#[derive(Clone, PartialEq)]
+pub struct RedNode {
+ id: SourceId,
+ offset: usize,
+ green: Green,
+}
+
+impl RedNode {
+ /// Create a new red node from a root [`GreenNode`].
+ pub fn from_root(root: Rc<GreenNode>, id: SourceId) -> Self {
+ Self { id, offset: 0, green: root.into() }
+ }
+
+ /// Convert to a borrowed representation.
+ pub fn as_ref(&self) -> RedRef<'_> {
+ RedRef {
+ id: self.id,
+ offset: self.offset,
+ green: &self.green,
+ }
+ }
+
+ /// The type of the node.
+ pub fn kind(&self) -> &NodeKind {
+ self.as_ref().kind()
+ }
+
+ /// The length of the node.
+ pub fn len(&self) -> usize {
+ self.as_ref().len()
+ }
+
+ /// The span of the node.
+ pub fn span(&self) -> Span {
+ self.as_ref().span()
+ }
+
+ /// The error messages for this node and its descendants.
+ pub fn errors(&self) -> Vec<Error> {
+ self.as_ref().errors()
+ }
+
+ /// Convert the node to a typed AST node.
+ pub fn cast<T>(self) -> Option<T>
+ where
+ T: TypedNode,
+ {
+ self.as_ref().cast()
+ }
+
+ /// The children of the node.
+ pub fn children(&self) -> Children<'_> {
+ self.as_ref().children()
+ }
+
+ /// Get the first child that can cast to some AST type.
+ pub fn cast_first_child<T: TypedNode>(&self) -> Option<T> {
+ self.as_ref().cast_first_child()
+ }
+
+ /// Get the last child that can cast to some AST type.
+ pub fn cast_last_child<T: TypedNode>(&self) -> Option<T> {
+ self.as_ref().cast_last_child()
+ }
+}
+
+impl Debug for RedNode {
+ fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+ self.as_ref().fmt(f)
+ }
+}
+
+/// A borrowed wrapper for a green node with span information.
+///
+/// Borrowed variant of [`RedNode`]. Can be [cast](Self::cast) to an AST node.
+#[derive(Copy, Clone, PartialEq)]
+pub struct RedRef<'a> {
+ id: SourceId,
+ offset: usize,
+ green: &'a Green,
+}
+
+impl<'a> RedRef<'a> {
+ /// Convert to an owned representation.
+ pub fn own(self) -> RedNode {
+ RedNode {
+ id: self.id,
+ offset: self.offset,
+ green: self.green.clone(),
+ }
+ }
+
+ /// The type of the node.
+ pub fn kind(self) -> &'a NodeKind {
+ self.green.kind()
+ }
+
+ /// The length of the node.
+ pub fn len(self) -> usize {
+ self.green.len()
+ }
+
+ /// The span of the node.
+ pub fn span(self) -> Span {
+ Span::new(self.id, self.offset, self.offset + self.green.len())
+ }
+
+ /// The error messages for this node and its descendants.
+ pub fn errors(self) -> Vec<Error> {
+ if !self.green.erroneous() {
+ return vec![];
+ }
+
+ match self.kind() {
+ NodeKind::Error(pos, msg) => {
+ let span = match pos {
+ ErrorPos::Start => self.span().at_start(),
+ ErrorPos::Full => self.span(),
+ ErrorPos::End => self.span().at_end(),
+ };
+
+ vec![Error::new(span, msg.to_string())]
+ }
+ _ => self
+ .children()
+ .filter(|red| red.green.erroneous())
+ .flat_map(|red| red.errors())
+ .collect(),
+ }
+ }
+
+ /// Convert the node to a typed AST node.
+ pub fn cast<T>(self) -> Option<T>
+ where
+ T: TypedNode,
+ {
+ T::from_red(self)
+ }
+
+ /// The node's children.
+ pub fn children(self) -> Children<'a> {
+ let children = match &self.green {
+ Green::Node(node) => node.children(),
+ Green::Token(_) => &[],
+ };
+
+ Children {
+ id: self.id,
+ iter: children.iter(),
+ front: self.offset,
+ back: self.offset + self.len(),
+ }
+ }
+
+ /// Get the first child that can cast to some AST type.
+ pub fn cast_first_child<T: TypedNode>(self) -> Option<T> {
+ self.children().find_map(RedRef::cast)
+ }
+
+ /// Get the last child that can cast to some AST type.
+ pub fn cast_last_child<T: TypedNode>(self) -> Option<T> {
+ self.children().rev().find_map(RedRef::cast)
+ }
+}
+
+impl Debug for RedRef<'_> {
+ fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+ write!(f, "{:?}: {:?}", self.kind(), self.span())?;
+ let mut children = self.children().peekable();
+ if children.peek().is_some() {
+ f.write_str(" ")?;
+ f.debug_list().entries(children.map(RedRef::own)).finish()?;
+ }
+ Ok(())
+ }
+}
+
+/// An iterator over the children of a red node.
+pub struct Children<'a> {
+ id: SourceId,
+ iter: std::slice::Iter<'a, Green>,
+ front: usize,
+ back: usize,
+}
+
+impl<'a> Iterator for Children<'a> {
+ type Item = RedRef<'a>;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ self.iter.next().map(|green| {
+ let offset = self.front;
+ self.front += green.len();
+ RedRef { id: self.id, offset, green }
+ })
+ }
+
+ fn size_hint(&self) -> (usize, Option<usize>) {
+ self.iter.size_hint()
+ }
+}
+
+impl DoubleEndedIterator for Children<'_> {
+ fn next_back(&mut self) -> Option<Self::Item> {
+ self.iter.next_back().map(|green| {
+ self.back -= green.len();
+ RedRef { id: self.id, offset: self.back, green }
+ })
+ }
+}
+
+impl ExactSizeIterator for Children<'_> {}
+
+/// All syntactical building blocks that can be part of a Typst document.
+///
+/// Can be emitted as a token by the tokenizer or as part of a green node by
+/// the parser.
+#[derive(Debug, Clone, PartialEq)]
+pub enum NodeKind {
+ /// A left square bracket: `[`.
+ LeftBracket,
+ /// A right square bracket: `]`.
+ RightBracket,
+ /// A left curly brace: `{`.
+ LeftBrace,
+ /// A right curly brace: `}`.
+ RightBrace,
+ /// A left round parenthesis: `(`.
+ LeftParen,
+ /// A right round parenthesis: `)`.
+ RightParen,
+ /// An asterisk: `*`.
+ Star,
+ /// A comma: `,`.
+ Comma,
+ /// A semicolon: `;`.
+ Semicolon,
+ /// A colon: `:`.
+ Colon,
+ /// A plus: `+`.
+ Plus,
+ /// A hyphen: `-`.
+ Minus,
+ /// A slash: `/`.
+ Slash,
+ /// A single equals sign: `=`.
+ Eq,
+ /// Two equals signs: `==`.
+ EqEq,
+ /// An exclamation mark followed by an equals sign: `!=`.
+ ExclEq,
+ /// A less-than sign: `<`.
+ Lt,
+ /// A less-than sign followed by an equals sign: `<=`.
+ LtEq,
+ /// A greater-than sign: `>`.
+ Gt,
+ /// A greater-than sign followed by an equals sign: `>=`.
+ GtEq,
+ /// A plus followed by an equals sign: `+=`.
+ PlusEq,
+ /// A hyphen followed by an equals sign: `-=`.
+ HyphEq,
+ /// An asterisk followed by an equals sign: `*=`.
+ StarEq,
+ /// A slash followed by an equals sign: `/=`.
+ SlashEq,
+ /// The `not` operator.
+ Not,
+ /// The `and` operator.
+ And,
+ /// The `or` operator.
+ Or,
+ /// The `with` operator.
+ With,
+ /// Two dots: `..`.
+ Dots,
+ /// An equals sign followed by a greater-than sign: `=>`.
+ Arrow,
+ /// The none literal: `none`.
+ None,
+ /// The auto literal: `auto`.
+ Auto,
+ /// The `let` keyword.
+ Let,
+ /// The `if` keyword.
+ If,
+ /// The `else` keyword.
+ Else,
+ /// The `for` keyword.
+ For,
+ /// The `in` keyword.
+ In,
+ /// The `while` keyword.
+ While,
+ /// The `break` keyword.
+ Break,
+ /// The `continue` keyword.
+ Continue,
+ /// The `return` keyword.
+ Return,
+ /// The `import` keyword.
+ Import,
+ /// The `include` keyword.
+ Include,
+ /// The `from` keyword.
+ From,
+ /// Template markup.
+ Markup,
+ /// One or more whitespace characters.
+ Space(usize),
+ /// A forced line break: `\`.
+ Linebreak,
+ /// A paragraph break: Two or more newlines.
+ Parbreak,
+ /// A consecutive non-markup string.
+ Text(EcoString),
+ /// A non-breaking space: `~`.
+ NonBreakingSpace,
+ /// An en-dash: `--`.
+ EnDash,
+ /// An em-dash: `---`.
+ EmDash,
+ /// A slash and the letter "u" followed by a hexadecimal unicode entity
+ /// enclosed in curly braces: `\u{1F5FA}`.
+ UnicodeEscape(char),
+ /// Strong text was enabled / disabled: `*`.
+ Strong,
+ /// Emphasized text was enabled / disabled: `_`.
+ Emph,
+ /// A section heading: `= Introduction`.
+ Heading,
+ /// An item in an enumeration (ordered list): `1. ...`.
+ Enum,
+ /// A numbering: `23.`.
+ ///
+ /// Can also exist without the number: `.`.
+ EnumNumbering(Option<usize>),
+ /// An item in an unordered list: `- ...`.
+ List,
+ /// An arbitrary number of backticks followed by inner contents, terminated
+ /// with the same number of backticks: `` `...` ``.
+ Raw(Rc<RawNode>),
+ /// Dollar signs surrounding inner contents.
+ Math(Rc<MathNode>),
+ /// An identifier: `center`.
+ Ident(EcoString),
+ /// A boolean: `true`, `false`.
+ Bool(bool),
+ /// An integer: `120`.
+ Int(i64),
+ /// A floating-point number: `1.2`, `10e-4`.
+ Float(f64),
+ /// A length: `12pt`, `3cm`.
+ Length(f64, LengthUnit),
+ /// An angle: `90deg`.
+ Angle(f64, AngularUnit),
+ /// A percentage: `50%`.
+ ///
+ /// _Note_: `50%` is stored as `50.0` here, as in the corresponding
+ /// [literal](ast::LitKind::Percent).
+ Percentage(f64),
+ /// A fraction unit: `3fr`.
+ Fraction(f64),
+ /// A quoted string: `"..."`.
+ Str(EcoString),
+ /// An array expression: `(1, "hi", 12cm)`.
+ Array,
+ /// A dictionary expression: `(thickness: 3pt, pattern: dashed)`.
+ Dict,
+ /// A named pair: `thickness: 3pt`.
+ Named,
+ /// A grouped expression: `(1 + 2)`.
+ Group,
+ /// A unary operation: `-x`.
+ Unary,
+ /// A binary operation: `a + b`.
+ Binary,
+ /// An invocation of a function: `f(x, y)`.
+ Call,
+ /// A function call's argument list: `(x, y)`.
+ CallArgs,
+ /// A closure expression: `(x, y) => z`.
+ Closure,
+ /// A closure's parameters: `(x, y)`.
+ ClosureParams,
+ /// A parameter sink: `..x`.
+ Spread,
+ /// A template expression: `[*Hi* there!]`.
+ Template,
+ /// A block expression: `{ let x = 1; x + 2 }`.
+ Block,
+ /// A for loop expression: `for x in y { ... }`.
+ ForExpr,
+ /// A while loop expression: `while x { ... }`.
+ WhileExpr,
+ /// An if expression: `if x { ... }`.
+ IfExpr,
+ /// A let expression: `let x = 1`.
+ LetExpr,
+ /// The `with` expression: `with (1)`.
+ WithExpr,
+ /// A for loop's destructuring pattern: `x` or `x, y`.
+ ForPattern,
+ /// The import expression: `import x from "foo.typ"`.
+ ImportExpr,
+ /// Items to import: `a, b, c`.
+ ImportItems,
+ /// The include expression: `include "foo.typ"`.
+ IncludeExpr,
+ /// Two slashes followed by inner contents, terminated with a newline:
+ /// `//<str>\n`.
+ LineComment,
+ /// A slash and a star followed by inner contents, terminated with a star
+ /// and a slash: `/*<str>*/`.
+ ///
+ /// The comment can contain nested block comments.
+ BlockComment,
+ /// Tokens that appear in the wrong place.
+ Error(ErrorPos, EcoString),
+ /// Unknown character sequences.
+ Unknown(EcoString),
+}
+
+/// Where in a node an error should be annotated.
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum ErrorPos {
+ /// At the start of the node.
+ Start,
+ /// Over the full width of the node.
+ Full,
+ /// At the end of the node.
+ End,
+}
+
+impl NodeKind {
+ /// Whether this is some kind of parenthesis.
+ pub fn is_paren(&self) -> bool {
+ matches!(self, Self::LeftParen | Self::RightParen)
+ }
+
+ /// Whether this is some kind of bracket.
+ pub fn is_bracket(&self) -> bool {
+ matches!(self, Self::LeftBracket | Self::RightBracket)
+ }
+
+ /// Whether this is some kind of brace.
+ pub fn is_brace(&self) -> bool {
+ matches!(self, Self::LeftBrace | Self::RightBrace)
+ }
+
+ /// Whether this is some kind of error.
+ pub fn is_error(&self) -> bool {
+ matches!(self, NodeKind::Error(_, _) | NodeKind::Unknown(_))
+ }
+
+ /// A human-readable name for the kind.
+ pub fn as_str(&self) -> &'static str {
+ match self {
+ Self::LeftBracket => "opening bracket",
+ Self::RightBracket => "closing bracket",
+ Self::LeftBrace => "opening brace",
+ Self::RightBrace => "closing brace",
+ Self::LeftParen => "opening paren",
+ Self::RightParen => "closing paren",
+ Self::Star => "star",
+ Self::Comma => "comma",
+ Self::Semicolon => "semicolon",
+ Self::Colon => "colon",
+ Self::Plus => "plus",
+ Self::Minus => "minus",
+ Self::Slash => "slash",
+ Self::Eq => "assignment operator",
+ Self::EqEq => "equality operator",
+ Self::ExclEq => "inequality operator",
+ Self::Lt => "less-than operator",
+ Self::LtEq => "less-than or equal operator",
+ Self::Gt => "greater-than operator",
+ Self::GtEq => "greater-than or equal operator",
+ Self::PlusEq => "add-assign operator",
+ Self::HyphEq => "subtract-assign operator",
+ Self::StarEq => "multiply-assign operator",
+ Self::SlashEq => "divide-assign operator",
+ Self::Not => "operator `not`",
+ Self::And => "operator `and`",
+ Self::Or => "operator `or`",
+ Self::With => "operator `with`",
+ Self::Dots => "dots",
+ Self::Arrow => "arrow",
+ Self::None => "`none`",
+ Self::Auto => "`auto`",
+ Self::Let => "keyword `let`",
+ Self::If => "keyword `if`",
+ Self::Else => "keyword `else`",
+ Self::For => "keyword `for`",
+ Self::In => "keyword `in`",
+ Self::While => "keyword `while`",
+ Self::Break => "keyword `break`",
+ Self::Continue => "keyword `continue`",
+ Self::Return => "keyword `return`",
+ Self::Import => "keyword `import`",
+ Self::Include => "keyword `include`",
+ Self::From => "keyword `from`",
+ Self::Markup => "markup",
+ Self::Space(_) => "space",
+ Self::Linebreak => "forced linebreak",
+ Self::Parbreak => "paragraph break",
+ Self::Text(_) => "text",
+ Self::NonBreakingSpace => "non-breaking space",
+ Self::EnDash => "en dash",
+ Self::EmDash => "em dash",
+ Self::UnicodeEscape(_) => "unicode escape sequence",
+ Self::Strong => "strong",
+ Self::Emph => "emphasis",
+ Self::Heading => "heading",
+ Self::Enum => "enumeration item",
+ Self::EnumNumbering(_) => "enumeration item numbering",
+ Self::List => "list item",
+ Self::Raw(_) => "raw block",
+ Self::Math(_) => "math formula",
+ Self::Ident(_) => "identifier",
+ Self::Bool(_) => "boolean",
+ Self::Int(_) => "integer",
+ Self::Float(_) => "float",
+ Self::Length(_, _) => "length",
+ Self::Angle(_, _) => "angle",
+ Self::Percentage(_) => "percentage",
+ Self::Fraction(_) => "`fr` value",
+ Self::Str(_) => "string",
+ Self::Array => "array",
+ Self::Dict => "dictionary",
+ Self::Named => "named argument",
+ Self::Group => "group",
+ Self::Unary => "unary expression",
+ Self::Binary => "binary expression",
+ Self::Call => "call",
+ Self::CallArgs => "call arguments",
+ Self::Closure => "closure",
+ Self::ClosureParams => "closure parameters",
+ Self::Spread => "parameter sink",
+ Self::Template => "template",
+ Self::Block => "block",
+ Self::ForExpr => "for-loop expression",
+ Self::WhileExpr => "while-loop expression",
+ Self::IfExpr => "`if` expression",
+ Self::LetExpr => "`let` expression",
+ Self::WithExpr => "`with` expression",
+ Self::ForPattern => "for-loop destructuring pattern",
+ Self::ImportExpr => "`import` expression",
+ Self::ImportItems => "import items",
+ Self::IncludeExpr => "`include` expression",
+ Self::LineComment => "line comment",
+ Self::BlockComment => "block comment",
+ Self::Error(_, _) => "parse error",
+ Self::Unknown(src) => match src.as_str() {
+ "*/" => "end of block comment",
+ _ => "invalid token",
+ },
+ }
+ }
+}
+
+impl Display for NodeKind {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.pad(self.as_str())
+ }
+}
diff --git a/src/syntax/pretty.rs b/src/syntax/pretty.rs
index 3d02f39f..c453fb56 100644
--- a/src/syntax/pretty.rs
+++ b/src/syntax/pretty.rs
@@ -2,7 +2,7 @@
use std::fmt::{self, Arguments, Write};
-use super::*;
+use super::ast::*;
/// Pretty print an item and return the resulting string.
pub fn pretty<T>(item: &T) -> String
@@ -46,20 +46,24 @@ impl Printer {
Write::write_fmt(self, fmt)
}
- /// Write a list of items joined by a joiner.
- pub fn join<T, I, F>(&mut self, items: I, joiner: &str, mut write_item: F)
+ /// Write a list of items joined by a joiner and return how many there were.
+ pub fn join<T, I, F>(&mut self, items: I, joiner: &str, mut write_item: F) -> usize
where
I: IntoIterator<Item = T>,
F: FnMut(T, &mut Self),
{
+ let mut count = 0;
let mut iter = items.into_iter();
if let Some(first) = iter.next() {
write_item(first, self);
+ count += 1;
}
for item in iter {
self.push_str(joiner);
write_item(item, self);
+ count += 1;
}
+ count
}
/// Finish pretty printing and return the underlying buffer.
@@ -77,7 +81,7 @@ impl Write for Printer {
impl Pretty for Markup {
fn pretty(&self, p: &mut Printer) {
- for node in self {
+ for node in self.nodes() {
node.pretty(p);
}
}
@@ -88,12 +92,13 @@ impl Pretty for MarkupNode {
match self {
// TODO: Handle escaping.
Self::Space => p.push(' '),
- Self::Linebreak(_) => p.push_str(r"\"),
- Self::Parbreak(_) => p.push_str("\n\n"),
- Self::Strong(_) => p.push('*'),
- Self::Emph(_) => p.push('_'),
+ Self::Linebreak => p.push_str(r"\"),
+ Self::Parbreak => p.push_str("\n\n"),
+ Self::Strong => p.push('*'),
+ Self::Emph => p.push('_'),
Self::Text(text) => p.push_str(text),
Self::Raw(raw) => raw.pretty(p),
+ Self::Math(math) => math.pretty(p),
Self::Heading(heading) => heading.pretty(p),
Self::List(list) => list.pretty(p),
Self::Enum(enum_) => enum_.pretty(p),
@@ -136,7 +141,7 @@ impl Pretty for RawNode {
// Language tag.
if let Some(lang) = &self.lang {
- lang.pretty(p);
+ p.push_str(lang);
}
// Start untrimming.
@@ -163,38 +168,52 @@ impl Pretty for RawNode {
}
}
+impl Pretty for MathNode {
+ fn pretty(&self, p: &mut Printer) {
+ p.push('$');
+ if self.display {
+ p.push('[');
+ }
+ p.push_str(&self.formula);
+ if self.display {
+ p.push(']');
+ }
+ p.push('$');
+ }
+}
+
impl Pretty for HeadingNode {
fn pretty(&self, p: &mut Printer) {
- for _ in 0 .. self.level {
+ for _ in 0 .. self.level() {
p.push('=');
}
p.push(' ');
- self.body.pretty(p);
+ self.body().pretty(p);
}
}
impl Pretty for ListNode {
fn pretty(&self, p: &mut Printer) {
p.push_str("- ");
- self.body.pretty(p);
+ self.body().pretty(p);
}
}
impl Pretty for EnumNode {
fn pretty(&self, p: &mut Printer) {
- if let Some(number) = self.number {
+ if let Some(number) = self.number() {
write!(p, "{}", number).unwrap();
}
p.push_str(". ");
- self.body.pretty(p);
+ self.body().pretty(p);
}
}
impl Pretty for Expr {
fn pretty(&self, p: &mut Printer) {
match self {
- Self::Ident(v) => v.pretty(p),
Self::Lit(v) => v.pretty(p),
+ Self::Ident(v) => v.pretty(p),
Self::Array(v) => v.pretty(p),
Self::Dict(v) => v.pretty(p),
Self::Template(v) => v.pretty(p),
@@ -217,17 +236,17 @@ impl Pretty for Expr {
impl Pretty for Lit {
fn pretty(&self, p: &mut Printer) {
- match self {
- Self::None(_) => p.push_str("none"),
- Self::Auto(_) => p.push_str("auto"),
- Self::Bool(_, v) => write!(p, "{}", v).unwrap(),
- Self::Int(_, v) => write!(p, "{}", v).unwrap(),
- Self::Float(_, v) => write!(p, "{}", v).unwrap(),
- Self::Length(_, v, u) => write!(p, "{}{:?}", v, u).unwrap(),
- Self::Angle(_, v, u) => write!(p, "{}{:?}", v, u).unwrap(),
- Self::Percent(_, v) => write!(p, "{}%", v).unwrap(),
- Self::Fractional(_, v) => write!(p, "{}fr", v).unwrap(),
- Self::Str(_, v) => write!(p, "{:?}", v).unwrap(),
+ match self.kind() {
+ LitKind::None => p.push_str("none"),
+ LitKind::Auto => p.push_str("auto"),
+ LitKind::Bool(v) => write!(p, "{}", v).unwrap(),
+ LitKind::Int(v) => write!(p, "{}", v).unwrap(),
+ LitKind::Float(v) => write!(p, "{}", v).unwrap(),
+ LitKind::Length(v, u) => write!(p, "{}{:?}", v, u).unwrap(),
+ LitKind::Angle(v, u) => write!(p, "{}{:?}", v, u).unwrap(),
+ LitKind::Percent(v) => write!(p, "{}%", v).unwrap(),
+ LitKind::Fractional(v) => write!(p, "{}fr", v).unwrap(),
+ LitKind::Str(v) => write!(p, "{:?}", v).unwrap(),
}
}
}
@@ -235,8 +254,10 @@ impl Pretty for Lit {
impl Pretty for ArrayExpr {
fn pretty(&self, p: &mut Printer) {
p.push('(');
- p.join(&self.items, ", ", |item, p| item.pretty(p));
- if self.items.len() == 1 {
+
+ let items = self.items();
+ let len = p.join(items, ", ", |item, p| item.pretty(p));
+ if len == 1 {
p.push(',');
}
p.push(')');
@@ -246,10 +267,9 @@ impl Pretty for ArrayExpr {
impl Pretty for DictExpr {
fn pretty(&self, p: &mut Printer) {
p.push('(');
- if self.items.is_empty() {
+ let len = p.join(self.items(), ", ", |named, p| named.pretty(p));
+ if len == 0 {
p.push(':');
- } else {
- p.join(&self.items, ", ", |named, p| named.pretty(p));
}
p.push(')');
}
@@ -257,16 +277,16 @@ impl Pretty for DictExpr {
impl Pretty for Named {
fn pretty(&self, p: &mut Printer) {
- self.name.pretty(p);
+ self.name().pretty(p);
p.push_str(": ");
- self.expr.pretty(p);
+ self.expr().pretty(p);
}
}
impl Pretty for TemplateExpr {
fn pretty(&self, p: &mut Printer) {
p.push('[');
- self.body.pretty(p);
+ self.body().pretty(p);
p.push(']');
}
}
@@ -274,7 +294,7 @@ impl Pretty for TemplateExpr {
impl Pretty for GroupExpr {
fn pretty(&self, p: &mut Printer) {
p.push('(');
- self.expr.pretty(p);
+ self.expr().pretty(p);
p.push(')');
}
}
@@ -282,11 +302,11 @@ impl Pretty for GroupExpr {
impl Pretty for BlockExpr {
fn pretty(&self, p: &mut Printer) {
p.push('{');
- if self.exprs.len() > 1 {
+ if self.exprs().count() > 1 {
p.push(' ');
}
- p.join(&self.exprs, "; ", |expr, p| expr.pretty(p));
- if self.exprs.len() > 1 {
+ let len = p.join(self.exprs(), "; ", |expr, p| expr.pretty(p));
+ if len > 1 {
p.push(' ');
}
p.push('}');
@@ -295,11 +315,12 @@ impl Pretty for BlockExpr {
impl Pretty for UnaryExpr {
fn pretty(&self, p: &mut Printer) {
- self.op.pretty(p);
- if self.op == UnOp::Not {
+ let op = self.op();
+ op.pretty(p);
+ if op == UnOp::Not {
p.push(' ');
}
- self.expr.pretty(p);
+ self.expr().pretty(p);
}
}
@@ -311,11 +332,11 @@ impl Pretty for UnOp {
impl Pretty for BinaryExpr {
fn pretty(&self, p: &mut Printer) {
- self.lhs.pretty(p);
+ self.lhs().pretty(p);
p.push(' ');
- self.op.pretty(p);
+ self.op().pretty(p);
p.push(' ');
- self.rhs.pretty(p);
+ self.rhs().pretty(p);
}
}
@@ -327,7 +348,7 @@ impl Pretty for BinOp {
impl Pretty for CallExpr {
fn pretty(&self, p: &mut Printer) {
- self.callee.pretty(p);
+ self.callee().pretty(p);
let mut write_args = |items: &[CallArg]| {
p.push('(');
@@ -335,7 +356,8 @@ impl Pretty for CallExpr {
p.push(')');
};
- match self.args.items.as_slice() {
+ let args: Vec<_> = self.args().items().collect();
+ match args.as_slice() {
// This can be moved behind the arguments.
//
// Example: Transforms "#v(a, [b])" => "#v(a)[b]".
@@ -345,7 +367,6 @@ impl Pretty for CallExpr {
}
template.pretty(p);
}
-
items => write_args(items),
}
}
@@ -353,7 +374,7 @@ impl Pretty for CallExpr {
impl Pretty for CallArgs {
fn pretty(&self, p: &mut Printer) {
- p.join(&self.items, ", ", |item, p| item.pretty(p));
+ p.join(self.items(), ", ", |item, p| item.pretty(p));
}
}
@@ -372,15 +393,16 @@ impl Pretty for CallArg {
impl Pretty for ClosureExpr {
fn pretty(&self, p: &mut Printer) {
- if let [param] = self.params.as_slice() {
+ let params: Vec<_> = self.params().collect();
+ if let [param] = params.as_slice() {
param.pretty(p);
} else {
p.push('(');
- p.join(self.params.iter(), ", ", |item, p| item.pretty(p));
+ p.join(params.iter(), ", ", |item, p| item.pretty(p));
p.push(')');
}
p.push_str(" => ");
- self.body.pretty(p);
+ self.body().pretty(p);
}
}
@@ -399,9 +421,9 @@ impl Pretty for ClosureParam {
impl Pretty for WithExpr {
fn pretty(&self, p: &mut Printer) {
- self.callee.pretty(p);
+ self.callee().pretty(p);
p.push_str(" with (");
- self.args.pretty(p);
+ self.args().pretty(p);
p.push(')');
}
}
@@ -409,13 +431,13 @@ impl Pretty for WithExpr {
impl Pretty for LetExpr {
fn pretty(&self, p: &mut Printer) {
p.push_str("let ");
- self.binding.pretty(p);
- if let Some(Expr::Closure(closure)) = &self.init {
+ self.binding().pretty(p);
+ if let Some(Expr::Closure(closure)) = self.init() {
p.push('(');
- p.join(closure.params.iter(), ", ", |item, p| item.pretty(p));
+ p.join(closure.params(), ", ", |item, p| item.pretty(p));
p.push_str(") = ");
- closure.body.pretty(p);
- } else if let Some(init) = &self.init {
+ closure.body().pretty(p);
+ } else if let Some(init) = self.init() {
p.push_str(" = ");
init.pretty(p);
}
@@ -425,10 +447,10 @@ impl Pretty for LetExpr {
impl Pretty for IfExpr {
fn pretty(&self, p: &mut Printer) {
p.push_str("if ");
- self.condition.pretty(p);
+ self.condition().pretty(p);
p.push(' ');
- self.if_body.pretty(p);
- if let Some(expr) = &self.else_body {
+ self.if_body().pretty(p);
+ if let Some(expr) = self.else_body() {
p.push_str(" else ");
expr.pretty(p);
}
@@ -438,42 +460,40 @@ impl Pretty for IfExpr {
impl Pretty for WhileExpr {
fn pretty(&self, p: &mut Printer) {
p.push_str("while ");
- self.condition.pretty(p);
+ self.condition().pretty(p);
p.push(' ');
- self.body.pretty(p);
+ self.body().pretty(p);
}
}
impl Pretty for ForExpr {
fn pretty(&self, p: &mut Printer) {
p.push_str("for ");
- self.pattern.pretty(p);
+ self.pattern().pretty(p);
p.push_str(" in ");
- self.iter.pretty(p);
+ self.iter().pretty(p);
p.push(' ');
- self.body.pretty(p);
+ self.body().pretty(p);
}
}
impl Pretty for ForPattern {
fn pretty(&self, p: &mut Printer) {
- match self {
- Self::Value(v) => v.pretty(p),
- Self::KeyValue(k, v) => {
- k.pretty(p);
- p.push_str(", ");
- v.pretty(p);
- }
+ if let Some(key) = self.key() {
+ key.pretty(p);
+ p.push_str(", ");
}
+
+ self.value().pretty(p);
}
}
impl Pretty for ImportExpr {
fn pretty(&self, p: &mut Printer) {
p.push_str("import ");
- self.imports.pretty(p);
+ self.imports().pretty(p);
p.push_str(" from ");
- self.path.pretty(p);
+ self.path().pretty(p);
}
}
@@ -481,7 +501,9 @@ impl Pretty for Imports {
fn pretty(&self, p: &mut Printer) {
match self {
Self::Wildcard => p.push('*'),
- Self::Idents(idents) => p.join(idents, ", ", |item, p| item.pretty(p)),
+ Self::Items(idents) => {
+ p.join(idents, ", ", |item, p| item.pretty(p));
+ }
}
}
}
@@ -489,20 +511,19 @@ impl Pretty for Imports {
impl Pretty for IncludeExpr {
fn pretty(&self, p: &mut Printer) {
p.push_str("include ");
- self.path.pretty(p);
+ self.path().pretty(p);
}
}
impl Pretty for Ident {
fn pretty(&self, p: &mut Printer) {
- p.push_str(self.as_str());
+ p.push_str(self);
}
}
#[cfg(test)]
mod tests {
use super::*;
- use crate::parse::parse;
use crate::source::SourceFile;
#[track_caller]
@@ -513,7 +534,7 @@ mod tests {
#[track_caller]
fn test_parse(src: &str, expected: &str) {
let source = SourceFile::detached(src);
- let ast = parse(&source).unwrap();
+ let ast = source.ast().unwrap();
let found = pretty(&ast);
if found != expected {
println!("tree: {:#?}", ast);
@@ -551,6 +572,11 @@ mod tests {
test_parse("``` 1```", "`1`");
test_parse("``` 1 ```", "`1 `");
test_parse("```` ` ````", "``` ` ```");
+
+ // Math node.
+ roundtrip("$$");
+ roundtrip("$a+b$");
+ roundtrip("$[ a^2 + b^2 = c^2 ]$");
}
#[test]
diff --git a/src/syntax/span.rs b/src/syntax/span.rs
index bfb9e755..47d96589 100644
--- a/src/syntax/span.rs
+++ b/src/syntax/span.rs
@@ -1,6 +1,6 @@
use std::cmp::Ordering;
use std::fmt::{self, Debug, Formatter};
-use std::ops::{Add, Range};
+use std::ops::Range;
use serde::{Deserialize, Serialize};
@@ -53,23 +53,19 @@ pub struct Span {
/// The id of the source file.
pub source: SourceId,
/// The inclusive start position.
- pub start: Pos,
+ pub start: usize,
/// The inclusive end position.
- pub end: Pos,
+ pub end: usize,
}
impl Span {
/// Create a new span from start and end positions.
- pub fn new(source: SourceId, start: impl Into<Pos>, end: impl Into<Pos>) -> Self {
- Self {
- source,
- start: start.into(),
- end: end.into(),
- }
+ pub fn new(source: SourceId, start: usize, end: usize) -> Self {
+ Self { source, start, end }
}
/// Create a span including just a single position.
- pub fn at(source: SourceId, pos: impl Into<Pos> + Copy) -> Self {
+ pub fn at(source: SourceId, pos: usize) -> Self {
Self::new(source, pos, pos)
}
@@ -77,19 +73,34 @@ impl Span {
pub fn detached() -> Self {
Self {
source: SourceId::from_raw(0),
- start: Pos::ZERO,
- end: Pos::ZERO,
+ start: 0,
+ end: 0,
}
}
/// Create a span with a different start position.
- pub fn with_start(self, start: impl Into<Pos>) -> Self {
- Self { start: start.into(), ..self }
+ pub fn with_start(self, start: usize) -> Self {
+ Self { start, ..self }
}
/// Create a span with a different end position.
- pub fn with_end(self, end: impl Into<Pos>) -> Self {
- Self { end: end.into(), ..self }
+ pub fn with_end(self, end: usize) -> Self {
+ Self { end, ..self }
+ }
+
+ /// The byte length of the spanned region.
+ pub fn len(self) -> usize {
+ self.end - self.start
+ }
+
+ /// A new span at the position of this span's start.
+ pub fn at_start(&self) -> Span {
+ Self::at(self.source, self.start)
+ }
+
+ /// A new span at the position of this span's end.
+ pub fn at_end(&self) -> Span {
+ Self::at(self.source, self.end)
}
/// Create a new span with the earlier start and later end position.
@@ -109,14 +120,19 @@ impl Span {
*self = self.join(other)
}
+ /// Test whether a position is within the span.
+ pub fn contains(&self, pos: usize) -> bool {
+ self.start <= pos && self.end >= pos
+ }
+
/// Test whether one span complete contains the other span.
- pub fn contains(self, other: Self) -> bool {
+ pub fn surrounds(self, other: Self) -> bool {
self.source == other.source && self.start <= other.start && self.end >= other.end
}
- /// Convert to a `Range<Pos>` for indexing.
+ /// Convert to a `Range<usize>` for indexing.
pub fn to_range(self) -> Range<usize> {
- self.start.to_usize() .. self.end.to_usize()
+ self.start .. self.end
}
}
@@ -135,77 +151,3 @@ impl PartialOrd for Span {
}
}
}
-
-/// A byte position in source code.
-#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
-pub struct Pos(pub u32);
-
-impl Pos {
- /// The zero position.
- pub const ZERO: Self = Self(0);
-
- /// Convert to a usize for indexing.
- pub fn to_usize(self) -> usize {
- self.0 as usize
- }
-}
-
-impl Debug for Pos {
- fn fmt(&self, f: &mut Formatter) -> fmt::Result {
- Debug::fmt(&self.0, f)
- }
-}
-
-impl From<u32> for Pos {
- fn from(index: u32) -> Self {
- Self(index)
- }
-}
-
-impl From<usize> for Pos {
- fn from(index: usize) -> Self {
- Self(index as u32)
- }
-}
-
-impl<T> Add<T> for Pos
-where
- T: Into<Pos>,
-{
- type Output = Self;
-
- fn add(self, rhs: T) -> Self {
- Pos(self.0 + rhs.into().0)
- }
-}
-
-/// Convert a position or range into a span.
-pub trait IntoSpan {
- /// Convert into a span by providing the source id.
- fn into_span(self, source: SourceId) -> Span;
-}
-
-impl IntoSpan for Span {
- fn into_span(self, source: SourceId) -> Span {
- debug_assert_eq!(self.source, source);
- self
- }
-}
-
-impl IntoSpan for Pos {
- fn into_span(self, source: SourceId) -> Span {
- Span::new(source, self, self)
- }
-}
-
-impl IntoSpan for usize {
- fn into_span(self, source: SourceId) -> Span {
- Span::new(source, self, self)
- }
-}
-
-impl IntoSpan for Range<usize> {
- fn into_span(self, source: SourceId) -> Span {
- Span::new(source, self.start, self.end)
- }
-}
diff --git a/src/syntax/token.rs b/src/syntax/token.rs
deleted file mode 100644
index 22dd104b..00000000
--- a/src/syntax/token.rs
+++ /dev/null
@@ -1,276 +0,0 @@
-use crate::geom::{AngularUnit, LengthUnit};
-
-/// A minimal semantic entity of source code.
-#[derive(Debug, Copy, Clone, PartialEq)]
-pub enum Token<'s> {
- /// A left square bracket: `[`.
- LeftBracket,
- /// A right square bracket: `]`.
- RightBracket,
- /// A left curly brace: `{`.
- LeftBrace,
- /// A right curly brace: `}`.
- RightBrace,
- /// A left round parenthesis: `(`.
- LeftParen,
- /// A right round parenthesis: `)`.
- RightParen,
- /// An asterisk: `*`.
- Star,
- /// An underscore: `_`.
- Underscore,
- /// A tilde: `~`.
- Tilde,
- /// Two hyphens: `--`.
- HyphHyph,
- /// Three hyphens: `---`.
- HyphHyphHyph,
- /// A backslash followed by nothing or whitespace: `\`.
- Backslash,
- /// A comma: `,`.
- Comma,
- /// A semicolon: `;`.
- Semicolon,
- /// A colon: `:`.
- Colon,
- /// A plus: `+`.
- Plus,
- /// A hyphen: `-`.
- Hyph,
- /// A slash: `/`.
- Slash,
- /// A single equals sign: `=`.
- Eq,
- /// Two equals signs: `==`.
- EqEq,
- /// An exclamation mark followed by an equals sign: `!=`.
- ExclEq,
- /// A less-than sign: `<`.
- Lt,
- /// A less-than sign followed by an equals sign: `<=`.
- LtEq,
- /// A greater-than sign: `>`.
- Gt,
- /// A greater-than sign followed by an equals sign: `>=`.
- GtEq,
- /// A plus followed by an equals sign: `+=`.
- PlusEq,
- /// A hyphen followed by an equals sign: `-=`.
- HyphEq,
- /// An asterisk followed by an equals sign: `*=`.
- StarEq,
- /// A slash followed by an equals sign: `/=`.
- SlashEq,
- /// Two dots: `..`.
- Dots,
- /// An equals sign followed by a greater-than sign: `=>`.
- Arrow,
- /// The `not` operator.
- Not,
- /// The `and` operator.
- And,
- /// The `or` operator.
- Or,
- /// The `with` operator.
- With,
- /// The none literal: `none`.
- None,
- /// The auto literal: `auto`.
- Auto,
- /// The `let` keyword.
- Let,
- /// The `if` keyword.
- If,
- /// The `else` keyword.
- Else,
- /// The `for` keyword.
- For,
- /// The `in` keyword.
- In,
- /// The `while` keyword.
- While,
- /// The `break` keyword.
- Break,
- /// The `continue` keyword.
- Continue,
- /// The `return` keyword.
- Return,
- /// The `import` keyword.
- Import,
- /// The `include` keyword.
- Include,
- /// The `from` keyword.
- From,
- /// One or more whitespace characters.
- ///
- /// The contained `usize` denotes the number of newlines that were contained
- /// in the whitespace.
- Space(usize),
- /// A consecutive non-markup string.
- Text(&'s str),
- /// A slash and the letter "u" followed by a hexadecimal unicode entity
- /// enclosed in curly braces: `\u{1F5FA}`.
- UnicodeEscape(UnicodeEscapeToken<'s>),
- /// An arbitrary number of backticks followed by inner contents, terminated
- /// with the same number of backticks: `` `...` ``.
- Raw(RawToken<'s>),
- /// One or two dollar signs followed by inner contents, terminated with the
- /// same number of dollar signs.
- Math(MathToken<'s>),
- /// A numbering: `23.`.
- ///
- /// Can also exist without the number: `.`.
- Numbering(Option<usize>),
- /// An identifier: `center`.
- Ident(&'s str),
- /// A boolean: `true`, `false`.
- Bool(bool),
- /// An integer: `120`.
- Int(i64),
- /// A floating-point number: `1.2`, `10e-4`.
- Float(f64),
- /// A length: `12pt`, `3cm`.
- Length(f64, LengthUnit),
- /// An angle: `90deg`.
- Angle(f64, AngularUnit),
- /// A percentage: `50%`.
- ///
- /// _Note_: `50%` is stored as `50.0` here, as in the corresponding
- /// [literal](super::Lit::Percent).
- Percent(f64),
- /// A fraction unit: `3fr`.
- Fraction(f64),
- /// A quoted string: `"..."`.
- Str(StrToken<'s>),
- /// Two slashes followed by inner contents, terminated with a newline:
- /// `//<str>\n`.
- LineComment(&'s str),
- /// A slash and a star followed by inner contents, terminated with a star
- /// and a slash: `/*<str>*/`.
- ///
- /// The comment can contain nested block comments.
- BlockComment(&'s str),
- /// Things that are not valid tokens.
- Invalid(&'s str),
-}
-
-/// A quoted string token: `"..."`.
-#[derive(Debug, Copy, Clone, PartialEq)]
-pub struct StrToken<'s> {
- /// The string inside the quotes.
- ///
- /// _Note_: If the string contains escape sequences these are not yet
- /// applied to be able to just store a string slice here instead of
- /// a `String`. The resolving is done later in the parser.
- pub string: &'s str,
- /// Whether the closing quote was present.
- pub terminated: bool,
-}
-
-/// A raw block token: `` `...` ``.
-#[derive(Debug, Copy, Clone, PartialEq)]
-pub struct RawToken<'s> {
- /// The raw text between the backticks.
- pub text: &'s str,
- /// The number of opening backticks.
- pub backticks: usize,
- /// Whether all closing backticks were present.
- pub terminated: bool,
-}
-
-/// A math formula token: `$2pi + x$` or `$[f'(x) = x^2]$`.
-#[derive(Debug, Copy, Clone, PartialEq)]
-pub struct MathToken<'s> {
- /// The formula between the dollars.
- pub formula: &'s str,
- /// Whether the formula is display-level, that is, it is surrounded by
- /// `$[..]`.
- pub display: bool,
- /// Whether the closing dollars were present.
- pub terminated: bool,
-}
-
-/// A unicode escape sequence token: `\u{1F5FA}`.
-#[derive(Debug, Copy, Clone, PartialEq)]
-pub struct UnicodeEscapeToken<'s> {
- /// The escape sequence between the braces.
- pub sequence: &'s str,
- /// Whether the closing brace was present.
- pub terminated: bool,
-}
-
-impl<'s> Token<'s> {
- /// The English name of this token for use in error messages.
- pub fn name(self) -> &'static str {
- match self {
- Self::LeftBracket => "opening bracket",
- Self::RightBracket => "closing bracket",
- Self::LeftBrace => "opening brace",
- Self::RightBrace => "closing brace",
- Self::LeftParen => "opening paren",
- Self::RightParen => "closing paren",
- Self::Star => "star",
- Self::Underscore => "underscore",
- Self::Tilde => "tilde",
- Self::HyphHyph => "en dash",
- Self::HyphHyphHyph => "em dash",
- Self::Backslash => "backslash",
- Self::Comma => "comma",
- Self::Semicolon => "semicolon",
- Self::Colon => "colon",
- Self::Plus => "plus",
- Self::Hyph => "minus",
- Self::Slash => "slash",
- Self::Eq => "assignment operator",
- Self::EqEq => "equality operator",
- Self::ExclEq => "inequality operator",
- Self::Lt => "less-than operator",
- Self::LtEq => "less-than or equal operator",
- Self::Gt => "greater-than operator",
- Self::GtEq => "greater-than or equal operator",
- Self::PlusEq => "add-assign operator",
- Self::HyphEq => "subtract-assign operator",
- Self::StarEq => "multiply-assign operator",
- Self::SlashEq => "divide-assign operator",
- Self::Dots => "dots",
- Self::Arrow => "arrow",
- Self::Not => "operator `not`",
- Self::And => "operator `and`",
- Self::Or => "operator `or`",
- Self::With => "operator `with`",
- Self::None => "`none`",
- Self::Auto => "`auto`",
- Self::Let => "keyword `let`",
- Self::If => "keyword `if`",
- Self::Else => "keyword `else`",
- Self::For => "keyword `for`",
- Self::In => "keyword `in`",
- Self::While => "keyword `while`",
- Self::Break => "keyword `break`",
- Self::Continue => "keyword `continue`",
- Self::Return => "keyword `return`",
- Self::Import => "keyword `import`",
- Self::Include => "keyword `include`",
- Self::From => "keyword `from`",
- Self::Space(_) => "space",
- Self::Text(_) => "text",
- Self::UnicodeEscape(_) => "unicode escape sequence",
- Self::Raw(_) => "raw block",
- Self::Math(_) => "math formula",
- Self::Numbering(_) => "numbering",
- Self::Ident(_) => "identifier",
- Self::Bool(_) => "boolean",
- Self::Int(_) => "integer",
- Self::Float(_) => "float",
- Self::Length(_, _) => "length",
- Self::Angle(_, _) => "angle",
- Self::Percent(_) => "percentage",
- Self::Fraction(_) => "`fr` value",
- Self::Str(_) => "string",
- Self::LineComment(_) => "line comment",
- Self::BlockComment(_) => "block comment",
- Self::Invalid("*/") => "end of block comment",
- Self::Invalid(_) => "invalid token",
- }
- }
-}
diff --git a/src/syntax/visit.rs b/src/syntax/visit.rs
deleted file mode 100644
index 40e8eb93..00000000
--- a/src/syntax/visit.rs
+++ /dev/null
@@ -1,263 +0,0 @@
-//! Mutable and immutable syntax tree traversal.
-
-use super::*;
-
-/// Implement the immutable and the mutable visitor version.
-macro_rules! impl_visitors {
- ($($name:ident($($tts:tt)*) $body:block)*) => {
- macro_rules! r {
- (rc: $x:expr) => { $x.as_ref() };
- ($x:expr) => { &$x };
- }
-
- impl_visitor! {
- Visit,
- immutable,
- immutably,
- [$(($name($($tts)*) $body))*]
- }
-
- macro_rules! r {
- (rc: $x:expr) => { std::rc::Rc::make_mut(&mut $x) };
- ($x:expr) => { &mut $x };
- }
-
- impl_visitor! {
- VisitMut,
- mutable,
- mutably,
- [$(($name($($tts)*) $body mut))*] mut
- }
- };
-}
-
-/// Implement an immutable or mutable visitor.
-macro_rules! impl_visitor {
- (
- $visit:ident,
- $mutability:ident,
- $adjective:ident,
- [$((
- $name:ident($v:ident, $node:ident: $ty:ty)
- $body:block
- $($fmut:tt)?
- ))*]
- $($mut:tt)?
- ) => {
- #[doc = concat!("Visit syntax trees ", stringify!($adjective), ".")]
- pub trait $visit<'ast> {
- /// Visit a definition of a binding.
- ///
- /// Bindings are, for example, left-hand side of let expressions,
- /// and key/value patterns in for loops.
- fn visit_binding(&mut self, _: &'ast $($mut)? Ident) {}
-
- /// Visit the entry into a scope.
- fn visit_enter(&mut self) {}
-
- /// Visit the exit from a scope.
- fn visit_exit(&mut self) {}
-
- $(fn $name(&mut self, $node: &'ast $($fmut)? $ty) {
- $mutability::$name(self, $node);
- })*
- }
-
- #[doc = concat!("Visitor functions that are ", stringify!($mutability), ".")]
- pub mod $mutability {
- use super::*;
- $(
- #[doc = concat!("Visit a node of type [`", stringify!($ty), "`].")]
- pub fn $name<'ast, V>($v: &mut V, $node: &'ast $($fmut)? $ty)
- where
- V: $visit<'ast> + ?Sized
- $body
- )*
- }
- };
-}
-
-impl_visitors! {
- visit_tree(v, markup: Markup) {
- for node in markup {
- v.visit_node(node);
- }
- }
-
- visit_node(v, node: MarkupNode) {
- match node {
- MarkupNode::Space => {}
- MarkupNode::Linebreak(_) => {}
- MarkupNode::Parbreak(_) => {}
- MarkupNode::Strong(_) => {}
- MarkupNode::Emph(_) => {}
- MarkupNode::Text(_) => {}
- MarkupNode::Raw(_) => {}
- MarkupNode::Heading(n) => v.visit_heading(n),
- MarkupNode::List(n) => v.visit_list(n),
- MarkupNode::Enum(n) => v.visit_enum(n),
- MarkupNode::Expr(n) => v.visit_expr(n),
- }
- }
-
- visit_heading(v, heading: HeadingNode) {
- v.visit_tree(r!(heading.body));
- }
-
- visit_list(v, list: ListNode) {
- v.visit_tree(r!(list.body));
- }
-
- visit_enum(v, enum_: EnumNode) {
- v.visit_tree(r!(enum_.body));
- }
-
- visit_expr(v, expr: Expr) {
- match expr {
- Expr::Ident(_) => {}
- Expr::Lit(_) => {},
- Expr::Array(e) => v.visit_array(e),
- Expr::Dict(e) => v.visit_dict(e),
- Expr::Template(e) => v.visit_template(e),
- Expr::Group(e) => v.visit_group(e),
- Expr::Block(e) => v.visit_block(e),
- Expr::Unary(e) => v.visit_unary(e),
- Expr::Binary(e) => v.visit_binary(e),
- Expr::Call(e) => v.visit_call(e),
- Expr::Closure(e) => v.visit_closure(e),
- Expr::With(e) => v.visit_with(e),
- Expr::Let(e) => v.visit_let(e),
- Expr::If(e) => v.visit_if(e),
- Expr::While(e) => v.visit_while(e),
- Expr::For(e) => v.visit_for(e),
- Expr::Import(e) => v.visit_import(e),
- Expr::Include(e) => v.visit_include(e),
- }
- }
-
- visit_array(v, array: ArrayExpr) {
- for expr in r!(array.items) {
- v.visit_expr(expr);
- }
- }
-
- visit_dict(v, dict: DictExpr) {
- for named in r!(dict.items) {
- v.visit_expr(r!(named.expr));
- }
- }
-
- visit_template(v, template: TemplateExpr) {
- v.visit_enter();
- v.visit_tree(r!(template.body));
- v.visit_exit();
- }
-
- visit_group(v, group: GroupExpr) {
- v.visit_expr(r!(group.expr));
- }
-
- visit_block(v, block: BlockExpr) {
- v.visit_enter();
- for expr in r!(block.exprs) {
- v.visit_expr(expr);
- }
- v.visit_exit();
- }
-
- visit_binary(v, binary: BinaryExpr) {
- v.visit_expr(r!(binary.lhs));
- v.visit_expr(r!(binary.rhs));
- }
-
- visit_unary(v, unary: UnaryExpr) {
- v.visit_expr(r!(unary.expr));
- }
-
- visit_call(v, call: CallExpr) {
- v.visit_expr(r!(call.callee));
- v.visit_args(r!(call.args));
- }
-
- visit_args(v, args: CallArgs) {
- for arg in r!(args.items) {
- v.visit_arg(arg);
- }
- }
-
- visit_arg(v, arg: CallArg) {
- match arg {
- CallArg::Pos(expr) => v.visit_expr(expr),
- CallArg::Named(named) => v.visit_expr(r!(named.expr)),
- CallArg::Spread(expr) => v.visit_expr(expr),
- }
- }
-
- visit_closure(v, closure: ClosureExpr) {
- for param in r!(closure.params) {
- v.visit_param(param);
- }
- v.visit_expr(r!(rc: closure.body));
- }
-
- visit_param(v, param: ClosureParam) {
- match param {
- ClosureParam::Pos(binding) => v.visit_binding(binding),
- ClosureParam::Named(named) => {
- v.visit_binding(r!(named.name));
- v.visit_expr(r!(named.expr));
- }
- ClosureParam::Sink(binding) => v.visit_binding(binding),
- }
- }
-
- visit_with(v, with_expr: WithExpr) {
- v.visit_expr(r!(with_expr.callee));
- v.visit_args(r!(with_expr.args));
- }
-
- visit_let(v, let_expr: LetExpr) {
- if let Some(init) = r!(let_expr.init) {
- v.visit_expr(init);
- }
- v.visit_binding(r!(let_expr.binding));
- }
-
- visit_if(v, if_expr: IfExpr) {
- v.visit_expr(r!(if_expr.condition));
- v.visit_expr(r!(if_expr.if_body));
- if let Some(body) = r!(if_expr.else_body) {
- v.visit_expr(body);
- }
- }
-
- visit_while(v, while_expr: WhileExpr) {
- v.visit_expr(r!(while_expr.condition));
- v.visit_expr(r!(while_expr.body));
- }
-
- visit_for(v, for_expr: ForExpr) {
- v.visit_expr(r!(for_expr.iter));
- match r!(for_expr.pattern) {
- ForPattern::Value(value) => v.visit_binding(value),
- ForPattern::KeyValue(key, value) => {
- v.visit_binding(key);
- v.visit_binding(value);
- }
- }
- v.visit_expr(r!(for_expr.body));
- }
-
- visit_import(v, import_expr: ImportExpr) {
- v.visit_expr(r!(import_expr.path));
- if let Imports::Idents(idents) = r!(import_expr.imports) {
- for ident in idents {
- v.visit_binding(ident);
- }
- }
- }
-
- visit_include(v, include_expr: IncludeExpr) {
- v.visit_expr(r!(include_expr.path));
- }
-}
diff --git a/tests/ref/markup/heading.png b/tests/ref/markup/heading.png
index ca52644b..c33da420 100644
--- a/tests/ref/markup/heading.png
+++ b/tests/ref/markup/heading.png
Binary files differ
diff --git a/tests/ref/markup/math.png b/tests/ref/markup/math.png
new file mode 100644
index 00000000..426f3dbf
--- /dev/null
+++ b/tests/ref/markup/math.png
Binary files differ
diff --git a/tests/typ/code/closure.typ b/tests/typ/code/closure.typ
index 3b8b4261..14e74e7e 100644
--- a/tests/typ/code/closure.typ
+++ b/tests/typ/code/closure.typ
@@ -57,6 +57,52 @@
}
---
+// Import bindings.
+{
+ let b = "target.typ"
+ let f() = {
+ import b from b
+ b
+ }
+ test(f(), 1)
+}
+
+---
+// For loop bindings.
+{
+ let v = (1, 2, 3)
+ let s = 0
+ let f() = {
+ for v in v { s += v }
+ }
+ f()
+ test(s, 6)
+}
+
+---
+// Let + closure bindings.
+{
+ let g = "hi"
+ let f() = {
+ let g() = "bye"
+ g()
+ }
+ test(f(), "bye")
+}
+
+---
+// Parameter bindings.
+{
+ let x = 5
+ let g() = {
+ let f(x, y: x) = x + y
+ f
+ }
+
+ test(g()(8), 13)
+}
+
+---
// Don't leak environment.
{
// Error: 16-17 unknown variable
diff --git a/tests/typ/code/import.typ b/tests/typ/code/import.typ
index bc96e80c..683bb52a 100644
--- a/tests/typ/code/import.typ
+++ b/tests/typ/code/import.typ
@@ -79,7 +79,7 @@ This is never reached.
// Error: 22 expected keyword `from`
#import afrom, "b", c
-// Error: 8 expected import items
+// Error: 9 expected import items
#import from "target.typ"
// Error: 9-10 expected expression, found assignment operator
diff --git a/tests/typ/code/let.typ b/tests/typ/code/let.typ
index 3f3f9d35..cd7531b7 100644
--- a/tests/typ/code/let.typ
+++ b/tests/typ/code/let.typ
@@ -56,7 +56,7 @@ Three
#let v4 = 4 Four
// Terminated by semicolon even though we are in a paren group.
-// Error: 19 expected expression
+// Error: 18 expected expression
// Error: 19 expected closing paren
#let v5 = (1, 2 + ; Five
diff --git a/tests/typ/code/spread.typ b/tests/typ/code/spread.typ
index 8a9491d0..41e790a4 100644
--- a/tests/typ/code/spread.typ
+++ b/tests/typ/code/spread.typ
@@ -62,7 +62,7 @@
#min(.."nope")
---
-// Error: 10-14 expected identifier
+// Error: 8-14 expected identifier
#let f(..true) = none
---
@@ -70,9 +70,9 @@
#let f(..a, ..b) = none
---
-// Error: 5-6 spreading is not allowed here
+// Error: 3-6 spreading is not allowed here
{(..x)}
---
-// Error: 11-17 spreading is not allowed here
+// Error: 9-17 spreading is not allowed here
{(1, 2, ..(1, 2))}
diff --git a/tests/typ/markup/heading.typ b/tests/typ/markup/heading.typ
index 4647e7a6..cb022617 100644
--- a/tests/typ/markup/heading.typ
+++ b/tests/typ/markup/heading.typ
@@ -8,8 +8,8 @@
=== Level 2
====== Level 6
-// Too many hashtags.
-======= Level 7
+// At some point, it should stop shrinking.
+=========== Level 11
---
// Heading vs. no heading.
diff --git a/tests/typ/markup/math.typ b/tests/typ/markup/math.typ
new file mode 100644
index 00000000..cad01d10
--- /dev/null
+++ b/tests/typ/markup/math.typ
@@ -0,0 +1,12 @@
+// Test math formulas.
+
+---
+The sum of $a$ and $b$ is $a + b$.
+
+---
+We will show that:
+$[ a^2 + b^2 = c^2 ]$
+
+---
+// Error: 2:1 expected closing bracket and dollar sign
+$[a
diff --git a/tests/typ/markup/raw.typ b/tests/typ/markup/raw.typ
index d48432f7..0e053a9b 100644
--- a/tests/typ/markup/raw.typ
+++ b/tests/typ/markup/raw.typ
@@ -55,5 +55,5 @@ The keyword ```rust let```.
---
// Unterminated.
-// Error: 2:1 expected backtick(s)
+// Error: 2:1 expected 1 backtick
`endless
diff --git a/tests/typeset.rs b/tests/typeset.rs
index bde383c4..68e56343 100644
--- a/tests/typeset.rs
+++ b/tests/typeset.rs
@@ -24,7 +24,7 @@ use typst::loading::FsLoader;
use typst::parse::Scanner;
use typst::source::SourceFile;
use typst::style::Style;
-use typst::syntax::{Pos, Span};
+use typst::syntax::Span;
use typst::Context;
const TYP_DIR: &str = "./typ";
@@ -355,12 +355,12 @@ fn parse_metadata(source: &SourceFile) -> (Option<bool>, Vec<Error>) {
let comments =
lines[i ..].iter().take_while(|line| line.starts_with("//")).count();
- let pos = |s: &mut Scanner| -> Pos {
+ let pos = |s: &mut Scanner| -> usize {
let first = num(s) - 1;
let (delta, column) =
if s.eat_if(':') { (first, num(s) - 1) } else { (0, first) };
let line = (i + comments) + delta;
- source.line_column_to_byte(line, column).unwrap().into()
+ source.line_column_to_byte(line, column).unwrap()
};
let mut s = Scanner::new(rest);
@@ -375,10 +375,10 @@ fn parse_metadata(source: &SourceFile) -> (Option<bool>, Vec<Error>) {
}
fn print_error(source: &SourceFile, line: usize, error: &Error) {
- let start_line = 1 + line + source.byte_to_line(error.span.start.to_usize()).unwrap();
- let start_col = 1 + source.byte_to_column(error.span.start.to_usize()).unwrap();
- let end_line = 1 + line + source.byte_to_line(error.span.end.to_usize()).unwrap();
- let end_col = 1 + source.byte_to_column(error.span.end.to_usize()).unwrap();
+ let start_line = 1 + line + source.byte_to_line(error.span.start).unwrap();
+ let start_col = 1 + source.byte_to_column(error.span.start).unwrap();
+ let end_line = 1 + line + source.byte_to_line(error.span.end).unwrap();
+ let end_col = 1 + source.byte_to_column(error.span.end).unwrap();
println!(
"Error: {}:{}-{}:{}: {}",
start_line, start_col, end_line, end_col, error.message