diff options
| -rw-r--r-- | src/func.rs | 36 | ||||
| -rw-r--r-- | src/layout/model.rs | 2 | ||||
| -rw-r--r-- | src/lib.rs | 19 | ||||
| -rw-r--r-- | src/library/mod.rs | 4 | ||||
| -rw-r--r-- | src/library/page.rs | 4 | ||||
| -rw-r--r-- | src/library/spacing.rs | 6 | ||||
| -rw-r--r-- | src/macros.rs | 25 | ||||
| -rw-r--r-- | src/syntax/func/mod.rs | 17 | ||||
| -rw-r--r-- | src/syntax/parsing.rs | 749 | ||||
| -rw-r--r-- | src/syntax/scope.rs | 22 | ||||
| -rw-r--r-- | src/syntax/span.rs | 11 | ||||
| -rw-r--r-- | src/syntax/test.rs | 4 | ||||
| -rw-r--r-- | src/syntax/tokens.rs | 24 |
13 files changed, 460 insertions, 463 deletions
diff --git a/src/func.rs b/src/func.rs index 777e0b26..e948d049 100644 --- a/src/func.rs +++ b/src/func.rs @@ -1,9 +1,8 @@ //! Trait and prelude for custom functions. use crate::Pass; -use crate::syntax::ParseContext; -use crate::syntax::func::FuncHeader; -use crate::syntax::span::Spanned; +use crate::syntax::ParseState; +use crate::syntax::func::FuncCall; /// Types that are useful for creating your own functions. pub mod prelude { @@ -17,7 +16,6 @@ pub mod prelude { pub use crate::syntax::span::{Span, Spanned}; } - /// Parse a function from source code. pub trait ParseFunc { /// A metadata type whose value is passed into the function parser. This @@ -33,9 +31,8 @@ pub trait ParseFunc { /// Parse the header and body into this function given a context. fn parse( - header: FuncHeader, - body: Option<Spanned<&str>>, - ctx: ParseContext, + header: FuncCall, + state: &ParseState, metadata: Self::Meta, ) -> Pass<Self> where Self: Sized; } @@ -53,8 +50,8 @@ pub trait ParseFunc { /// body: Option<SyntaxModel>, /// } /// -/// parse(header, body, ctx, f) { -/// let body = body!(opt: body, ctx, f); +/// parse(header, body, state, f) { +/// let body = body!(opt: body, state, f); /// let hidden = header.args.pos.get::<bool>(&mut f.problems) /// .or_missing(&mut f.problems, header.name.span, "hidden") /// .unwrap_or(false); @@ -112,7 +109,7 @@ macro_rules! function { (@parse($name:ident, $meta:ty) parse( $header:ident, $body:ident, - $ctx:ident, + $state:ident, $feedback:ident, $metadata:ident ) $code:block $($r:tt)*) => { @@ -120,18 +117,18 @@ macro_rules! function { type Meta = $meta; fn parse( - #[allow(unused)] mut header: $crate::syntax::func::FuncHeader, - #[allow(unused)] $body: Option<$crate::syntax::span::Spanned<&str>>, - #[allow(unused)] $ctx: $crate::syntax::ParseContext, + #[allow(unused)] mut call: $crate::syntax::func::FuncCall, + #[allow(unused)] $state: &$crate::syntax::ParseState, #[allow(unused)] $metadata: Self::Meta, ) -> $crate::Pass<Self> where Self: Sized { let mut feedback = $crate::Feedback::new(); - #[allow(unused)] let $header = &mut header; + #[allow(unused)] let $header = &mut call.header; + #[allow(unused)] let $body = &mut call.body; #[allow(unused)] let $feedback = &mut feedback; let func = $code; - for arg in header.args.into_iter() { + for arg in call.header.args.into_iter() { error!(@feedback, arg.span, "unexpected argument"); } @@ -167,21 +164,20 @@ macro_rules! function { /// Parse the body of a function. /// /// - If the function does not expect a body, use `body!(nope: body, feedback)`. -/// - If the function can have a body, use `body!(opt: body, ctx, feedback, +/// - If the function can have a body, use `body!(opt: body, state, feedback, /// decos)`. /// /// # Arguments /// - The `$body` should be of type `Option<Spanned<&str>>`. -/// - The `$ctx` is the [`ParseContext`](crate::syntax::ParseContext) to use for -/// parsing. +/// - The `$state` is the parse state to use. /// - The `$feedback` should be a mutable references to a /// [`Feedback`](crate::Feedback) struct which is filled with the feedback /// information arising from parsing. #[macro_export] macro_rules! body { - (opt: $body:expr, $ctx:expr, $feedback:expr) => ({ + (opt: $body:expr, $state:expr, $feedback:expr) => ({ $body.map(|body| { - let parsed = $crate::syntax::parse(body.span.start, body.v, $ctx); + let parsed = $crate::syntax::parse(body.v, body.span.start, $state); $feedback.extend(parsed.feedback); parsed.output }) diff --git a/src/layout/model.rs b/src/layout/model.rs index 3f1c9e7a..91d21037 100644 --- a/src/layout/model.rs +++ b/src/layout/model.rs @@ -142,7 +142,7 @@ impl<'a> ModelLayouter<'a> { }).await; // Add the errors generated by the model to the error list. - self.feedback.extend_offset(model.span.start, layouted.feedback); + self.feedback.extend_offset(layouted.feedback, model.span.start); for command in layouted.output { self.execute_command(command, model.span).await; @@ -30,7 +30,7 @@ use toddle::query::{FontProvider, FontIndex, FontDescriptor}; use crate::problem::Problems; use crate::layout::MultiLayout; use crate::style::{LayoutStyle, PageStyle, TextStyle}; -use crate::syntax::{SyntaxModel, Scope, Decoration, ParseContext, parse}; +use crate::syntax::{SyntaxModel, Scope, Decoration, ParseState, parse}; use crate::syntax::span::{Position, SpanVec, offset_spans}; @@ -43,6 +43,8 @@ macro_rules! pub_use_mod { } #[macro_use] +mod macros; +#[macro_use] pub mod problem; pub mod export; #[macro_use] @@ -57,14 +59,13 @@ pub mod syntax; /// Transforms source code into typesetted layouts. /// /// A typesetter can be configured through various methods. -#[derive(Debug)] pub struct Typesetter { /// The font loader shared by all typesetting processes. loader: GlobalFontLoader, /// The base layouting style. style: LayoutStyle, - /// The standard library scope. - scope: Scope, + /// The base parser state. + parse_state: ParseState, /// Whether to render debug boxes. debug: bool, } @@ -84,7 +85,7 @@ impl Typesetter { Typesetter { loader: RefCell::new(FontLoader::new(provider)), style: LayoutStyle::default(), - scope: Scope::with_std(), + parse_state: ParseState { scope: Scope::with_std() }, debug: false, } } @@ -111,7 +112,7 @@ impl Typesetter { /// Parse source code into a syntax tree. pub fn parse(&self, src: &str) -> Pass<SyntaxModel> { - parse(Position::ZERO, src, ParseContext { scope: &self.scope }) + parse(src, Position::ZERO, &self.parse_state) } /// Layout a syntax tree and return the produced layout. @@ -204,9 +205,9 @@ impl Feedback { /// Add more feedback whose spans are local and need to be offset by an /// `offset` to be correct for this feedbacks context. - pub fn extend_offset(&mut self, offset: Position, other: Feedback) { - self.problems.extend(offset_spans(offset, other.problems)); - self.decos.extend(offset_spans(offset, other.decos)); + pub fn extend_offset(&mut self, other: Feedback, offset: Position) { + self.problems.extend(offset_spans(other.problems, offset)); + self.decos.extend(offset_spans(other.decos, offset)); } } diff --git a/src/library/mod.rs b/src/library/mod.rs index c8ca374d..e1aa4ac2 100644 --- a/src/library/mod.rs +++ b/src/library/mod.rs @@ -53,10 +53,10 @@ function! { body: Option<SyntaxModel>, } - parse(header, body, ctx, f) { + parse(header, body, state, f) { header.args.pos.items.clear(); header.args.key.pairs.clear(); - ValFunc { body: body!(opt: body, ctx, f) } + ValFunc { body: body!(opt: body, state, f) } } layout(self, ctx, f) { diff --git a/src/library/page.rs b/src/library/page.rs index 4d92ea91..1e782f8f 100644 --- a/src/library/page.rs +++ b/src/library/page.rs @@ -12,7 +12,7 @@ function! { flip: bool, } - parse(header, body, ctx, f) { + parse(header, body, state, f) { body!(nope: body, f); PageSizeFunc { paper: header.args.pos.get::<Paper>(&mut f.problems), @@ -50,7 +50,7 @@ function! { padding: PaddingMap, } - parse(header, body, ctx, f) { + parse(header, body, state, f) { body!(nope: body, f); PageMarginsFunc { padding: PaddingMap::parse(&mut f.problems, &mut header.args), diff --git a/src/library/spacing.rs b/src/library/spacing.rs index 8d9c46aa..adca20af 100644 --- a/src/library/spacing.rs +++ b/src/library/spacing.rs @@ -46,9 +46,9 @@ function! { type Meta = ContentKind; - parse(header, body, ctx, f, meta) { + parse(header, body, state, f, meta) { ContentSpacingFunc { - body: body!(opt: body, ctx, f), + body: body!(opt: body, state, f), content: meta, spacing: header.args.pos.get::<f64>(&mut f.problems) .map(|num| num as f32) @@ -84,7 +84,7 @@ function! { type Meta = Option<SpecificAxis>; - parse(header, body, ctx, f, meta) { + parse(header, body, state, f, meta) { body!(nope: body, f); SpacingFunc { spacing: if let Some(axis) = meta { diff --git a/src/macros.rs b/src/macros.rs new file mode 100644 index 00000000..8ba32650 --- /dev/null +++ b/src/macros.rs @@ -0,0 +1,25 @@ +#![allow(unused)] + +/// Unwrap the result if it is `Ok(T)` or evaluate `$or` if it is `Err(_)`. +/// This fits use cases the `?`-operator does not cover, like: +/// ``` +/// try_or!(result, continue); +/// ``` +macro_rules! try_or { + ($result:expr, $or:expr $(,)?) => { + match $result { + Ok(v) => v, + Err(_) => { $or } + } + }; +} + +/// Unwrap the option if it is `Some(T)` or evaluate `$or` if it is `None`. +macro_rules! try_opt_or { + ($option:expr, $or:expr $(,)?) => { + match $option { + Some(v) => v, + None => { $or } + } + }; +} diff --git a/src/syntax/func/mod.rs b/src/syntax/func/mod.rs index d379b407..4228488d 100644 --- a/src/syntax/func/mod.rs +++ b/src/syntax/func/mod.rs @@ -9,22 +9,23 @@ pub_use_mod!(maps); pub_use_mod!(keys); pub_use_mod!(values); +/// An invocation of a function. +#[derive(Debug, Clone, PartialEq)] +pub struct FuncCall<'s> { + pub header: FuncHeader, + /// The body as a raw string containing what's inside of the brackets. + pub body: Option<Spanned<&'s str>>, +} -/// The parsed header of a function. +/// The parsed header of a function (everything in the first set of brackets). #[derive(Debug, Clone, PartialEq)] pub struct FuncHeader { - /// The function name, that is: - /// ```typst - /// [box: w=5cm] - /// ^^^ - /// ``` pub name: Spanned<Ident>, - /// The arguments passed to the function. pub args: FuncArgs, } /// The positional and keyword arguments passed to a function. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Default, Clone, PartialEq)] pub struct FuncArgs { /// The positional arguments. pub pos: Tuple, diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs index 89d4f0fd..ece09a86 100644 --- a/src/syntax/parsing.rs +++ b/src/syntax/parsing.rs @@ -1,52 +1,45 @@ //! Parsing of source code into syntax models. -use std::iter::FromIterator; use std::str::FromStr; -use crate::{Pass, Feedback}; -use super::func::{FuncHeader, FuncArgs, FuncArg}; use super::expr::*; -use super::scope::Scope; +use super::func::{FuncCall, FuncHeader, FuncArgs, FuncArg}; use super::span::{Position, Span, Spanned}; -use super::tokens::{Token, Tokens, TokenizationMode}; use super::*; - -/// The context for parsing. -#[derive(Debug, Copy, Clone)] -pub struct ParseContext<'a> { - /// The scope containing function definitions. - pub scope: &'a Scope, +/// The state which can influence how a string of source code is parsed. +/// +/// Parsing is pure - when passed in the same state and source code, the output +/// must be the same. +pub struct ParseState { + /// The scope containing all function definitions. + pub scope: Scope, } -/// Parse source code into a syntax model. +/// Parse a string of source code. /// -/// All errors and decorations are offset by the `start` position. -pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Pass<SyntaxModel> { +/// All spans in the resulting model and feedback are offset by the given +/// `offset` position. This is used to make spans of a function body relative to +/// the start of the function as a whole as opposed to the start of the +/// function's body. +pub fn parse(src: &str, offset: Position, state: &ParseState) -> Pass<SyntaxModel> { let mut model = SyntaxModel::new(); let mut feedback = Feedback::new(); - // We always start in body mode. The header tokenization mode is only used - // in the `FuncParser`. - let mut tokens = Tokens::new(start, src, TokenizationMode::Body); - - while let Some(token) = tokens.next() { + for token in Tokens::new(src, offset, TokenMode::Body) { let span = token.span; - let node = match token.v { - Token::LineComment(_) | Token::BlockComment(_) => continue, - - // Only at least two newlines mean a _real_ newline indicating a - // paragraph break. + // Starting from two newlines counts as a paragraph break, a single + // newline not. Token::Space(newlines) => if newlines >= 2 { Node::Parbreak } else { Node::Space - }, + } Token::Function { header, body, terminated } => { - let parsed = FuncParser::new(header, body, ctx).parse(); - feedback.extend_offset(span.start, parsed.feedback); + let parsed = FuncParser::new(header, body, state).parse(); + feedback.extend_offset(parsed.feedback, span.start); if !terminated { error!(@feedback, Span::at(span.end), "expected closing bracket"); @@ -55,9 +48,9 @@ pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Pass<SyntaxModel> parsed.output } - Token::Star => Node::ToggleBolder, + Token::Star => Node::ToggleBolder, Token::Underscore => Node::ToggleItalic, - Token::Backslash => Node::Linebreak, + Token::Backslash => Node::Linebreak, Token::Raw { raw, terminated } => { if !terminated { @@ -69,103 +62,101 @@ pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Pass<SyntaxModel> Token::Text(text) => Node::Text(text.to_string()), - other => { - error!(@feedback, span, "unexpected {}", other.name()); + Token::LineComment(_) | Token::BlockComment(_) => continue, + unexpected => { + error!(@feedback, span, "unexpected {}", unexpected.name()); continue; } }; - model.add(Spanned { v: node, span: token.span }); + model.add(Spanned::new(node, span)); } Pass::new(model, feedback) } -/// Performs the function parsing. struct FuncParser<'s> { - ctx: ParseContext<'s>, - feedback: Feedback, - + state: &'s ParseState, /// ```typst /// [tokens][body] /// ^^^^^^ /// ``` tokens: Tokens<'s>, peeked: Option<Option<Spanned<Token<'s>>>>, - /// The spanned body string if there is a body. /// ```typst /// [tokens][body] /// ^^^^ /// ``` body: Option<Spanned<&'s str>>, + feedback: Feedback, } impl<'s> FuncParser<'s> { - /// Create a new function parser. fn new( header: &'s str, body: Option<Spanned<&'s str>>, - ctx: ParseContext<'s> + state: &'s ParseState, ) -> FuncParser<'s> { FuncParser { - ctx, - feedback: Feedback::new(), - tokens: Tokens::new(Position::new(0, 1), header, TokenizationMode::Header), + state, + // Start at column 1 because the opening bracket is also part of + // the function, but not part of the `header` string. + tokens: Tokens::new(header, Position::new(0, 1), TokenMode::Header), peeked: None, body, + feedback: Feedback::new(), } } - /// Do the parsing. fn parse(mut self) -> Pass<Node> { - let parsed = if let Some(header) = self.parse_func_header() { + let (parser, header) = if let Some(header) = self.parse_func_header() { let name = header.name.v.as_str(); - let (parser, deco) = match self.ctx.scope.get_parser(name) { - // A valid function. - Ok(parser) => (parser, Decoration::ValidFuncName), - - // The fallback parser was returned. Invalid function. - Err(parser) => { + let (parser, deco) = match self.state.scope.get_parser(name) { + // The function exists in the scope. + Some(parser) => (parser, Decoration::ValidFuncName), + + // The function does not exist in the scope. The parser that is + // returned here is a fallback parser which exists to make sure + // the content of the function is not totally dropped (on a best + // effort basis). + None => { error!(@self.feedback, header.name.span, "unknown function"); + let parser = self.state.scope.get_fallback_parser(); (parser, Decoration::InvalidFuncName) } }; self.feedback.decos.push(Spanned::new(deco, header.name.span)); - - parser(header, self.body, self.ctx) + (parser, header) } else { - let default = FuncHeader { - name: Spanned::new(Ident("".to_string()), Span::ZERO), + // Parse the body with the fallback parser even when the header is + // completely unparsable. + let parser = self.state.scope.get_fallback_parser(); + let header = FuncHeader { + name: Spanned::new(Ident(String::new()), Span::ZERO), args: FuncArgs::new(), }; - - // Use the fallback function such that the body is still rendered - // even if the header is completely unparsable. - self.ctx.scope.get_fallback_parser()(default, self.body, self.ctx) + (parser, header) }; - self.feedback.extend(parsed.feedback); + let call = FuncCall { header, body: self.body }; + let parsed = parser(call, self.state); + self.feedback.extend(parsed.feedback); Pass::new(Node::Model(parsed.output), self.feedback) } - /// Parse the header tokens. fn parse_func_header(&mut self) -> Option<FuncHeader> { - let start = self.pos(); - self.skip_whitespace(); - - let name = match self.parse_ident() { - Some(ident) => ident, - None => { - let other = self.eat(); - self.expected_found_or_at("identifier", other, start); - return None; - } - }; + let after_bracket = self.pos(); - self.skip_whitespace(); + self.skip_white(); + let name = try_opt_or!(self.parse_ident(), { + self.expected_found_or_at("function name", after_bracket); + return None; + }); + + self.skip_white(); let args = match self.eat().map(Spanned::value) { Some(Token::Colon) => self.parse_func_args(), Some(_) => { @@ -178,58 +169,74 @@ impl<'s> FuncParser<'s> { Some(FuncHeader { name, args }) } - /// Parse the argument list between colons and end of the header. fn parse_func_args(&mut self) -> FuncArgs { - // Parse a collection until the token is `None`, that is, the end of the - // header. - self.parse_collection(None, |p| { - // If we have an identifier we might have a keyword argument, - // otherwise its for sure a postional argument. - if let Some(ident) = p.parse_ident() { - // This could still be a named tuple - if let Some(Token::LeftParen) = p.peekv() { - let tuple = p.parse_named_tuple(ident); - return Ok(tuple.map(|t| FuncArg::Pos(Expr::NamedTuple(t)))); - } - - p.skip_whitespace(); + let mut args = FuncArgs::new(); + loop { + self.skip_white(); + if self.eof() { + break; + } - if let Some(Token::Equals) = p.peekv() { - p.eat(); - p.skip_whitespace(); + let arg = if let Some(ident) = self.parse_ident() { + self.skip_white(); - // Semantic highlighting for argument keys. - p.feedback.decos.push( - Spanned::new(Decoration::ArgumentKey, ident.span)); + // This could be a keyword argument, or a positional argument of + // type named tuple or identifier. + if self.check_eat(Token::Equals).is_some() { + self.skip_white(); - let value = p.parse_expr().ok_or(("value", None))?; + let key = ident; + self.feedback.decos.push( + Spanned::new(Decoration::ArgumentKey, key.span) + ); - // Add a keyword argument. - let span = Span::merge(ident.span, value.span); - let pair = Pair { key: ident, value }; - Ok(Spanned::new(FuncArg::Key(pair), span)) + let value = try_opt_or!(self.parse_expr(), { + self.expected("value"); + continue; + }); + + let span = Span::merge(key.span, value.span); + let arg = FuncArg::Key(Pair { key, value }); + Spanned::new(arg, span) + } else if self.check(Token::LeftParen) { + let tuple = self.parse_named_tuple(ident); + tuple.map(|tup| FuncArg::Pos(Expr::NamedTuple(tup))) } else { - // Add a positional argument because there was no equals - // sign after the identifier that could have been a key. - Ok(ident.map(|id| FuncArg::Pos(Expr::Ident(id)))) + ident.map(|id| FuncArg::Pos(Expr::Ident(id))) } } else { - // Add a positional argument because we haven't got an - // identifier that could be an argument key. - let value = p.parse_expr().ok_or(("argument", None))?; - Ok(value.map(|expr| FuncArg::Pos(expr))) + // It's a positional argument. + try_opt_or!(self.parse_expr(), { + self.expected("argument"); + continue; + }).map(|expr| FuncArg::Pos(expr)) + }; + + let behind_arg = arg.span.end; + args.add(arg); + + self.skip_white(); + if self.eof() { + break; } - }).v + + self.expect_at(Token::Comma, behind_arg); + } + args + } +} + +// Parsing expressions and values +impl FuncParser<'_> { + fn parse_ident(&mut self) -> Option<Spanned<Ident>> { + self.peek().and_then(|token| match token.v { + Token::ExprIdent(id) => self.eat_span(Ident(id.to_string())), + _ => None, + }) } - /// Parse an expression which may contain math operands. For this, this - /// method looks for operators in descending order of associativity, i.e. we - /// first drill down to find all negations, brackets and tuples, the next - /// level, we look for multiplication and division and here finally, for - /// addition and subtraction. fn parse_expr(&mut self) -> Option<Spanned<Expr>> { - let o1 = self.parse_term()?; - self.parse_binop(o1, "summand", Self::parse_expr, |token| match token { + self.parse_binops("summand", Self::parse_term, |token| match token { Token::Plus => Some(Expr::Add), Token::Hyphen => Some(Expr::Sub), _ => None, @@ -237,60 +244,57 @@ impl<'s> FuncParser<'s> { } fn parse_term(&mut self) -> Option<Spanned<Expr>> { - let o1 = self.parse_factor()?; - self.parse_binop(o1, "factor", Self::parse_term, |token| match token { + self.parse_binops("factor", Self::parse_factor, |token| match token { Token::Star => Some(Expr::Mul), Token::Slash => Some(Expr::Div), _ => None, }) } - fn parse_binop<F, G>( + /// Parse expression of the form `<operand> (<op> <operand>)*`. + fn parse_binops( &mut self, - o1: Spanned<Expr>, operand_name: &str, - parse_operand: F, - parse_op: G, - ) -> Option<Spanned<Expr>> - where - F: FnOnce(&mut Self) -> Option<Spanned<Expr>>, - G: FnOnce(Token) -> Option<fn(Box<Spanned<Expr>>, Box<Spanned<Expr>>) -> Expr>, - { - self.skip_whitespace(); - - if let Some(next) = self.peek() { - if let Some(binop) = parse_op(next.v) { + mut parse_operand: impl FnMut(&mut Self) -> Option<Spanned<Expr>>, + mut parse_op: impl FnMut(Token) -> Option< + fn(Box<Spanned<Expr>>, Box<Spanned<Expr>>) -> Expr + >, + ) -> Option<Spanned<Expr>> { + let mut left = parse_operand(self)?; + + self.skip_white(); + while let Some(token) = self.peek() { + if let Some(op) = parse_op(token.v) { self.eat(); - self.skip_whitespace(); - - if let Some(o2) = parse_operand(self) { - let span = Span::merge(o1.span, o2.span); - let expr = binop(Box::new(o1), Box::new(o2)); - return Some(Spanned::new(expr, span)); - } else { - error!( - @self.feedback, Span::merge(next.span, o1.span), - "missing right {}", operand_name, - ); + self.skip_white(); + + if let Some(right) = parse_operand(self) { + let span = Span::merge(left.span, right.span); + let v = op(Box::new(left), Box::new(right)); + left = Spanned::new(v, span); + self.skip_white(); + continue; } + + error!( + @self.feedback, Span::merge(left.span, token.span), + "missing right {}", operand_name, + ); } + break; } - Some(o1) + Some(left) } - /// Parse expressions that are of the form value or -value. fn parse_factor(&mut self) -> Option<Spanned<Expr>> { - let first = self.peek()?; - if first.v == Token::Hyphen { - self.eat(); - self.skip_whitespace(); - - if let Some(factor) = self.parse_value() { - let span = Span::merge(first.span, factor.span); - Some(Spanned::new(Expr::Neg(Box::new(factor)), span)) + if let Some(hyph) = self.check_eat(Token::Hyphen) { + self.skip_white(); + if let Some(value) = self.parse_value() { + let span = Span::merge(hyph.span, value.span); + Some(Spanned::new(Expr::Neg(Box::new(value)), span)) } else { - error!(@self.feedback, first.span, "dangling minus"); + error!(@self.feedback, hyph.span, "dangling minus"); None } } else { @@ -299,283 +303,247 @@ impl<'s> FuncParser<'s> { } fn parse_value(&mut self) -> Option<Spanned<Expr>> { - let first = self.peek()?; - macro_rules! take { - ($v:expr) => ({ self.eat(); Spanned { v: $v, span: first.span } }); - } - - Some(match first.v { - Token::ExprIdent(i) => { - let name = take!(Ident(i.to_string())); - - // This could be a named tuple or an identifier - if let Some(Token::LeftParen) = self.peekv() { - self.parse_named_tuple(name).map(|t| Expr::NamedTuple(t)) + let Spanned { v: token, span } = self.peek()?; + match token { + // This could be a named tuple or an identifier. + Token::ExprIdent(id) => { + let name = Spanned::new(Ident(id.to_string()), span); + self.eat(); + self.skip_white(); + Some(if self.check(Token::LeftParen) { + self.parse_named_tuple(name).map(|tup| Expr::NamedTuple(tup)) } else { - name.map(|i| Expr::Ident(i)) - } - }, + name.map(|id| Expr::Ident(id)) + }) + } + Token::ExprStr { string, terminated } => { if !terminated { - self.expected_at("quote", first.span.end); + self.expected_at("quote", span.end); } - - take!(Expr::Str(unescape_string(string))) + self.eat_span(Expr::Str(unescape_string(string))) } - Token::ExprNumber(n) => take!(Expr::Number(n)), - Token::ExprSize(s) => take!(Expr::Size(s)), - Token::ExprBool(b) => take!(Expr::Bool(b)), + Token::ExprNumber(n) => self.eat_span(Expr::Number(n)), + Token::ExprSize(s) => self.eat_span(Expr::Size(s)), + Token::ExprBool(b) => self.eat_span(Expr::Bool(b)), Token::ExprHex(s) => { if let Ok(color) = RgbaColor::from_str(s) { - take!(Expr::Color(color)) + self.eat_span(Expr::Color(color)) } else { - // Heal color by assuming black - error!(@self.feedback, first.span, "invalid color"); - take!(Expr::Color(RgbaColor::new_healed(0, 0, 0, 255))) + // Heal color by assuming black. + error!(@self.feedback, span, "invalid color"); + let healed = RgbaColor::new_healed(0, 0, 0, 255); + self.eat_span(Expr::Color(healed)) } }, + // This could be a tuple or a parenthesized expression. We parse as + // a tuple in any case and coerce the tuple into a value if it is + // coercable (length 1 and no trailing comma). Token::LeftParen => { - let (mut tuple, can_be_coerced) = self.parse_tuple(); - // Coerce 1-tuple into value - if can_be_coerced && tuple.v.items.len() > 0 { - tuple.v.items.pop().expect("length is at least one") + let (mut tuple, coercable) = self.parse_tuple(); + Some(if coercable { + tuple.v.items.pop().expect("tuple is coercable") } else { - tuple.map(|t| Expr::Tuple(t)) - } - }, - Token::LeftBrace => self.parse_object().map(|o| Expr::Object(o)), - - _ => return None, - }) - } - - /// Parse a tuple expression: `(<expr>, ...)`. The boolean in the return - /// values showes whether the tuple can be coerced into a single value. - fn parse_tuple(&mut self) -> (Spanned<Tuple>, bool) { - let token = self.eat(); - debug_assert_eq!(token.map(Spanned::value), Some(Token::LeftParen)); + tuple.map(|tup| Expr::Tuple(tup)) + }) + } + Token::LeftBrace => { + Some(self.parse_object().map(|obj| Expr::Object(obj))) + } - // Parse a collection until a right paren appears and complain about - // missing a `value` when an invalid token is encoutered. - self.parse_collection_comma_aware(Some(Token::RightParen), - |p| p.parse_expr().ok_or(("value", None))) + _ => None, + } } - /// Parse a tuple expression: `name(<expr>, ...)` with a given identifier. fn parse_named_tuple(&mut self, name: Spanned<Ident>) -> Spanned<NamedTuple> { let tuple = self.parse_tuple().0; let span = Span::merge(name.span, tuple.span); Spanned::new(NamedTuple::new(name, tuple), span) } - /// Parse an object expression: `{ <key>: <value>, ... }`. - fn parse_object(&mut self) -> Spanned<Object> { - let token = self.eat(); - debug_assert_eq!(token.map(Spanned::value), Some(Token::LeftBrace)); - - // Parse a collection until a right brace appears. - self.parse_collection(Some(Token::RightBrace), |p| { - // Expect an identifier as the key. - let key = p.parse_ident().ok_or(("key", None))?; - - // Expect a colon behind the key (only separated by whitespace). - let behind_key = p.pos(); - p.skip_whitespace(); - if p.peekv() != Some(Token::Colon) { - return Err(("colon", Some(behind_key))); + /// The boolean tells you whether the tuple can be coerced into a value + /// (this is the case when it's length 1 and has no trailing comma). + fn parse_tuple(&mut self) -> (Spanned<Tuple>, bool) { + let start = self.pos(); + self.assert(Token::LeftParen); + + let mut tuple = Tuple::new(); + let mut commaless = true; + loop { + self.skip_white(); + if self.eof() || self.check(Token::RightParen) { + break; } - p.eat(); - p.skip_whitespace(); + let expr = try_opt_or!(self.parse_expr(), { + self.expected("value"); + continue; + }); - // Semantic highlighting for object keys. - p.feedback.decos.push( - Spanned::new(Decoration::ObjectKey, key.span)); + let behind_expr = expr.span.end; + tuple.add(expr); - let value = p.parse_expr().ok_or(("value", None))?; + self.skip_white(); + if self.eof() || self.check(Token::RightParen) { + break; + } - let span = Span::merge(key.span, value.span); - Ok(Spanned::new(Pair { key, value }, span)) - }) - } + self.expect_at(Token::Comma, behind_expr); + commaless = false; + } - /// Parse a comma-separated collection where each item is parsed through - /// `parse_item` until the `end` token is met. - fn parse_collection<C, I, F>( - &mut self, - end: Option<Token>, - parse_item: F - ) -> Spanned<C> - where - C: FromIterator<Spanned<I>>, - F: FnMut(&mut Self) -> Result<Spanned<I>, (&'static str, Option<Position>)>, - { - self.parse_collection_comma_aware(end, parse_item).0 + self.expect(Token::RightParen); + let end = self.pos(); + let coercable = commaless && !tuple.items.is_empty(); + + (Spanned::new(tuple, Span::new(start, end)), coercable) } - /// Parse a comma-separated collection where each item is parsed through - /// `parse_item` until the `end` token is met. The first item in the return - /// tuple is the collection, the second item indicates whether the - /// collection can be coerced into a single item (i.e. no comma appeared). - fn parse_collection_comma_aware<C, I, F>( - &mut self, - end: Option<Token>, - mut parse_item: F - ) -> (Spanned<C>, bool) - where - C: FromIterator<Spanned<I>>, - F: FnMut(&mut Self) -> Result<Spanned<I>, (&'static str, Option<Position>)>, - { + fn parse_object(&mut self) -> Spanned<Object> { let start = self.pos(); - let mut can_be_coerced = true; + self.assert(Token::LeftBrace); - // Parse the comma separated items. - let collection = std::iter::from_fn(|| { - self.skip_whitespace(); - let peeked = self.peekv(); - - // We finished as expected. - if peeked == end { - self.eat(); - return None; + let mut object = Object::new(); + loop { + self.skip_white(); + if self.eof() || self.check(Token::RightBrace) { + break; } - // We finished without the expected end token (which has to be a - // `Some` value at this point since otherwise we would have already - // returned in the previous case). - if peeked == None { - self.eat(); - self.expected_at(end.unwrap().name(), self.pos()); - return None; + let key = try_opt_or!(self.parse_ident(), { + self.expected("key"); + continue; + }); + + let after_key = self.pos(); + self.skip_white(); + if !self.expect_at(Token::Colon, after_key) { + continue; } - // Try to parse a collection item. - match parse_item(self) { - Ok(item) => { - // Expect a comma behind the item (only separated by - // whitespace). - self.skip_whitespace(); - match self.peekv() { - Some(Token::Comma) => { - can_be_coerced = false; - self.eat(); - } - t @ Some(_) if t != end => { - can_be_coerced = false; - self.expected_at("comma", item.span.end); - }, - _ => {} - } - - return Some(Some(item)); - } + self.feedback.decos.push( + Spanned::new(Decoration::ObjectKey, key.span) + ); - // The item parser expected something different at either some - // given position or instead of the currently peekable token. - Err((expected, Some(pos))) => self.expected_at(expected, pos), - Err((expected, None)) => { - let token = self.peek(); - if token.map(Spanned::value) != end { - self.eat(); - } - self.expected_found_or_at(expected, token, self.pos()); - } + self.skip_white(); + let value = try_opt_or!(self.parse_expr(), { + self.expected("value"); + continue; + }); + + let behind_value = value.span.end; + let span = Span::merge(key.span, value.span); + object.add(Spanned::new(Pair { key, value }, span)); + + self.skip_white(); + if self.eof() || self.check(Token::RightBrace) { + break; } - Some(None) - }).filter_map(|x| x).collect(); + self.expect_at(Token::Comma, behind_value); + } + self.expect(Token::RightBrace); let end = self.pos(); - (Spanned::new(collection, Span { start, end }), can_be_coerced) + + Spanned::new(object, Span::new(start, end)) } +} - /// Try to parse an identifier and do nothing if the peekable token is no - /// identifier. - fn parse_ident(&mut self) -> Option<Spanned<Ident>> { - match self.peek() { - Some(Spanned { v: Token::ExprIdent(s), span }) => { - self.eat(); - Some(Spanned { v: Ident(s.to_string()), span }) - } - _ => None +// Error handling +impl FuncParser<'_> { + fn expect(&mut self, token: Token<'_>) -> bool { + if self.check(token) { + self.eat(); + true + } else { + self.expected(token.name()); + false } } - /// Skip all whitespace/comment tokens. - fn skip_whitespace(&mut self) { - self.eat_until(|t| match t { - Token::Space(_) | Token::LineComment(_) | - Token::BlockComment(_) => false, - _ => true, - }, false) + fn expect_at(&mut self, token: Token<'_>, pos: Position) -> bool { + if self.check(token) { + self.eat(); + true + } else { + self.expected_at(token.name(), pos); + false + } } - /// Add an error about an expected `thing` which was not found, showing - /// what was found instead. - fn expected_found(&mut self, thing: &str, found: Spanned<Token>) { - error!( - @self.feedback, found.span, - "expected {}, found {}", thing, found.v.name(), - ); + fn expected(&mut self, thing: &str) { + if let Some(found) = self.eat() { + error!( + @self.feedback, found.span, + "expected {}, found {}", thing, found.v.name(), + ); + } else { + error!(@self.feedback, Span::at(self.pos()), "expected {}", thing); + } } - /// Add an error about an `thing` which was expected but not found at the - /// given position. fn expected_at(&mut self, thing: &str, pos: Position) { error!(@self.feedback, Span::at(pos), "expected {}", thing); } - /// Add a expected-found-error if `found` is `Some` and an expected-error - /// otherwise. - fn expected_found_or_at( - &mut self, - thing: &str, - found: Option<Spanned<Token>>, - pos: Position - ) { - match found { - Some(found) => self.expected_found(thing, found), - None => self.expected_at(thing, pos), + fn expected_found_or_at(&mut self, thing: &str, pos: Position) { + if self.eof() { + self.expected_at(thing, pos) + } else { + self.expected(thing); } } +} - /// Consume tokens until the function returns true and only consume the last - /// token if instructed to so by `eat_match`. - fn eat_until<F>(&mut self, mut f: F, eat_match: bool) - where F: FnMut(Token<'s>) -> bool { - while let Some(token) = self.peek() { - if f(token.v) { - if eat_match { - self.eat(); - } - break; +// Parsing primitives +impl<'s> FuncParser<'s> { + fn skip_white(&mut self) { + loop { + match self.peek().map(Spanned::value) { + Some(Token::Space(_)) + | Some(Token::LineComment(_)) + | Some(Token::BlockComment(_)) => { self.eat(); } + _ => break, } - - self.eat(); } } - /// Consume and return the next token. fn eat(&mut self) -> Option<Spanned<Token<'s>>> { - self.peeked.take() - .unwrap_or_else(|| self.tokens.next()) + self.peeked.take().unwrap_or_else(|| self.tokens.next()) + } + + fn eat_span<T>(&mut self, v: T) -> Option<Spanned<T>> { + self.eat().map(|spanned| spanned.map(|_| v)) } - /// Peek at the next token without consuming it. fn peek(&mut self) -> Option<Spanned<Token<'s>>> { - let iter = &mut self.tokens; - *self.peeked.get_or_insert_with(|| iter.next()) + let tokens = &mut self.tokens; + *self.peeked.get_or_insert_with(|| tokens.next()) + } + + fn assert(&mut self, token: Token<'_>) { + assert!(self.check_eat(token).is_some()); } - /// Peek at the unspanned value of the next token. - fn peekv(&mut self) -> Option<Token<'s>> { - self.peek().map(Spanned::value) + fn check(&mut self, token: Token<'_>) -> bool { + self.peek().map(Spanned::value) == Some(token) + } + + fn check_eat(&mut self, token: Token<'_>) -> Option<Spanned<Token<'s>>> { + if self.check(token) { + self.eat() + } else { + None + } + } + + fn eof(&mut self) -> bool { + self.peek().is_none() } - /// The position at the end of the last eaten token / start of the peekable - /// token. fn pos(&self) -> Position { self.peeked.flatten() .map(|s| s.span.start) @@ -583,64 +551,62 @@ impl<'s> FuncParser<'s> { } } -/// Unescape a string: `the string is \"this\"` => `the string is "this"`. fn unescape_string(string: &str) -> String { - let mut s = String::with_capacity(string.len()); let mut iter = string.chars(); + let mut out = String::with_capacity(string.len()); while let Some(c) = iter.next() { if c == '\\' { match iter.next() { - Some('\\') => s.push('\\'), - Some('"') => s.push('"'), - Some('n') => s.push('\n'), - Some('t') => s.push('\t'), - Some(c) => { s.push('\\'); s.push(c); } - None => s.push('\\'), + Some('\\') => out.push('\\'), + Some('"') => out.push('"'), + Some('n') => out.push('\n'), + Some('t') => out.push('\t'), + Some(c) => { out.push('\\'); out.push(c); } + None => out.push('\\'), } } else { - s.push(c); + out.push(c); } } - s + out } -/// Unescape raw markup into lines. +/// Unescape raw markup and split it into into lines. fn unescape_raw(raw: &str) -> Vec<String> { - let mut lines = Vec::new(); - let mut s = String::new(); let mut iter = raw.chars().peekable(); + let mut line = String::new(); + let mut lines = Vec::new(); while let Some(c) = iter.next() { if c == '\\' { match iter.next() { - Some('`') => s.push('`'), - Some(c) => { s.push('\\'); s.push(c); } - None => s.push('\\'), + Some('`') => line.push('`'), + Some(c) => { line.push('\\'); line.push(c); } + None => line.push('\\'), } } else if is_newline_char(c) { if c == '\r' && iter.peek() == Some(&'\n') { iter.next(); } - lines.push(std::mem::replace(&mut s, String::new())); + lines.push(std::mem::take(&mut line)); } else { - s.push(c); + line.push(c); } } - lines.push(s); + lines.push(line); lines } - #[cfg(test)] #[allow(non_snake_case)] mod tests { use crate::size::Size; - use crate::syntax::test::{DebugFn, check}; - use crate::syntax::func::Value; + use super::super::test::{check, DebugFn}; + use super::super::func::Value; use super::*; use Decoration::*; @@ -666,8 +632,8 @@ mod tests { scope.add::<DebugFn>("box"); scope.add::<DebugFn>("val"); - let ctx = ParseContext { scope: &scope }; - let pass = parse(Position::ZERO, $source, ctx); + let state = ParseState { scope }; + let pass = parse($source, Position::ZERO, &state); // Test model. let (exp, cmp) = span_vec![$($model)*]; @@ -839,14 +805,14 @@ mod tests { fn parse_function_names() { // No closing bracket. p!("[" => [func!("")], [ - (0:1, 0:1, "expected identifier"), + (0:1, 0:1, "expected function name"), (0:1, 0:1, "expected closing bracket") ]); // No name. - p!("[]" => [func!("")], [(0:1, 0:1, "expected identifier")]); + p!("[]" => [func!("")], [(0:1, 0:1, "expected function name")]); p!("[\"]" => [func!("")], [ - (0:1, 0:3, "expected identifier, found string"), + (0:1, 0:3, "expected function name, found string"), (0:3, 0:3, "expected closing bracket"), ]); @@ -862,9 +828,9 @@ mod tests { p!("[ f]" => [func!("f")], [], [(0:3, 0:4, ValidFuncName)]); // An invalid token for a name. - p!("[12]" => [func!("")], [(0:1, 0:3, "expected identifier, found number")], []); - p!("[🌎]" => [func!("")], [(0:1, 0:2, "expected identifier, found invalid token")], []); - p!("[ 🌎]" => [func!("")], [(0:3, 0:4, "expected identifier, found invalid token")], []); + p!("[12]" => [func!("")], [(0:1, 0:3, "expected function name, found number")], []); + p!("[🌎]" => [func!("")], [(0:1, 0:2, "expected function name, found invalid token")], []); + p!("[ 🌎]" => [func!("")], [(0:3, 0:4, "expected function name, found invalid token")], []); } #[test] @@ -954,6 +920,11 @@ mod tests { Num(2.0) ))); + // Associativity of multiplication and division. + p!("[val: 3/4*5]" => + [func!("val": (Mul(Div(Num(3.0), Num(4.0)), Num(5.0))), {})] + ); + // Invalid expressions. p!("[val: 4pt--]" => [func!("val": (Pt(4.0)))], [ (0:10, 0:11, "dangling minus"), @@ -971,6 +942,12 @@ mod tests { pval!("()" => (tuple!())); pval!("empty()" => (named_tuple!("empty"))); + // Space between name and tuple. + pval!("add ( 1 , 2 )" => (named_tuple!("add", Num(1.0), Num(2.0)))); + pval!("num = add ( 1 , 2 )" => (), { + "num" => named_tuple!("add", Num(1.0), Num(2.0)) + }); + // Invalid value. p!("[val: sound(\x07)]" => [func!("val": (named_tuple!("sound")), {})], diff --git a/src/syntax/scope.rs b/src/syntax/scope.rs index f7b20b9f..74c64280 100644 --- a/src/syntax/scope.rs +++ b/src/syntax/scope.rs @@ -5,12 +5,10 @@ use std::fmt::{self, Debug, Formatter}; use crate::Pass; use crate::func::ParseFunc; -use super::func::FuncHeader; -use super::parsing::ParseContext; -use super::span::Spanned; +use super::func::FuncCall; +use super::parsing::ParseState; use super::Model; - /// A map from identifiers to function parsers. pub struct Scope { parsers: HashMap<String, Box<Parser>>, @@ -50,10 +48,8 @@ impl Scope { } /// Return the parser with the given name if there is one. - pub fn get_parser(&self, name: &str) -> Result<&Parser, &Parser> { - self.parsers.get(name) - .map(|x| &**x) - .ok_or_else(|| &*self.fallback) + pub fn get_parser(&self, name: &str) -> Option<&Parser> { + self.parsers.get(name).map(AsRef::as_ref) } /// Return the fallback parser. @@ -72,16 +68,12 @@ impl Debug for Scope { /// A function which parses the source of a function into a model type which /// implements [`Model`]. -type Parser = dyn Fn( - FuncHeader, - Option<Spanned<&str>>, - ParseContext, -) -> Pass<Box<dyn Model>>; +type Parser = dyn Fn(FuncCall, &ParseState) -> Pass<Box<dyn Model>>; fn parser<F>(metadata: <F as ParseFunc>::Meta) -> Box<Parser> where F: ParseFunc + Model + 'static { - Box::new(move |h, b, c| { - F::parse(h, b, c, metadata.clone()) + Box::new(move |f, s| { + F::parse(f, s, metadata.clone()) .map(|model| Box::new(model) as Box<dyn Model>) }) } diff --git a/src/syntax/span.rs b/src/syntax/span.rs index 8083bc06..c8e2cddb 100644 --- a/src/syntax/span.rs +++ b/src/syntax/span.rs @@ -4,14 +4,14 @@ use std::fmt::{self, Debug, Formatter}; use std::ops::{Add, Sub}; use serde::Serialize; -/// A vector of spanned things. +/// A vector of spanned values of type `T`. pub type SpanVec<T> = Vec<Spanned<T>>; /// [Offset](Span::offset) all spans in a vector of spanned things by a start /// position. pub fn offset_spans<T>( - start: Position, vec: SpanVec<T>, + start: Position, ) -> impl Iterator<Item=Spanned<T>> { vec.into_iter().map(move |s| s.map_span(|span| span.offset(start))) } @@ -95,6 +95,11 @@ impl Span { Span { start: pos, end: pos } } + /// Expand a span by merging it with another span. + pub fn expand(&mut self, other: Span) { + *self = Span::merge(*self, other) + } + /// Offset a span by a start position. /// /// This is, for example, used to translate error spans from function local @@ -126,7 +131,7 @@ impl Position { /// The line 0, column 0 position. pub const ZERO: Position = Position { line: 0, column: 0 }; - /// Crete a new instance from line and column. + /// Create a new position from line and column. pub fn new(line: usize, column: usize) -> Position { Position { line, column } } diff --git a/src/syntax/test.rs b/src/syntax/test.rs index aaad5c12..7b1e0830 100644 --- a/src/syntax/test.rs +++ b/src/syntax/test.rs @@ -63,13 +63,13 @@ function! { pub body: Option<SyntaxModel>, } - parse(header, body, ctx, f) { + parse(header, body, state, f) { let cloned = header.clone(); header.args.pos.items.clear(); header.args.key.pairs.clear(); DebugFn { header: cloned, - body: body!(opt: body, ctx, f), + body: body!(opt: body, state, f), } } diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs index b1bf76e3..10200708 100644 --- a/src/syntax/tokens.rs +++ b/src/syntax/tokens.rs @@ -5,9 +5,8 @@ use unicode_xid::UnicodeXID; use crate::size::Size; use super::span::{Position, Span, Spanned}; -use self::Token::*; -use self::TokenizationMode::*; - +use Token::*; +use TokenMode::*; /// A minimal semantic entity of source code. #[derive(Debug, Copy, Clone, PartialEq)] @@ -152,7 +151,7 @@ impl<'s> Token<'s> { #[derive(Debug)] pub struct Tokens<'s> { src: &'s str, - mode: TokenizationMode, + mode: TokenMode, iter: Peekable<Chars<'s>>, position: Position, index: usize, @@ -163,20 +162,22 @@ pub struct Tokens<'s> { /// backtick tokens. #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] #[allow(missing_docs)] -pub enum TokenizationMode { +pub enum TokenMode { Header, Body, } impl<'s> Tokens<'s> { - /// Create a new token iterator with the given mode where the first token - /// span starts an the given `start` position. - pub fn new(start: Position, src: &'s str, mode: TokenizationMode) -> Tokens<'s> { + /// Create a new token iterator with the given mode. + /// + /// The first token's span starts an the given `offset` position instead of + /// the zero position. + pub fn new(src: &'s str, offset: Position, mode: TokenMode) -> Tokens<'s> { Tokens { src, mode, iter: src.chars().peekable(), - position: start, + position: offset, index: 0, } } @@ -341,7 +342,7 @@ impl<'s> Tokens<'s> { Function { header, body: Some(Spanned { v: body, span }), terminated } } - fn read_function_part(&mut self, mode: TokenizationMode) -> (&'s str, bool) { + fn read_function_part(&mut self, mode: TokenMode) -> (&'s str, bool) { let start = self.index(); let mut terminated = false; @@ -530,7 +531,6 @@ pub fn is_identifier(string: &str) -> bool { true } - #[cfg(test)] #[allow(non_snake_case)] mod tests { @@ -557,7 +557,7 @@ mod tests { macro_rules! t { ($mode:expr, $source:expr => [$($tokens:tt)*]) => { let (exp, spans) = span_vec![$($tokens)*]; - let found = Tokens::new(Position::ZERO, $source, $mode).collect::<Vec<_>>(); + let found = Tokens::new($source, Position::ZERO, $mode).collect::<Vec<_>>(); check($source, exp, found, spans); } } |