summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorLaurenz <laurmaedje@gmail.com>2020-07-29 17:46:57 +0200
committerGitHub <noreply@github.com>2020-07-29 17:46:57 +0200
commitf34ba3dcda182d9b9c14cc94fdb48810bf18bef0 (patch)
tree667a7aba2f26996c7ada8ce85952c384a1dbd5a1 /src
parente7ffdde43d09f60238590723c2829554806e23d5 (diff)
parent9672d4320052d08b67d497febed4a0ad78bf9252 (diff)
Merge pull request #7 from typst/parser-update
Parser update
Diffstat (limited to 'src')
-rw-r--r--src/func.rs36
-rw-r--r--src/layout/model.rs2
-rw-r--r--src/lib.rs21
-rw-r--r--src/library/mod.rs4
-rw-r--r--src/library/page.rs4
-rw-r--r--src/library/spacing.rs6
-rw-r--r--src/macros.rs25
-rw-r--r--src/syntax/expr.rs6
-rw-r--r--src/syntax/func/mod.rs17
-rw-r--r--src/syntax/parsing.rs1162
-rw-r--r--src/syntax/scope.rs22
-rw-r--r--src/syntax/span.rs178
-rw-r--r--src/syntax/test.rs68
-rw-r--r--src/syntax/tokens.rs134
14 files changed, 817 insertions, 868 deletions
diff --git a/src/func.rs b/src/func.rs
index 777e0b26..e948d049 100644
--- a/src/func.rs
+++ b/src/func.rs
@@ -1,9 +1,8 @@
//! Trait and prelude for custom functions.
use crate::Pass;
-use crate::syntax::ParseContext;
-use crate::syntax::func::FuncHeader;
-use crate::syntax::span::Spanned;
+use crate::syntax::ParseState;
+use crate::syntax::func::FuncCall;
/// Types that are useful for creating your own functions.
pub mod prelude {
@@ -17,7 +16,6 @@ pub mod prelude {
pub use crate::syntax::span::{Span, Spanned};
}
-
/// Parse a function from source code.
pub trait ParseFunc {
/// A metadata type whose value is passed into the function parser. This
@@ -33,9 +31,8 @@ pub trait ParseFunc {
/// Parse the header and body into this function given a context.
fn parse(
- header: FuncHeader,
- body: Option<Spanned<&str>>,
- ctx: ParseContext,
+ header: FuncCall,
+ state: &ParseState,
metadata: Self::Meta,
) -> Pass<Self> where Self: Sized;
}
@@ -53,8 +50,8 @@ pub trait ParseFunc {
/// body: Option<SyntaxModel>,
/// }
///
-/// parse(header, body, ctx, f) {
-/// let body = body!(opt: body, ctx, f);
+/// parse(header, body, state, f) {
+/// let body = body!(opt: body, state, f);
/// let hidden = header.args.pos.get::<bool>(&mut f.problems)
/// .or_missing(&mut f.problems, header.name.span, "hidden")
/// .unwrap_or(false);
@@ -112,7 +109,7 @@ macro_rules! function {
(@parse($name:ident, $meta:ty) parse(
$header:ident,
$body:ident,
- $ctx:ident,
+ $state:ident,
$feedback:ident,
$metadata:ident
) $code:block $($r:tt)*) => {
@@ -120,18 +117,18 @@ macro_rules! function {
type Meta = $meta;
fn parse(
- #[allow(unused)] mut header: $crate::syntax::func::FuncHeader,
- #[allow(unused)] $body: Option<$crate::syntax::span::Spanned<&str>>,
- #[allow(unused)] $ctx: $crate::syntax::ParseContext,
+ #[allow(unused)] mut call: $crate::syntax::func::FuncCall,
+ #[allow(unused)] $state: &$crate::syntax::ParseState,
#[allow(unused)] $metadata: Self::Meta,
) -> $crate::Pass<Self> where Self: Sized {
let mut feedback = $crate::Feedback::new();
- #[allow(unused)] let $header = &mut header;
+ #[allow(unused)] let $header = &mut call.header;
+ #[allow(unused)] let $body = &mut call.body;
#[allow(unused)] let $feedback = &mut feedback;
let func = $code;
- for arg in header.args.into_iter() {
+ for arg in call.header.args.into_iter() {
error!(@feedback, arg.span, "unexpected argument");
}
@@ -167,21 +164,20 @@ macro_rules! function {
/// Parse the body of a function.
///
/// - If the function does not expect a body, use `body!(nope: body, feedback)`.
-/// - If the function can have a body, use `body!(opt: body, ctx, feedback,
+/// - If the function can have a body, use `body!(opt: body, state, feedback,
/// decos)`.
///
/// # Arguments
/// - The `$body` should be of type `Option<Spanned<&str>>`.
-/// - The `$ctx` is the [`ParseContext`](crate::syntax::ParseContext) to use for
-/// parsing.
+/// - The `$state` is the parse state to use.
/// - The `$feedback` should be a mutable references to a
/// [`Feedback`](crate::Feedback) struct which is filled with the feedback
/// information arising from parsing.
#[macro_export]
macro_rules! body {
- (opt: $body:expr, $ctx:expr, $feedback:expr) => ({
+ (opt: $body:expr, $state:expr, $feedback:expr) => ({
$body.map(|body| {
- let parsed = $crate::syntax::parse(body.span.start, body.v, $ctx);
+ let parsed = $crate::syntax::parse(body.v, body.span.start, $state);
$feedback.extend(parsed.feedback);
parsed.output
})
diff --git a/src/layout/model.rs b/src/layout/model.rs
index 3f1c9e7a..91d21037 100644
--- a/src/layout/model.rs
+++ b/src/layout/model.rs
@@ -142,7 +142,7 @@ impl<'a> ModelLayouter<'a> {
}).await;
// Add the errors generated by the model to the error list.
- self.feedback.extend_offset(model.span.start, layouted.feedback);
+ self.feedback.extend_offset(layouted.feedback, model.span.start);
for command in layouted.output {
self.execute_command(command, model.span).await;
diff --git a/src/lib.rs b/src/lib.rs
index b140f47d..8f5bbdd6 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -30,7 +30,7 @@ use toddle::query::{FontProvider, FontIndex, FontDescriptor};
use crate::problem::Problems;
use crate::layout::MultiLayout;
use crate::style::{LayoutStyle, PageStyle, TextStyle};
-use crate::syntax::{SyntaxModel, Scope, Decoration, ParseContext, parse};
+use crate::syntax::{SyntaxModel, Scope, Decoration, ParseState, parse};
use crate::syntax::span::{Position, SpanVec, offset_spans};
@@ -43,6 +43,8 @@ macro_rules! pub_use_mod {
}
#[macro_use]
+mod macros;
+#[macro_use]
pub mod problem;
pub mod export;
#[macro_use]
@@ -57,14 +59,13 @@ pub mod syntax;
/// Transforms source code into typesetted layouts.
///
/// A typesetter can be configured through various methods.
-#[derive(Debug)]
pub struct Typesetter {
/// The font loader shared by all typesetting processes.
loader: GlobalFontLoader,
/// The base layouting style.
style: LayoutStyle,
- /// The standard library scope.
- scope: Scope,
+ /// The base parser state.
+ parse_state: ParseState,
/// Whether to render debug boxes.
debug: bool,
}
@@ -84,7 +85,7 @@ impl Typesetter {
Typesetter {
loader: RefCell::new(FontLoader::new(provider)),
style: LayoutStyle::default(),
- scope: Scope::with_std(),
+ parse_state: ParseState { scope: Scope::with_std() },
debug: false,
}
}
@@ -111,7 +112,7 @@ impl Typesetter {
/// Parse source code into a syntax tree.
pub fn parse(&self, src: &str) -> Pass<SyntaxModel> {
- parse(Position::ZERO, src, ParseContext { scope: &self.scope })
+ parse(src, Position::ZERO, &self.parse_state)
}
/// Layout a syntax tree and return the produced layout.
@@ -203,10 +204,10 @@ impl Feedback {
}
/// Add more feedback whose spans are local and need to be offset by an
- /// `offset` to be correct for this feedbacks context.
- pub fn extend_offset(&mut self, offset: Position, other: Feedback) {
- self.problems.extend(offset_spans(offset, other.problems));
- self.decos.extend(offset_spans(offset, other.decos));
+ /// `offset` to be correct in this feedback's context.
+ pub fn extend_offset(&mut self, more: Feedback, offset: Position) {
+ self.problems.extend(offset_spans(more.problems, offset));
+ self.decos.extend(offset_spans(more.decos, offset));
}
}
diff --git a/src/library/mod.rs b/src/library/mod.rs
index c8ca374d..e1aa4ac2 100644
--- a/src/library/mod.rs
+++ b/src/library/mod.rs
@@ -53,10 +53,10 @@ function! {
body: Option<SyntaxModel>,
}
- parse(header, body, ctx, f) {
+ parse(header, body, state, f) {
header.args.pos.items.clear();
header.args.key.pairs.clear();
- ValFunc { body: body!(opt: body, ctx, f) }
+ ValFunc { body: body!(opt: body, state, f) }
}
layout(self, ctx, f) {
diff --git a/src/library/page.rs b/src/library/page.rs
index 4d92ea91..1e782f8f 100644
--- a/src/library/page.rs
+++ b/src/library/page.rs
@@ -12,7 +12,7 @@ function! {
flip: bool,
}
- parse(header, body, ctx, f) {
+ parse(header, body, state, f) {
body!(nope: body, f);
PageSizeFunc {
paper: header.args.pos.get::<Paper>(&mut f.problems),
@@ -50,7 +50,7 @@ function! {
padding: PaddingMap,
}
- parse(header, body, ctx, f) {
+ parse(header, body, state, f) {
body!(nope: body, f);
PageMarginsFunc {
padding: PaddingMap::parse(&mut f.problems, &mut header.args),
diff --git a/src/library/spacing.rs b/src/library/spacing.rs
index 8d9c46aa..adca20af 100644
--- a/src/library/spacing.rs
+++ b/src/library/spacing.rs
@@ -46,9 +46,9 @@ function! {
type Meta = ContentKind;
- parse(header, body, ctx, f, meta) {
+ parse(header, body, state, f, meta) {
ContentSpacingFunc {
- body: body!(opt: body, ctx, f),
+ body: body!(opt: body, state, f),
content: meta,
spacing: header.args.pos.get::<f64>(&mut f.problems)
.map(|num| num as f32)
@@ -84,7 +84,7 @@ function! {
type Meta = Option<SpecificAxis>;
- parse(header, body, ctx, f, meta) {
+ parse(header, body, state, f, meta) {
body!(nope: body, f);
SpacingFunc {
spacing: if let Some(axis) = meta {
diff --git a/src/macros.rs b/src/macros.rs
new file mode 100644
index 00000000..8ba32650
--- /dev/null
+++ b/src/macros.rs
@@ -0,0 +1,25 @@
+#![allow(unused)]
+
+/// Unwrap the result if it is `Ok(T)` or evaluate `$or` if it is `Err(_)`.
+/// This fits use cases the `?`-operator does not cover, like:
+/// ```
+/// try_or!(result, continue);
+/// ```
+macro_rules! try_or {
+ ($result:expr, $or:expr $(,)?) => {
+ match $result {
+ Ok(v) => v,
+ Err(_) => { $or }
+ }
+ };
+}
+
+/// Unwrap the option if it is `Some(T)` or evaluate `$or` if it is `None`.
+macro_rules! try_opt_or {
+ ($option:expr, $or:expr $(,)?) => {
+ match $option {
+ Some(v) => v,
+ None => { $or }
+ }
+ };
+}
diff --git a/src/syntax/expr.rs b/src/syntax/expr.rs
index a27bdb62..a1b3fd62 100644
--- a/src/syntax/expr.rs
+++ b/src/syntax/expr.rs
@@ -93,9 +93,6 @@ impl Debug for Expr {
/// A unicode identifier.
///
-/// The identifier must be valid! This is checked in [`Ident::new`] or
-/// [`is_identifier`].
-///
/// # Example
/// ```typst
/// [func: "hi", ident]
@@ -105,7 +102,8 @@ impl Debug for Expr {
pub struct Ident(pub String);
impl Ident {
- /// Create a new identifier from a string checking that it is valid.
+ /// Create a new identifier from a string checking that it is a valid
+ /// unicode identifier.
pub fn new<S>(ident: S) -> Option<Ident> where S: AsRef<str> + Into<String> {
if is_identifier(ident.as_ref()) {
Some(Ident(ident.into()))
diff --git a/src/syntax/func/mod.rs b/src/syntax/func/mod.rs
index d379b407..4228488d 100644
--- a/src/syntax/func/mod.rs
+++ b/src/syntax/func/mod.rs
@@ -9,22 +9,23 @@ pub_use_mod!(maps);
pub_use_mod!(keys);
pub_use_mod!(values);
+/// An invocation of a function.
+#[derive(Debug, Clone, PartialEq)]
+pub struct FuncCall<'s> {
+ pub header: FuncHeader,
+ /// The body as a raw string containing what's inside of the brackets.
+ pub body: Option<Spanned<&'s str>>,
+}
-/// The parsed header of a function.
+/// The parsed header of a function (everything in the first set of brackets).
#[derive(Debug, Clone, PartialEq)]
pub struct FuncHeader {
- /// The function name, that is:
- /// ```typst
- /// [box: w=5cm]
- /// ^^^
- /// ```
pub name: Spanned<Ident>,
- /// The arguments passed to the function.
pub args: FuncArgs,
}
/// The positional and keyword arguments passed to a function.
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Default, Clone, PartialEq)]
pub struct FuncArgs {
/// The positional arguments.
pub pos: Tuple,
diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs
index 09405d7f..a0d9c4e4 100644
--- a/src/syntax/parsing.rs
+++ b/src/syntax/parsing.rs
@@ -1,52 +1,45 @@
//! Parsing of source code into syntax models.
-use std::iter::FromIterator;
use std::str::FromStr;
-use crate::{Pass, Feedback};
-use super::func::{FuncHeader, FuncArgs, FuncArg};
use super::expr::*;
-use super::scope::Scope;
+use super::func::{FuncCall, FuncHeader, FuncArgs, FuncArg};
use super::span::{Position, Span, Spanned};
-use super::tokens::{Token, Tokens, TokenizationMode};
use super::*;
-
-/// The context for parsing.
-#[derive(Debug, Copy, Clone)]
-pub struct ParseContext<'a> {
- /// The scope containing function definitions.
- pub scope: &'a Scope,
+/// The state which can influence how a string of source code is parsed.
+///
+/// Parsing is pure - when passed in the same state and source code, the output
+/// must be the same.
+pub struct ParseState {
+ /// The scope containing all function definitions.
+ pub scope: Scope,
}
-/// Parse source code into a syntax model.
+/// Parse a string of source code.
///
-/// All errors and decorations are offset by the `start` position.
-pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Pass<SyntaxModel> {
+/// All spans in the resulting model and feedback are offset by the given
+/// `offset` position. This is used to make spans of a function body relative to
+/// the start of the function as a whole as opposed to the start of the
+/// function's body.
+pub fn parse(src: &str, offset: Position, state: &ParseState) -> Pass<SyntaxModel> {
let mut model = SyntaxModel::new();
let mut feedback = Feedback::new();
- // We always start in body mode. The header tokenization mode is only used
- // in the `FuncParser`.
- let mut tokens = Tokens::new(start, src, TokenizationMode::Body);
-
- while let Some(token) = tokens.next() {
+ for token in Tokens::new(src, offset, TokenMode::Body) {
let span = token.span;
-
let node = match token.v {
- Token::LineComment(_) | Token::BlockComment(_) => continue,
-
- // Only at least two newlines mean a _real_ newline indicating a
- // paragraph break.
+ // Starting from two newlines counts as a paragraph break, a single
+ // newline does not.
Token::Space(newlines) => if newlines >= 2 {
Node::Parbreak
} else {
Node::Space
- },
+ }
Token::Function { header, body, terminated } => {
- let parsed = FuncParser::new(header, body, ctx).parse();
- feedback.extend_offset(span.start, parsed.feedback);
+ let parsed = FuncParser::new(header, body, state).parse();
+ feedback.extend_offset(parsed.feedback, span.start);
if !terminated {
error!(@feedback, Span::at(span.end), "expected closing bracket");
@@ -55,9 +48,9 @@ pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Pass<SyntaxModel>
parsed.output
}
- Token::Star => Node::ToggleBolder,
+ Token::Star => Node::ToggleBolder,
Token::Underscore => Node::ToggleItalic,
- Token::Backslash => Node::Linebreak,
+ Token::Backslash => Node::Linebreak,
Token::Raw { raw, terminated } => {
if !terminated {
@@ -69,103 +62,101 @@ pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Pass<SyntaxModel>
Token::Text(text) => Node::Text(text.to_string()),
- other => {
- error!(@feedback, span, "unexpected {}", other.name());
+ Token::LineComment(_) | Token::BlockComment(_) => continue,
+ unexpected => {
+ error!(@feedback, span, "unexpected {}", unexpected.name());
continue;
}
};
- model.add(Spanned { v: node, span: token.span });
+ model.add(Spanned::new(node, span));
}
Pass::new(model, feedback)
}
-/// Performs the function parsing.
struct FuncParser<'s> {
- ctx: ParseContext<'s>,
- feedback: Feedback,
-
+ state: &'s ParseState,
/// ```typst
/// [tokens][body]
/// ^^^^^^
/// ```
tokens: Tokens<'s>,
peeked: Option<Option<Spanned<Token<'s>>>>,
-
/// The spanned body string if there is a body.
/// ```typst
/// [tokens][body]
/// ^^^^
/// ```
body: Option<Spanned<&'s str>>,
+ feedback: Feedback,
}
impl<'s> FuncParser<'s> {
- /// Create a new function parser.
fn new(
header: &'s str,
body: Option<Spanned<&'s str>>,
- ctx: ParseContext<'s>
+ state: &'s ParseState,
) -> FuncParser<'s> {
FuncParser {
- ctx,
- feedback: Feedback::new(),
- tokens: Tokens::new(Position::new(0, 1), header, TokenizationMode::Header),
+ state,
+ // Start at column 1 because the opening bracket is also part of
+ // the function, but not part of the `header` string.
+ tokens: Tokens::new(header, Position::new(0, 1), TokenMode::Header),
peeked: None,
body,
+ feedback: Feedback::new(),
}
}
- /// Do the parsing.
fn parse(mut self) -> Pass<Node> {
- let parsed = if let Some(header) = self.parse_func_header() {
+ let (parser, header) = if let Some(header) = self.parse_func_header() {
let name = header.name.v.as_str();
- let (parser, deco) = match self.ctx.scope.get_parser(name) {
- // A valid function.
- Ok(parser) => (parser, Decoration::ValidFuncName),
-
- // The fallback parser was returned. Invalid function.
- Err(parser) => {
+ let (parser, deco) = match self.state.scope.get_parser(name) {
+ // The function exists in the scope.
+ Some(parser) => (parser, Decoration::ValidFuncName),
+
+ // The function does not exist in the scope. The parser that is
+ // returned here is a fallback parser which exists to make sure
+ // the content of the function is not totally dropped (on a best
+ // effort basis).
+ None => {
error!(@self.feedback, header.name.span, "unknown function");
+ let parser = self.state.scope.get_fallback_parser();
(parser, Decoration::InvalidFuncName)
}
};
self.feedback.decos.push(Spanned::new(deco, header.name.span));
-
- parser(header, self.body, self.ctx)
+ (parser, header)
} else {
- let default = FuncHeader {
- name: Spanned::new(Ident("".to_string()), Span::ZERO),
+ // Parse the body with the fallback parser even when the header is
+ // completely unparsable.
+ let parser = self.state.scope.get_fallback_parser();
+ let header = FuncHeader {
+ name: Spanned::new(Ident(String::new()), Span::ZERO),
args: FuncArgs::new(),
};
-
- // Use the fallback function such that the body is still rendered
- // even if the header is completely unparsable.
- self.ctx.scope.get_fallback_parser()(default, self.body, self.ctx)
+ (parser, header)
};
- self.feedback.extend(parsed.feedback);
+ let call = FuncCall { header, body: self.body };
+ let parsed = parser(call, self.state);
+ self.feedback.extend(parsed.feedback);
Pass::new(Node::Model(parsed.output), self.feedback)
}
- /// Parse the header tokens.
fn parse_func_header(&mut self) -> Option<FuncHeader> {
- let start = self.pos();
- self.skip_whitespace();
-
- let name = match self.parse_ident() {
- Some(ident) => ident,
- None => {
- let other = self.eat();
- self.expected_found_or_at("identifier", other, start);
- return None;
- }
- };
+ let after_bracket = self.pos();
- self.skip_whitespace();
+ self.skip_white();
+ let name = try_opt_or!(self.parse_ident(), {
+ self.expected_found_or_at("function name", after_bracket);
+ return None;
+ });
+
+ self.skip_white();
let args = match self.eat().map(Spanned::value) {
Some(Token::Colon) => self.parse_func_args(),
Some(_) => {
@@ -178,58 +169,74 @@ impl<'s> FuncParser<'s> {
Some(FuncHeader { name, args })
}
- /// Parse the argument list between colons and end of the header.
fn parse_func_args(&mut self) -> FuncArgs {
- // Parse a collection until the token is `None`, that is, the end of the
- // header.
- self.parse_collection(None, |p| {
- // If we have an identifier we might have a keyword argument,
- // otherwise its for sure a postional argument.
- if let Some(ident) = p.parse_ident() {
- // This could still be a named tuple
- if let Some(Token::LeftParen) = p.peekv() {
- let tuple = p.parse_named_tuple(ident);
- return Ok(tuple.map(|t| FuncArg::Pos(Expr::NamedTuple(t))));
- }
-
- p.skip_whitespace();
+ let mut args = FuncArgs::new();
+ loop {
+ self.skip_white();
+ if self.eof() {
+ break;
+ }
- if let Some(Token::Equals) = p.peekv() {
- p.eat();
- p.skip_whitespace();
+ let arg = if let Some(ident) = self.parse_ident() {
+ self.skip_white();
- // Semantic highlighting for argument keys.
- p.feedback.decos.push(
- Spanned::new(Decoration::ArgumentKey, ident.span));
+ // This could be a keyword argument, or a positional argument of
+ // type named tuple or identifier.
+ if self.check_eat(Token::Equals).is_some() {
+ self.skip_white();
- let value = p.parse_expr().ok_or(("value", None))?;
+ let key = ident;
+ self.feedback.decos.push(
+ Spanned::new(Decoration::ArgumentKey, key.span)
+ );
- // Add a keyword argument.
- let span = Span::merge(ident.span, value.span);
- let pair = Pair { key: ident, value };
- Ok(Spanned::new(FuncArg::Key(pair), span))
+ let value = try_opt_or!(self.parse_expr(), {
+ self.expected("value");
+ continue;
+ });
+
+ let span = Span::merge(key.span, value.span);
+ let arg = FuncArg::Key(Pair { key, value });
+ Spanned::new(arg, span)
+ } else if self.check(Token::LeftParen) {
+ let tuple = self.parse_named_tuple(ident);
+ tuple.map(|tup| FuncArg::Pos(Expr::NamedTuple(tup)))
} else {
- // Add a positional argument because there was no equals
- // sign after the identifier that could have been a key.
- Ok(ident.map(|id| FuncArg::Pos(Expr::Ident(id))))
+ ident.map(|id| FuncArg::Pos(Expr::Ident(id)))
}
} else {
- // Add a positional argument because we haven't got an
- // identifier that could be an argument key.
- let value = p.parse_expr().ok_or(("argument", None))?;
- Ok(value.map(|expr| FuncArg::Pos(expr)))
+ // It's a positional argument.
+ try_opt_or!(self.parse_expr(), {
+ self.expected("argument");
+ continue;
+ }).map(|expr| FuncArg::Pos(expr))
+ };
+
+ let behind_arg = arg.span.end;
+ args.add(arg);
+
+ self.skip_white();
+ if self.eof() {
+ break;
}
- }).v
+
+ self.expect_at(Token::Comma, behind_arg);
+ }
+ args
+ }
+}
+
+// Parsing expressions and values
+impl FuncParser<'_> {
+ fn parse_ident(&mut self) -> Option<Spanned<Ident>> {
+ self.peek().and_then(|token| match token.v {
+ Token::ExprIdent(id) => self.eat_span(Ident(id.to_string())),
+ _ => None,
+ })
}
- /// Parse an expression which may contain math operands. For this, this
- /// method looks for operators in descending order of associativity, i.e. we
- /// first drill down to find all negations, brackets and tuples, the next
- /// level, we look for multiplication and division and here finally, for
- /// addition and subtraction.
fn parse_expr(&mut self) -> Option<Spanned<Expr>> {
- let o1 = self.parse_term()?;
- self.parse_binop(o1, "summand", Self::parse_expr, |token| match token {
+ self.parse_binops("summand", Self::parse_term, |token| match token {
Token::Plus => Some(Expr::Add),
Token::Hyphen => Some(Expr::Sub),
_ => None,
@@ -237,60 +244,57 @@ impl<'s> FuncParser<'s> {
}
fn parse_term(&mut self) -> Option<Spanned<Expr>> {
- let o1 = self.parse_factor()?;
- self.parse_binop(o1, "factor", Self::parse_term, |token| match token {
+ self.parse_binops("factor", Self::parse_factor, |token| match token {
Token::Star => Some(Expr::Mul),
Token::Slash => Some(Expr::Div),
_ => None,
})
}
- fn parse_binop<F, G>(
+ /// Parse expression of the form `<operand> (<op> <operand>)*`.
+ fn parse_binops(
&mut self,
- o1: Spanned<Expr>,
operand_name: &str,
- parse_operand: F,
- parse_op: G,
- ) -> Option<Spanned<Expr>>
- where
- F: FnOnce(&mut Self) -> Option<Spanned<Expr>>,
- G: FnOnce(Token) -> Option<fn(Box<Spanned<Expr>>, Box<Spanned<Expr>>) -> Expr>,
- {
- self.skip_whitespace();
-
- if let Some(next) = self.peek() {
- if let Some(binop) = parse_op(next.v) {
+ mut parse_operand: impl FnMut(&mut Self) -> Option<Spanned<Expr>>,
+ mut parse_op: impl FnMut(Token) -> Option<
+ fn(Box<Spanned<Expr>>, Box<Spanned<Expr>>) -> Expr
+ >,
+ ) -> Option<Spanned<Expr>> {
+ let mut left = parse_operand(self)?;
+
+ self.skip_white();
+ while let Some(token) = self.peek() {
+ if let Some(op) = parse_op(token.v) {
self.eat();
- self.skip_whitespace();
-
- if let Some(o2) = parse_operand(self) {
- let span = Span::merge(o1.span, o2.span);
- let expr = binop(Box::new(o1), Box::new(o2));
- return Some(Spanned::new(expr, span));
- } else {
- error!(
- @self.feedback, Span::merge(next.span, o1.span),
- "missing right {}", operand_name,
- );
+ self.skip_white();
+
+ if let Some(right) = parse_operand(self) {
+ let span = Span::merge(left.span, right.span);
+ let v = op(Box::new(left), Box::new(right));
+ left = Spanned::new(v, span);
+ self.skip_white();
+ continue;
}
+
+ error!(
+ @self.feedback, Span::merge(left.span, token.span),
+ "missing right {}", operand_name,
+ );
}
+ break;
}
- Some(o1)
+ Some(left)
}
- /// Parse expressions that are of the form value or -value.
fn parse_factor(&mut self) -> Option<Spanned<Expr>> {
- let first = self.peek()?;
- if first.v == Token::Hyphen {
- self.eat();
- self.skip_whitespace();
-
- if let Some(factor) = self.parse_value() {
- let span = Span::merge(first.span, factor.span);
- Some(Spanned::new(Expr::Neg(Box::new(factor)), span))
+ if let Some(hyph) = self.check_eat(Token::Hyphen) {
+ self.skip_white();
+ if let Some(value) = self.parse_value() {
+ let span = Span::merge(hyph.span, value.span);
+ Some(Spanned::new(Expr::Neg(Box::new(value)), span))
} else {
- error!(@self.feedback, first.span, "dangling minus");
+ error!(@self.feedback, hyph.span, "dangling minus");
None
}
} else {
@@ -299,283 +303,247 @@ impl<'s> FuncParser<'s> {
}
fn parse_value(&mut self) -> Option<Spanned<Expr>> {
- let first = self.peek()?;
- macro_rules! take {
- ($v:expr) => ({ self.eat(); Spanned { v: $v, span: first.span } });
- }
-
- Some(match first.v {
- Token::ExprIdent(i) => {
- let name = take!(Ident(i.to_string()));
-
- // This could be a named tuple or an identifier
- if let Some(Token::LeftParen) = self.peekv() {
- self.parse_named_tuple(name).map(|t| Expr::NamedTuple(t))
+ let Spanned { v: token, span } = self.peek()?;
+ match token {
+ // This could be a named tuple or an identifier.
+ Token::ExprIdent(id) => {
+ let name = Spanned::new(Ident(id.to_string()), span);
+ self.eat();
+ self.skip_white();
+ Some(if self.check(Token::LeftParen) {
+ self.parse_named_tuple(name).map(|tup| Expr::NamedTuple(tup))
} else {
- name.map(|i| Expr::Ident(i))
- }
- },
+ name.map(|id| Expr::Ident(id))
+ })
+ }
+
Token::ExprStr { string, terminated } => {
if !terminated {
- self.expected_at("quote", first.span.end);
+ self.expected_at("quote", span.end);
}
-
- take!(Expr::Str(unescape_string(string)))
+ self.eat_span(Expr::Str(unescape_string(string)))
}
- Token::ExprNumber(n) => take!(Expr::Number(n)),
- Token::ExprSize(s) => take!(Expr::Size(s)),
- Token::ExprBool(b) => take!(Expr::Bool(b)),
+ Token::ExprNumber(n) => self.eat_span(Expr::Number(n)),
+ Token::ExprSize(s) => self.eat_span(Expr::Size(s)),
+ Token::ExprBool(b) => self.eat_span(Expr::Bool(b)),
Token::ExprHex(s) => {
if let Ok(color) = RgbaColor::from_str(s) {
- take!(Expr::Color(color))
+ self.eat_span(Expr::Color(color))
} else {
- // Heal color by assuming black
- error!(@self.feedback, first.span, "invalid color");
- take!(Expr::Color(RgbaColor::new_healed(0, 0, 0, 255)))
+ // Heal color by assuming black.
+ error!(@self.feedback, span, "invalid color");
+ let healed = RgbaColor::new_healed(0, 0, 0, 255);
+ self.eat_span(Expr::Color(healed))
}
},
+ // This could be a tuple or a parenthesized expression. We parse as
+ // a tuple in any case and coerce the tuple into a value if it is
+ // coercable (length 1 and no trailing comma).
Token::LeftParen => {
- let (mut tuple, can_be_coerced) = self.parse_tuple();
- // Coerce 1-tuple into value
- if can_be_coerced && tuple.v.items.len() > 0 {
- tuple.v.items.pop().expect("length is at least one")
+ let (mut tuple, coercable) = self.parse_tuple();
+ Some(if coercable {
+ tuple.v.items.pop().expect("tuple is coercable")
} else {
- tuple.map(|t| Expr::Tuple(t))
- }
- },
- Token::LeftBrace => self.parse_object().map(|o| Expr::Object(o)),
-
- _ => return None,
- })
- }
-
- /// Parse a tuple expression: `(<expr>, ...)`. The boolean in the return
- /// values showes whether the tuple can be coerced into a single value.
- fn parse_tuple(&mut self) -> (Spanned<Tuple>, bool) {
- let token = self.eat();
- debug_assert_eq!(token.map(Spanned::value), Some(Token::LeftParen));
+ tuple.map(|tup| Expr::Tuple(tup))
+ })
+ }
+ Token::LeftBrace => {
+ Some(self.parse_object().map(|obj| Expr::Object(obj)))
+ }
- // Parse a collection until a right paren appears and complain about
- // missing a `value` when an invalid token is encoutered.
- self.parse_collection_comma_aware(Some(Token::RightParen),
- |p| p.parse_expr().ok_or(("value", None)))
+ _ => None,
+ }
}
- /// Parse a tuple expression: `name(<expr>, ...)` with a given identifier.
fn parse_named_tuple(&mut self, name: Spanned<Ident>) -> Spanned<NamedTuple> {
let tuple = self.parse_tuple().0;
let span = Span::merge(name.span, tuple.span);
Spanned::new(NamedTuple::new(name, tuple), span)
}
- /// Parse an object expression: `{ <key>: <value>, ... }`.
- fn parse_object(&mut self) -> Spanned<Object> {
- let token = self.eat();
- debug_assert_eq!(token.map(Spanned::value), Some(Token::LeftBrace));
-
- // Parse a collection until a right brace appears.
- self.parse_collection(Some(Token::RightBrace), |p| {
- // Expect an identifier as the key.
- let key = p.parse_ident().ok_or(("key", None))?;
-
- // Expect a colon behind the key (only separated by whitespace).
- let behind_key = p.pos();
- p.skip_whitespace();
- if p.peekv() != Some(Token::Colon) {
- return Err(("colon", Some(behind_key)));
+ /// The boolean tells you whether the tuple can be coerced into a value
+ /// (this is the case when it's length 1 and has no trailing comma).
+ fn parse_tuple(&mut self) -> (Spanned<Tuple>, bool) {
+ let start = self.pos();
+ self.assert(Token::LeftParen);
+
+ let mut tuple = Tuple::new();
+ let mut commaless = true;
+ loop {
+ self.skip_white();
+ if self.eof() || self.check(Token::RightParen) {
+ break;
}
- p.eat();
- p.skip_whitespace();
+ let expr = try_opt_or!(self.parse_expr(), {
+ self.expected("value");
+ continue;
+ });
- // Semantic highlighting for object keys.
- p.feedback.decos.push(
- Spanned::new(Decoration::ObjectKey, key.span));
+ let behind_expr = expr.span.end;
+ tuple.add(expr);
- let value = p.parse_expr().ok_or(("value", None))?;
+ self.skip_white();
+ if self.eof() || self.check(Token::RightParen) {
+ break;
+ }
- let span = Span::merge(key.span, value.span);
- Ok(Spanned::new(Pair { key, value }, span))
- })
- }
+ self.expect_at(Token::Comma, behind_expr);
+ commaless = false;
+ }
- /// Parse a comma-separated collection where each item is parsed through
- /// `parse_item` until the `end` token is met.
- fn parse_collection<C, I, F>(
- &mut self,
- end: Option<Token>,
- parse_item: F
- ) -> Spanned<C>
- where
- C: FromIterator<Spanned<I>>,
- F: FnMut(&mut Self) -> Result<Spanned<I>, (&'static str, Option<Position>)>,
- {
- self.parse_collection_comma_aware(end, parse_item).0
+ self.expect(Token::RightParen);
+ let end = self.pos();
+ let coercable = commaless && !tuple.items.is_empty();
+
+ (Spanned::new(tuple, Span::new(start, end)), coercable)
}
- /// Parse a comma-separated collection where each item is parsed through
- /// `parse_item` until the `end` token is met. The first item in the return
- /// tuple is the collection, the second item indicates whether the
- /// collection can be coerced into a single item (i.e. no comma appeared).
- fn parse_collection_comma_aware<C, I, F>(
- &mut self,
- end: Option<Token>,
- mut parse_item: F
- ) -> (Spanned<C>, bool)
- where
- C: FromIterator<Spanned<I>>,
- F: FnMut(&mut Self) -> Result<Spanned<I>, (&'static str, Option<Position>)>,
- {
+ fn parse_object(&mut self) -> Spanned<Object> {
let start = self.pos();
- let mut can_be_coerced = true;
+ self.assert(Token::LeftBrace);
- // Parse the comma separated items.
- let collection = std::iter::from_fn(|| {
- self.skip_whitespace();
- let peeked = self.peekv();
-
- // We finished as expected.
- if peeked == end {
- self.eat();
- return None;
+ let mut object = Object::new();
+ loop {
+ self.skip_white();
+ if self.eof() || self.check(Token::RightBrace) {
+ break;
}
- // We finished without the expected end token (which has to be a
- // `Some` value at this point since otherwise we would have already
- // returned in the previous case).
- if peeked == None {
- self.eat();
- self.expected_at(end.unwrap().name(), self.pos());
- return None;
+ let key = try_opt_or!(self.parse_ident(), {
+ self.expected("key");
+ continue;
+ });
+
+ let after_key = self.pos();
+ self.skip_white();
+ if !self.expect_at(Token::Colon, after_key) {
+ continue;
}
- // Try to parse a collection item.
- match parse_item(self) {
- Ok(item) => {
- // Expect a comma behind the item (only separated by
- // whitespace).
- self.skip_whitespace();
- match self.peekv() {
- Some(Token::Comma) => {
- can_be_coerced = false;
- self.eat();
- }
- t @ Some(_) if t != end => {
- can_be_coerced = false;
- self.expected_at("comma", item.span.end);
- },
- _ => {}
- }
-
- return Some(Some(item));
- }
+ self.feedback.decos.push(
+ Spanned::new(Decoration::ObjectKey, key.span)
+ );
- // The item parser expected something different at either some
- // given position or instead of the currently peekable token.
- Err((expected, Some(pos))) => self.expected_at(expected, pos),
- Err((expected, None)) => {
- let token = self.peek();
- if token.map(Spanned::value) != end {
- self.eat();
- }
- self.expected_found_or_at(expected, token, self.pos());
- }
+ self.skip_white();
+ let value = try_opt_or!(self.parse_expr(), {
+ self.expected("value");
+ continue;
+ });
+
+ let behind_value = value.span.end;
+ let span = Span::merge(key.span, value.span);
+ object.add(Spanned::new(Pair { key, value }, span));
+
+ self.skip_white();
+ if self.eof() || self.check(Token::RightBrace) {
+ break;
}
- Some(None)
- }).filter_map(|x| x).collect();
+ self.expect_at(Token::Comma, behind_value);
+ }
+ self.expect(Token::RightBrace);
let end = self.pos();
- (Spanned::new(collection, Span { start, end }), can_be_coerced)
+
+ Spanned::new(object, Span::new(start, end))
}
+}
- /// Try to parse an identifier and do nothing if the peekable token is no
- /// identifier.
- fn parse_ident(&mut self) -> Option<Spanned<Ident>> {
- match self.peek() {
- Some(Spanned { v: Token::ExprIdent(s), span }) => {
- self.eat();
- Some(Spanned { v: Ident(s.to_string()), span })
- }
- _ => None
+// Error handling
+impl FuncParser<'_> {
+ fn expect(&mut self, token: Token<'_>) -> bool {
+ if self.check(token) {
+ self.eat();
+ true
+ } else {
+ self.expected(token.name());
+ false
}
}
- /// Skip all whitespace/comment tokens.
- fn skip_whitespace(&mut self) {
- self.eat_until(|t| match t {
- Token::Space(_) | Token::LineComment(_) |
- Token::BlockComment(_) => false,
- _ => true,
- }, false)
+ fn expect_at(&mut self, token: Token<'_>, pos: Position) -> bool {
+ if self.check(token) {
+ self.eat();
+ true
+ } else {
+ self.expected_at(token.name(), pos);
+ false
+ }
}
- /// Add an error about an expected `thing` which was not found, showing
- /// what was found instead.
- fn expected_found(&mut self, thing: &str, found: Spanned<Token>) {
- error!(
- @self.feedback, found.span,
- "expected {}, found {}", thing, found.v.name(),
- );
+ fn expected(&mut self, thing: &str) {
+ if let Some(found) = self.eat() {
+ error!(
+ @self.feedback, found.span,
+ "expected {}, found {}", thing, found.v.name(),
+ );
+ } else {
+ error!(@self.feedback, Span::at(self.pos()), "expected {}", thing);
+ }
}
- /// Add an error about an `thing` which was expected but not found at the
- /// given position.
fn expected_at(&mut self, thing: &str, pos: Position) {
error!(@self.feedback, Span::at(pos), "expected {}", thing);
}
- /// Add a expected-found-error if `found` is `Some` and an expected-error
- /// otherwise.
- fn expected_found_or_at(
- &mut self,
- thing: &str,
- found: Option<Spanned<Token>>,
- pos: Position
- ) {
- match found {
- Some(found) => self.expected_found(thing, found),
- None => self.expected_at(thing, pos),
+ fn expected_found_or_at(&mut self, thing: &str, pos: Position) {
+ if self.eof() {
+ self.expected_at(thing, pos)
+ } else {
+ self.expected(thing);
}
}
+}
- /// Consume tokens until the function returns true and only consume the last
- /// token if instructed to so by `eat_match`.
- fn eat_until<F>(&mut self, mut f: F, eat_match: bool)
- where F: FnMut(Token<'s>) -> bool {
- while let Some(token) = self.peek() {
- if f(token.v) {
- if eat_match {
- self.eat();
- }
- break;
+// Parsing primitives
+impl<'s> FuncParser<'s> {
+ fn skip_white(&mut self) {
+ loop {
+ match self.peek().map(Spanned::value) {
+ Some(Token::Space(_))
+ | Some(Token::LineComment(_))
+ | Some(Token::BlockComment(_)) => { self.eat(); }
+ _ => break,
}
-
- self.eat();
}
}
- /// Consume and return the next token.
fn eat(&mut self) -> Option<Spanned<Token<'s>>> {
- self.peeked.take()
- .unwrap_or_else(|| self.tokens.next())
+ self.peeked.take().unwrap_or_else(|| self.tokens.next())
+ }
+
+ fn eat_span<T>(&mut self, v: T) -> Option<Spanned<T>> {
+ self.eat().map(|spanned| spanned.map(|_| v))
}
- /// Peek at the next token without consuming it.
fn peek(&mut self) -> Option<Spanned<Token<'s>>> {
- let iter = &mut self.tokens;
- *self.peeked.get_or_insert_with(|| iter.next())
+ let tokens = &mut self.tokens;
+ *self.peeked.get_or_insert_with(|| tokens.next())
+ }
+
+ fn assert(&mut self, token: Token<'_>) {
+ assert!(self.check_eat(token).is_some());
+ }
+
+ fn check(&mut self, token: Token<'_>) -> bool {
+ self.peek().map(Spanned::value) == Some(token)
+ }
+
+ fn check_eat(&mut self, token: Token<'_>) -> Option<Spanned<Token<'s>>> {
+ if self.check(token) {
+ self.eat()
+ } else {
+ None
+ }
}
- /// Peek at the unspanned value of the next token.
- fn peekv(&mut self) -> Option<Token<'s>> {
- self.peek().map(Spanned::value)
+ fn eof(&mut self) -> bool {
+ self.peek().is_none()
}
- /// The position at the end of the last eaten token / start of the peekable
- /// token.
fn pos(&self) -> Position {
self.peeked.flatten()
.map(|s| s.span.start)
@@ -583,140 +551,71 @@ impl<'s> FuncParser<'s> {
}
}
-/// Unescape a string: `the string is \"this\"` => `the string is "this"`.
fn unescape_string(string: &str) -> String {
- let mut s = String::with_capacity(string.len());
let mut iter = string.chars();
+ let mut out = String::with_capacity(string.len());
while let Some(c) = iter.next() {
if c == '\\' {
match iter.next() {
- Some('\\') => s.push('\\'),
- Some('"') => s.push('"'),
- Some('n') => s.push('\n'),
- Some('t') => s.push('\t'),
- Some(c) => { s.push('\\'); s.push(c); }
- None => s.push('\\'),
+ Some('\\') => out.push('\\'),
+ Some('"') => out.push('"'),
+ Some('n') => out.push('\n'),
+ Some('t') => out.push('\t'),
+ Some(c) => { out.push('\\'); out.push(c); }
+ None => out.push('\\'),
}
} else {
- s.push(c);
+ out.push(c);
}
}
- s
+ out
}
-/// Unescape raw markup into lines.
+/// Unescape raw markup and split it into into lines.
fn unescape_raw(raw: &str) -> Vec<String> {
- let mut lines = Vec::new();
- let mut s = String::new();
let mut iter = raw.chars().peekable();
+ let mut line = String::new();
+ let mut lines = Vec::new();
while let Some(c) = iter.next() {
if c == '\\' {
match iter.next() {
- Some('`') => s.push('`'),
- Some(c) => { s.push('\\'); s.push(c); }
- None => s.push('\\'),
+ Some('`') => line.push('`'),
+ Some(c) => { line.push('\\'); line.push(c); }
+ None => line.push('\\'),
}
} else if is_newline_char(c) {
if c == '\r' && iter.peek() == Some(&'\n') {
iter.next();
}
- lines.push(std::mem::replace(&mut s, String::new()));
+ lines.push(std::mem::take(&mut line));
} else {
- s.push(c);
+ line.push(c);
}
}
- lines.push(s);
+ lines.push(line);
lines
}
-
#[cfg(test)]
#[allow(non_snake_case)]
mod tests {
use crate::size::Size;
- use crate::syntax::test::{DebugFn, check, zspan};
- use crate::syntax::func::Value;
+ use super::super::test::{check, DebugFn};
+ use super::super::func::Value;
use super::*;
use Decoration::*;
+ use Expr::{Number as Num, Size as Sz, Bool};
use Node::{
Space as S, ToggleItalic as Italic, ToggleBolder as Bold,
Parbreak, Linebreak,
};
- use Expr::{Number as Num, Size as Sz, Bool};
- fn Id(text: &str) -> Expr { Expr::Ident(Ident(text.to_string())) }
- fn Str(text: &str) -> Expr { Expr::Str(text.to_string()) }
- fn Pt(points: f32) -> Expr { Expr::Size(Size::pt(points)) }
- fn Neg(e1: Expr) -> Expr { Expr::Neg(Box::new(zspan(e1))) }
- fn Add(e1: Expr, e2: Expr) -> Expr {
- Expr::Add(Box::new(zspan(e1)), Box::new(zspan(e2)))
- }
- fn Sub(e1: Expr, e2: Expr) -> Expr {
- Expr::Sub(Box::new(zspan(e1)), Box::new(zspan(e2)))
- }
- fn Mul(e1: Expr, e2: Expr) -> Expr {
- Expr::Mul(Box::new(zspan(e1)), Box::new(zspan(e2)))
- }
- fn Div(e1: Expr, e2: Expr) -> Expr {
- Expr::Div(Box::new(zspan(e1)), Box::new(zspan(e2)))
- }
-
- fn Clr(r: u8, g: u8, b: u8, a: u8) -> Expr {
- Expr::Color(RgbaColor::new(r, g, b, a))
- }
- fn ClrStr(color: &str) -> Expr {
- Expr::Color(RgbaColor::from_str(color).expect("invalid test color"))
- }
- fn ClrStrHealed() -> Expr {
- let mut c = RgbaColor::from_str("000f").expect("invalid test color");
- c.healed = true;
- Expr::Color(c)
- }
-
- fn T(text: &str) -> Node { Node::Text(text.to_string()) }
-
- /// Create a raw text node.
- macro_rules! raw {
- ($($line:expr),* $(,)?) => {
- Node::Raw(vec![$($line.to_string()),*])
- };
- }
-
- /// Create a tuple expression.
- macro_rules! tuple {
- ($($items:expr),* $(,)?) => {
- Expr::Tuple(Tuple { items: spanned![vec $($items),*].0 })
- };
- }
-
- /// Create a named tuple expression.
- macro_rules! named_tuple {
- ($name:expr $(, $items:expr)* $(,)?) => {
- Expr::NamedTuple(NamedTuple::new(
- zspan(Ident($name.to_string())),
- zspan(Tuple { items: spanned![vec $($items),*].0 })
- ))
- };
- }
-
- /// Create an object expression.
- macro_rules! object {
- ($($key:expr => $value:expr),* $(,)?) => {
- Expr::Object(Object {
- pairs: vec![$(zspan(Pair {
- key: zspan(Ident($key.to_string())),
- value: zspan($value),
- })),*]
- })
- };
- }
-
/// Test whether the given string parses into
/// - the given node list (required).
/// - the given error list (optional, if omitted checks against empty list).
@@ -733,15 +632,15 @@ mod tests {
scope.add::<DebugFn>("box");
scope.add::<DebugFn>("val");
- let ctx = ParseContext { scope: &scope };
- let pass = parse(Position::ZERO, $source, ctx);
+ let state = ParseState { scope };
+ let pass = parse($source, Position::ZERO, &state);
- // Test model
- let (exp, cmp) = spanned![vec $($model)*];
+ // Test model.
+ let (exp, cmp) = span_vec![$($model)*];
check($source, exp, pass.output.nodes, cmp);
- // Test problems
- let (exp, cmp) = spanned![vec $($problems)*];
+ // Test problems.
+ let (exp, cmp) = span_vec![$($problems)*];
let exp = exp.into_iter()
.map(|s: Spanned<&str>| s.map(|e| e.to_string()))
.collect::<Vec<_>>();
@@ -750,42 +649,90 @@ mod tests {
.collect::<Vec<_>>();
check($source, exp, found, cmp);
- // Test decos
- $(let (exp, cmp) = spanned![vec $($decos)*];
+ // Test decos.
+ $(let (exp, cmp) = span_vec![$($decos)*];
check($source, exp, pass.feedback.decos, cmp);)?
};
}
- /// Write down a `DebugFn` function model compactly.
+ /// Shorthand for `p!("[val: ...]" => func!("val", ...))`.
+ macro_rules! pval {
+ ($header:expr => $($tts:tt)*) => {
+ p!(concat!("[val: ", $header, "]") => [func!("val": $($tts)*)]);
+ }
+ }
+
+ fn Id(text: &str) -> Expr { Expr::Ident(Ident(text.to_string())) }
+ fn Str(text: &str) -> Expr { Expr::Str(text.to_string()) }
+ fn Pt(points: f32) -> Expr { Expr::Size(Size::pt(points)) }
+ fn Color(r: u8, g: u8, b: u8, a: u8) -> Expr { Expr::Color(RgbaColor::new(r, g, b, a)) }
+ fn ColorStr(color: &str) -> Expr { Expr::Color(RgbaColor::from_str(color).expect("invalid test color")) }
+ fn ColorHealed() -> Expr { Expr::Color(RgbaColor::new_healed(0, 0, 0, 255)) }
+ fn Neg(e1: Expr) -> Expr { Expr::Neg(Box::new(Z(e1))) }
+ fn Add(e1: Expr, e2: Expr) -> Expr { Expr::Add(Box::new(Z(e1)), Box::new(Z(e2))) }
+ fn Sub(e1: Expr, e2: Expr) -> Expr { Expr::Sub(Box::new(Z(e1)), Box::new(Z(e2))) }
+ fn Mul(e1: Expr, e2: Expr) -> Expr { Expr::Mul(Box::new(Z(e1)), Box::new(Z(e2))) }
+ fn Div(e1: Expr, e2: Expr) -> Expr { Expr::Div(Box::new(Z(e1)), Box::new(Z(e2))) }
+ fn T(text: &str) -> Node { Node::Text(text.to_string()) }
+ fn Z<T>(v: T) -> Spanned<T> { Spanned::zero(v) }
+
+ macro_rules! tuple {
+ ($($items:expr),* $(,)?) => {
+ Expr::Tuple(Tuple { items: span_vec![$($items),*].0 })
+ };
+ }
+
+ macro_rules! named_tuple {
+ ($name:expr $(, $items:expr)* $(,)?) => {
+ Expr::NamedTuple(NamedTuple::new(
+ Z(Ident($name.to_string())),
+ Z(Tuple { items: span_vec![$($items),*].0 })
+ ))
+ };
+ }
+
+ macro_rules! object {
+ ($($key:expr => $value:expr),* $(,)?) => {
+ Expr::Object(Object {
+ pairs: vec![$(Z(Pair {
+ key: Z(Ident($key.to_string())),
+ value: Z($value),
+ })),*]
+ })
+ };
+ }
+
+ macro_rules! raw {
+ ($($line:expr),* $(,)?) => {
+ Node::Raw(vec![$($line.to_string()),*])
+ };
+ }
+
macro_rules! func {
- ($name:tt $(: ($($pos:tt)*), { $($key:tt)* } )? $(; $($body:tt)*)?) => ({
+ ($name:tt $(: ($($pos:tt)*) $(, { $($key:tt)* })? )? $(; $($body:tt)*)?) => {{
#[allow(unused_mut)]
let mut args = FuncArgs::new();
- $(args.pos = Tuple::parse(zspan(tuple!($($pos)*))).unwrap();)?
- $(args.key = Object::parse(zspan(object! { $($key)* })).unwrap();)?
-
+ $(args.pos = Tuple::parse(Z(tuple!($($pos)*))).unwrap();
+ $(args.key = Object::parse(Z(object! { $($key)* })).unwrap();)?)?
Node::Model(Box::new(DebugFn {
header: FuncHeader {
- name: spanned!(item $name).map(|s| Ident(s.to_string())),
+ name: span_item!($name).map(|s| Ident(s.to_string())),
args,
},
body: func!(@body $($($body)*)?),
}))
- });
-
- (@body [$($body:tt)*]) => ({
- Some(SyntaxModel { nodes: spanned![vec $($body)*].0 })
- });
- (@body) => (None);
+ }};
+ (@body [$($body:tt)*]) => { Some(SyntaxModel { nodes: span_vec![$($body)*].0 }) };
+ (@body) => { None };
}
#[test]
fn parse_color_strings() {
- assert_eq!(Clr(0xf6, 0x12, 0x43, 0xff), ClrStr("f61243ff"));
- assert_eq!(Clr(0xb3, 0xd8, 0xb3, 0xff), ClrStr("b3d8b3"));
- assert_eq!(Clr(0xfc, 0xd2, 0xa9, 0xad), ClrStr("fCd2a9AD"));
- assert_eq!(Clr(0x22, 0x33, 0x33, 0xff), ClrStr("233"));
- assert_eq!(Clr(0x11, 0x11, 0x11, 0xbb), ClrStr("111b"));
+ assert_eq!(Color(0xf6, 0x12, 0x43, 0xff), ColorStr("f61243ff"));
+ assert_eq!(Color(0xb3, 0xd8, 0xb3, 0xff), ColorStr("b3d8b3"));
+ assert_eq!(Color(0xfc, 0xd2, 0xa9, 0xad), ColorStr("fCd2a9AD"));
+ assert_eq!(Color(0x22, 0x33, 0x33, 0xff), ColorStr("233"));
+ assert_eq!(Color(0x11, 0x11, 0x11, 0xbb), ColorStr("111b"));
}
#[test]
@@ -824,7 +771,7 @@ mod tests {
#[test]
fn parse_basic_nodes() {
- // Basic nodes
+ // Basic nodes.
p!("" => []);
p!("hi" => [T("hi")]);
p!("*hi" => [Bold, T("hi")]);
@@ -838,13 +785,13 @@ mod tests {
p!(r"a\ b" => [T("a"), Linebreak, S, T("b")]);
p!("πŸ’œ\n\n 🌍" => [T("πŸ’œ"), Parbreak, T("🌍")]);
- // Raw markup
+ // Raw markup.
p!("`py`" => [raw!["py"]]);
p!("[val][`hi]`]" => [func!("val"; [raw!["hi]"]])]);
p!("`hi\nyou" => [raw!["hi", "you"]], [(1:3, 1:3, "expected backtick")]);
p!("`hi\\`du`" => [raw!["hi`du"]]);
- // Spanned nodes
+ // Spanned nodes.
p!("Hi" => [(0:0, 0:2, T("Hi"))]);
p!("*Hi*" => [(0:0, 0:1, Bold), (0:1, 0:3, T("Hi")), (0:3, 0:4, Bold)]);
p!("🌎\n*/[n]" =>
@@ -856,45 +803,51 @@ mod tests {
#[test]
fn parse_function_names() {
- // No closing bracket
+ // No closing bracket.
p!("[" => [func!("")], [
- (0:1, 0:1, "expected identifier"),
+ (0:1, 0:1, "expected function name"),
(0:1, 0:1, "expected closing bracket")
]);
- // No name
- p!("[]" => [func!("")], [(0:1, 0:1, "expected identifier")]);
+ // No name.
+ p!("[]" => [func!("")], [(0:1, 0:1, "expected function name")]);
p!("[\"]" => [func!("")], [
- (0:1, 0:3, "expected identifier, found string"),
+ (0:1, 0:3, "expected function name, found string"),
(0:3, 0:3, "expected closing bracket"),
]);
- // An unknown name
+ // An unknown name.
p!("[hi]" =>
[func!("hi")],
[(0:1, 0:3, "unknown function")],
[(0:1, 0:3, InvalidFuncName)],
);
- // A valid name
+ // A valid name.
p!("[f]" => [func!("f")], [], [(0:1, 0:2, ValidFuncName)]);
p!("[ f]" => [func!("f")], [], [(0:3, 0:4, ValidFuncName)]);
- // An invalid token for a name
- p!("[12]" => [func!("")], [(0:1, 0:3, "expected identifier, found number")], []);
- p!("[🌎]" => [func!("")], [(0:1, 0:2, "expected identifier, found invalid token")], []);
- p!("[ 🌎]" => [func!("")], [(0:3, 0:4, "expected identifier, found invalid token")], []);
+ // An invalid token for a name.
+ p!("[12]" => [func!("")], [(0:1, 0:3, "expected function name, found number")], []);
+ p!("[🌎]" => [func!("")], [(0:1, 0:2, "expected function name, found invalid token")], []);
+ p!("[ 🌎]" => [func!("")], [(0:3, 0:4, "expected function name, found invalid token")], []);
}
#[test]
fn parse_colon_starting_function_arguments() {
- // No colon before arg
+ // Valid.
+ p!("[val: true]" =>
+ [func!["val": (Bool(true))]], [],
+ [(0:1, 0:4, ValidFuncName)],
+ );
+
+ // No colon before arg.
p!("[val\"s\"]" => [func!("val")], [(0:4, 0:4, "expected colon")]);
- // No colon before valid, but wrong token
+ // No colon before valid, but wrong token.
p!("[val=]" => [func!("val")], [(0:4, 0:4, "expected colon")]);
- // No colon before invalid tokens, which are ignored
+ // No colon before invalid tokens, which are ignored.
p!("[val/🌎:$]" =>
[func!("val")],
[(0:4, 0:4, "expected colon")],
@@ -909,36 +862,38 @@ mod tests {
(0:7, 0:7, "expected closing bracket"),
]);
- // Just colon without args
+ // Just colon without args.
p!("[val:]" => [func!("val")]);
p!("[val:/*12pt*/]" => [func!("val")]);
- // Whitespace / comments around colon
- p!("[val\n:\ntrue]" => [func!("val": (Bool(true)), {})]);
- p!("[val/*:*/://\ntrue]" => [func!("val": (Bool(true)), {})]);
+ // Whitespace / comments around colon.
+ p!("[val\n:\ntrue]" => [func!("val": (Bool(true)))]);
+ p!("[val/*:*/://\ntrue]" => [func!("val": (Bool(true)))]);
}
#[test]
fn parse_one_positional_argument() {
- // Different expressions
- p!("[val: true]" =>
- [func!("val": (Bool(true)), {})], [],
- [(0:1, 0:4, ValidFuncName)],
- );
- p!("[val: _]" => [func!("val": (Id("_")), {})]);
- p!("[val: name]" => [func!("val": (Id("name")), {})]);
- p!("[val: \"hi\"]" => [func!("val": (Str("hi")), {})]);
- p!("[val: \"a\n[]\\\"string\"]" => [func!("val": (Str("a\n[]\"string")), {})]);
- p!("[val: 3.14]" => [func!("val": (Num(3.14)), {})]);
- p!("[val: 4.5cm]" => [func!("val": (Sz(Size::cm(4.5))), {})]);
- p!("[val: 12e1pt]" => [func!("val": (Pt(12e1)), {})]);
- p!("[val: #f7a20500]" => [func!("val": (ClrStr("f7a20500")), {})]);
-
- // Math
- p!("[val: 3.2in + 6pt]" => [func!("val": (Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0)))), {})]);
- p!("[val: 5 - 0.01]" => [func!("val": (Sub(Num(5.0), Num(0.01))), {})]);
- p!("[val: (3mm * 2)]" => [func!("val": (Mul(Sz(Size::mm(3.0)), Num(2.0))), {})]);
- p!("[val: 12e-3cm/1pt]" => [func!("val": (Div(Sz(Size::cm(12e-3)), Sz(Size::pt(1.0)))), {})]);
+ // Different expressions.
+ pval!("_" => (Id("_")));
+ pval!("name" => (Id("name")));
+ pval!("\"hi\"" => (Str("hi")));
+ pval!("3.14" => (Num(3.14)));
+ pval!("4.5cm" => (Sz(Size::cm(4.5))));
+ pval!("12e1pt" => (Pt(12e1)));
+ pval!("#f7a20500" => (ColorStr("f7a20500")));
+ pval!("\"a\n[]\\\"string\"" => (Str("a\n[]\"string")));
+
+ // Trailing comma.
+ pval!("a," => (Id("a")));
+
+ // Simple coerced tuple.
+ pval!("(hi)" => (Id("hi")));
+
+ // Math.
+ pval!("3.2in + 6pt" => (Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0)))));
+ pval!("5 - 0.01" => (Sub(Num(5.0), Num(0.01))));
+ pval!("(3mm * 2)" => (Mul(Sz(Size::mm(3.0)), Num(2.0))));
+ pval!("12e-3cm/1pt" => (Div(Sz(Size::cm(12e-3)), Sz(Size::pt(1.0)))));
// Unclosed string.
p!("[val: \"hello]" => [func!("val": (Str("hello]")), {})], [
@@ -946,112 +901,91 @@ mod tests {
(0:13, 0:13, "expected closing bracket"),
]);
- //Invalid colors
- p!("[val: #12345]" => [func!("val": (ClrStrHealed()), {})], [
- (0:6, 0:12, "invalid color"),
- ]);
- p!("[val: #a5]" => [func!("val": (ClrStrHealed()), {})], [
- (0:6, 0:9, "invalid color"),
- ]);
- p!("[val: #14b2ah]" => [func!("val": (ClrStrHealed()), {})], [
- (0:6, 0:13, "invalid color"),
- ]);
- p!("[val: #f075ff011]" => [func!("val": (ClrStrHealed()), {})], [
- (0:6, 0:16, "invalid color"),
- ]);
+ // Invalid, healed colors.
+ p!("[val: #12345]" => [func!("val": (ColorHealed()))], [(0:6, 0:12, "invalid color")]);
+ p!("[val: #a5]" => [func!("val": (ColorHealed()))], [(0:6, 0:9, "invalid color")]);
+ p!("[val: #14b2ah]" => [func!("val": (ColorHealed()))], [(0:6, 0:13, "invalid color")]);
+ p!("[val: #f075ff011]" => [func!("val": (ColorHealed()))], [(0:6, 0:16, "invalid color")]);
}
#[test]
fn parse_complex_mathematical_expressions() {
- p!("[val: (3.2in + 6pt)*(5/2-1)]" => [func!("val": (
- Mul(
- Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0))),
- Sub(Div(Num(5.0), Num(2.0)), Num(1.0))
- )
- ), {})]);
- p!("[val: (6.3E+2+4* - 3.2pt)/2]" => [func!("val": (
- Div(Add(Num(6.3e2),Mul(Num(4.0), Neg(Pt(3.2)))), Num(2.0))
- ), {})]);
- p!("[val: 4pt--]" =>
- [func!("val": (Pt(4.0)), {})],
- [
- (0:10, 0:11, "dangling minus"),
- (0:6, 0:10, "missing right summand")
- ],
+ // Valid expressions.
+ pval!("(3.2in + 6pt)*(5/2-1)" => (Mul(
+ Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0))),
+ Sub(Div(Num(5.0), Num(2.0)), Num(1.0))
+ )));
+ pval!("(6.3E+2+4* - 3.2pt)/2" => (Div(
+ Add(Num(6.3e2),Mul(Num(4.0), Neg(Pt(3.2)))),
+ Num(2.0)
+ )));
+
+ // Associativity of multiplication and division.
+ p!("[val: 3/4*5]" =>
+ [func!("val": (Mul(Div(Num(3.0), Num(4.0)), Num(5.0))), {})]
);
+
+ // Invalid expressions.
+ p!("[val: 4pt--]" => [func!("val": (Pt(4.0)))], [
+ (0:10, 0:11, "dangling minus"),
+ (0:6, 0:10, "missing right summand")
+ ]);
p!("[val: 3mm+4pt*]" =>
- [func!("val": (Add(Sz(Size::mm(3.0)), Pt(4.0))), {})],
+ [func!("val": (Add(Sz(Size::mm(3.0)), Pt(4.0))))],
[(0:10, 0:14, "missing right factor")],
);
}
#[test]
fn parse_tuples() {
- // Empty tuple
- p!("[val: ()]" => [func!("val": (tuple!()), {})]);
- p!("[val: empty()]" => [func!("val": (named_tuple!("empty")), {})]);
+ // Empty tuple.
+ pval!("()" => (tuple!()));
+ pval!("empty()" => (named_tuple!("empty")));
+
+ // Space between name and tuple.
+ pval!("add ( 1 , 2 )" => (named_tuple!("add", Num(1.0), Num(2.0))));
+ pval!("num = add ( 1 , 2 )" => (), {
+ "num" => named_tuple!("add", Num(1.0), Num(2.0))
+ });
- // Invalid value
- p!("[val: (🌎)]" =>
- [func!("val": (tuple!()), {})],
- [(0:7, 0:8, "expected value, found invalid token")],
- );
+ // Invalid value.
p!("[val: sound(\x07)]" =>
[func!("val": (named_tuple!("sound")), {})],
[(0:12, 0:13, "expected value, found invalid token")],
);
- // Invalid tuple name
+ // Invalid tuple name.
p!("[val: πŸ‘ (\"abc\", 13e-5)]" =>
[func!("val": (tuple!(Str("abc"), Num(13.0e-5))), {})],
[(0:6, 0:7, "expected argument, found invalid token")],
);
- // Unclosed tuple
- p!("[val: (hello,]" =>
- [func!("val": (tuple!(Id("hello"),)), {})],
- [(0:13, 0:13, "expected closing paren")],
- );
+ // Unclosed tuple.
p!("[val: lang(δΈ­ζ–‡]" =>
[func!("val": (named_tuple!("lang", Id("δΈ­ζ–‡"))), {})],
[(0:13, 0:13, "expected closing paren")],
);
- // Valid values
- p!("[val: (1, 2)]" => [func!("val": (tuple!(Num(1.0), Num(2.0))), {})]);
- p!("[val: (\"s\",)]" => [func!("val": (tuple!(Str("s"))), {})]);
- p!("[val: cmyk(1, 46, 0, 0)]" =>
- [func!("val": (named_tuple!(
- "cmyk", Num(1.0), Num(46.0), Num(0.0), Num(0.0)
- )), {})]
- );
- p!("[val: items(\"fire\", #f93a6d)]" =>
- [func!("val": (named_tuple!(
- "items", Str("fire"), ClrStr("f93a6d")
- )), {})]
- );
-
- // Nested tuples
- p!("[val: (1, (2, 3))]" =>
- [func!("val": (tuple!(Num(1.0), tuple!(Num(2.0), Num(3.0)))), {})]
- );
- p!("[val: css(1pt, rgb(90, 102, 254), \"solid\")]" =>
- [func!("val": (named_tuple!(
- "css", Pt(1.0), named_tuple!(
- "rgb", Num(90.0), Num(102.0), Num(254.0)
- ), Str("solid")
- )), {})]
- );
-
- // Invalid commas
+ // Valid values.
+ pval!("(1, 2)" => (tuple!(Num(1.0), Num(2.0))));
+ pval!("(\"s\",)" => (tuple!(Str("s"))));
+ pval!("items(\"fire\", #f93a6d)" => (
+ named_tuple!("items", Str("fire"), ColorStr("f93a6d")
+ )));
+
+ // Nested tuples.
+ pval!("css(1pt, rgb(90, 102, 254), \"solid\")" => (named_tuple!(
+ "css",
+ Pt(1.0),
+ named_tuple!("rgb", Num(90.0), Num(102.0), Num(254.0)),
+ Str("solid"),
+ )));
+
+ // Invalid commas.
p!("[val: (,)]" =>
[func!("val": (tuple!()), {})],
[(0:7, 0:8, "expected value, found comma")],
);
- p!("[val: nose(,)]" =>
- [func!("val": (named_tuple!("nose")), {})],
- [(0:11, 0:12, "expected value, found comma")],
- );
p!("[val: (true false)]" =>
[func!("val": (tuple!(Bool(true), Bool(false))), {})],
[(0:11, 0:11, "expected comma")],
@@ -1060,14 +994,13 @@ mod tests {
#[test]
fn parse_objects() {
- let f = || func!("val": (object! {}), {});
+ let val = || func!("val": (object! {}), {});
- // Okay objects
- p!("[val: {}]" => [f()]);
- p!("[val: { key: value }]" =>
- [func!("val": (object! { "key" => Id("value") }), {})]);
+ // Okay objects.
+ pval!("{}" => (object! {}));
+ pval!("{ key: value }" => (object! { "key" => Id("value") }));
- // Unclosed object
+ // Unclosed object.
p!("[val: {hello: world]" =>
[func!("val": (object! { "hello" => Id("world") }), {})],
[(0:19, 0:19, "expected closing brace")],
@@ -1077,14 +1010,14 @@ mod tests {
[(0:9, 0:9, "expected colon"), (0:9, 0:9, "expected closing brace")],
);
- // Missing key
- p!("[val: {,}]" => [f()], [(0:7, 0:8, "expected key, found comma")]);
- p!("[val: { 12pt }]" => [f()], [(0:8, 0:12, "expected key, found size")]);
- p!("[val: { : }]" => [f()], [(0:8, 0:9, "expected key, found colon")]);
+ // Missing key.
+ p!("[val: {,}]" => [val()], [(0:7, 0:8, "expected key, found comma")]);
+ p!("[val: { 12pt }]" => [val()], [(0:8, 0:12, "expected key, found size")]);
+ p!("[val: { : }]" => [val()], [(0:8, 0:9, "expected key, found colon")]);
- // Missing colon
- p!("[val: { key }]" => [f()], [(0:11, 0:11, "expected colon")]);
- p!("[val: { key false }]" => [f()], [
+ // Missing colon.
+ p!("[val: { key }]" => [val()], [(0:11, 0:11, "expected colon")]);
+ p!("[val: { key false }]" => [val()], [
(0:11, 0:11, "expected colon"),
(0:12, 0:17, "expected key, found bool"),
]);
@@ -1093,14 +1026,14 @@ mod tests {
[(0:9, 0:9, "expected colon")],
);
- // Missing value
- p!("[val: { key: : }]" => [f()], [(0:13, 0:14, "expected value, found colon")]);
+ // Missing value.
+ p!("[val: { key: : }]" => [val()], [(0:13, 0:14, "expected value, found colon")]);
p!("[val: { key: , k: \"s\" }]" =>
[func!("val": (object! { "k" => Str("s") }), {})],
[(0:13, 0:14, "expected value, found comma")],
);
- // Missing comma, invalid token
+ // Missing comma, invalid token.
p!("[val: left={ a: 2, b: false 🌎 }]" =>
[func!("val": (), {
"left" => object! {
@@ -1115,7 +1048,7 @@ mod tests {
#[test]
fn parse_nested_tuples_and_objects() {
- p!("[val: (1, { ab: (), d: (3, 14pt) }), false]" => [func!("val": (
+ pval!("(1, { ab: (), d: (3, 14pt) }), false" => (
tuple!(
Num(1.0),
object!(
@@ -1124,7 +1057,7 @@ mod tests {
),
),
Bool(false),
- ), {})]);
+ ));
}
#[test]
@@ -1151,12 +1084,11 @@ mod tests {
#[test]
fn parse_multiple_mixed_arguments() {
- p!("[val: a,]" => [func!("val": (Id("a")), {})]);
p!("[val: 12pt, key=value]" =>
[func!("val": (Pt(12.0)), { "key" => Id("value") })], [],
[(0:12, 0:15, ArgumentKey), (0:1, 0:4, ValidFuncName)],
);
- p!("[val: a , \"b\" , c]" => [func!("val": (Id("a"), Str("b"), Id("c")), {})]);
+ pval!("a , x=\"b\" , c" => (Id("a"), Id("c")), { "x" => Str("b"), });
}
#[test]
@@ -1177,7 +1109,7 @@ mod tests {
#[test]
fn parse_invalid_key_value_pairs() {
- // Invalid keys
+ // Invalid keys.
p!("[val: true=you]" =>
[func!("val": (Bool(true), Id("you")), {})],
[(0:10, 0:10, "expected comma"),
@@ -1185,13 +1117,14 @@ mod tests {
[(0:1, 0:4, ValidFuncName)],
);
+ // Unexpected equals.
p!("[box: z=y=4]" =>
[func!("box": (Num(4.0)), { "z" => Id("y") })],
[(0:9, 0:9, "expected comma"),
(0:9, 0:10, "expected argument, found equals sign")],
);
- // Invalid colon after keyable positional argument
+ // Invalid colon after keyable positional argument.
p!("[val: key:12]" =>
[func!("val": (Id("key"), Num(12.0)), {})],
[(0:9, 0:9, "expected comma"),
@@ -1199,7 +1132,7 @@ mod tests {
[(0:1, 0:4, ValidFuncName)],
);
- // Invalid colon after non-keyable positional argument
+ // Invalid colon after unkeyable positional argument.
p!("[val: true:12]" => [func!("val": (Bool(true), Num(12.0)), {})],
[(0:10, 0:10, "expected comma"),
(0:10, 0:11, "expected argument, found colon")],
@@ -1209,7 +1142,7 @@ mod tests {
#[test]
fn parse_invalid_commas() {
- // Missing commas
+ // Missing commas.
p!("[val: 1pt 1]" =>
[func!("val": (Pt(1.0), Num(1.0)), {})],
[(0:9, 0:9, "expected comma")],
@@ -1219,7 +1152,7 @@ mod tests {
[(0:7, 0:7, "expected comma")],
);
- // Unexpected commas
+ // Unexpected commas.
p!("[val:,]" => [func!("val")], [(0:5, 0:6, "expected argument, found comma")]);
p!("[val: key=,]" => [func!("val")], [(0:10, 0:11, "expected value, found comma")]);
p!("[val:, true]" =>
@@ -1231,13 +1164,10 @@ mod tests {
#[test]
fn parse_bodies() {
p!("[val][Hi]" => [func!("val"; [T("Hi")])]);
-
- // Body nodes in bodies.
p!("[val:*][*Hi*]" =>
[func!("val"; [Bold, T("Hi"), Bold])],
[(0:5, 0:6, "expected argument, found star")],
);
-
// Errors in bodies.
p!(" [val][ */ ]" =>
[S, func!("val"; [S, S])],
@@ -1267,7 +1197,8 @@ mod tests {
(0:6, 0:18, func!((0:1, 0:4, "val"); [(0:6, 0:11, T("world"))])),
(0:18, 0:19, S),
(0:19, 0:20, T("🌎"))
- ], [],
+ ],
+ [],
[(0:7, 0:10, ValidFuncName)],
);
@@ -1281,7 +1212,8 @@ mod tests {
(1:4, 1:10, func!((0:2, 0:5, "box"))),
(1:10, 2:1, S),
]))
- ], [],
+ ],
+ [],
[(0:2, 0:5, ValidFuncName), (1:6, 1:9, ValidFuncName)],
);
}
diff --git a/src/syntax/scope.rs b/src/syntax/scope.rs
index f7b20b9f..74c64280 100644
--- a/src/syntax/scope.rs
+++ b/src/syntax/scope.rs
@@ -5,12 +5,10 @@ use std::fmt::{self, Debug, Formatter};
use crate::Pass;
use crate::func::ParseFunc;
-use super::func::FuncHeader;
-use super::parsing::ParseContext;
-use super::span::Spanned;
+use super::func::FuncCall;
+use super::parsing::ParseState;
use super::Model;
-
/// A map from identifiers to function parsers.
pub struct Scope {
parsers: HashMap<String, Box<Parser>>,
@@ -50,10 +48,8 @@ impl Scope {
}
/// Return the parser with the given name if there is one.
- pub fn get_parser(&self, name: &str) -> Result<&Parser, &Parser> {
- self.parsers.get(name)
- .map(|x| &**x)
- .ok_or_else(|| &*self.fallback)
+ pub fn get_parser(&self, name: &str) -> Option<&Parser> {
+ self.parsers.get(name).map(AsRef::as_ref)
}
/// Return the fallback parser.
@@ -72,16 +68,12 @@ impl Debug for Scope {
/// A function which parses the source of a function into a model type which
/// implements [`Model`].
-type Parser = dyn Fn(
- FuncHeader,
- Option<Spanned<&str>>,
- ParseContext,
-) -> Pass<Box<dyn Model>>;
+type Parser = dyn Fn(FuncCall, &ParseState) -> Pass<Box<dyn Model>>;
fn parser<F>(metadata: <F as ParseFunc>::Meta) -> Box<Parser>
where F: ParseFunc + Model + 'static {
- Box::new(move |h, b, c| {
- F::parse(h, b, c, metadata.clone())
+ Box::new(move |f, s| {
+ F::parse(f, s, metadata.clone())
.map(|model| Box::new(model) as Box<dyn Model>)
})
}
diff --git a/src/syntax/span.rs b/src/syntax/span.rs
index 9eb80d92..c8e2cddb 100644
--- a/src/syntax/span.rs
+++ b/src/syntax/span.rs
@@ -4,65 +4,63 @@ use std::fmt::{self, Debug, Formatter};
use std::ops::{Add, Sub};
use serde::Serialize;
+/// A vector of spanned values of type `T`.
+pub type SpanVec<T> = Vec<Spanned<T>>;
-/// Zero-indexed line-column position in source code.
-#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize)]
-pub struct Position {
- /// The zero-indexed line.
- pub line: usize,
- /// The zero-indexed column.
- pub column: usize,
+/// [Offset](Span::offset) all spans in a vector of spanned things by a start
+/// position.
+pub fn offset_spans<T>(
+ vec: SpanVec<T>,
+ start: Position,
+) -> impl Iterator<Item=Spanned<T>> {
+ vec.into_iter().map(move |s| s.map_span(|span| span.offset(start)))
}
-impl Position {
- /// The line 0, column 0 position.
- pub const ZERO: Position = Position { line: 0, column: 0 };
+/// A value with the span it corresponds to in the source code.
+#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)]
+pub struct Spanned<T> {
+ /// The value.
+ pub v: T,
+ /// The corresponding span.
+ pub span: Span,
+}
- /// Crete a new instance from line and column.
- pub fn new(line: usize, column: usize) -> Position {
- Position { line, column }
+impl<T> Spanned<T> {
+ /// Create a new instance from a value and its span.
+ pub fn new(v: T, span: Span) -> Spanned<T> {
+ Spanned { v, span }
}
-}
-impl Add for Position {
- type Output = Position;
+ /// Create a new instance from a value with the zero span.
+ pub fn zero(v: T) -> Spanned<T> {
+ Spanned { v, span: Span::ZERO }
+ }
- fn add(self, rhs: Position) -> Position {
- if rhs.line == 0 {
- Position {
- line: self.line,
- column: self.column + rhs.column
- }
- } else {
- Position {
- line: self.line + rhs.line,
- column: rhs.column,
- }
- }
+ /// Access the value.
+ pub fn value(self) -> T {
+ self.v
}
-}
-impl Sub for Position {
- type Output = Position;
+ /// Map the value using a function while keeping the span.
+ pub fn map<V, F>(self, f: F) -> Spanned<V> where F: FnOnce(T) -> V {
+ Spanned { v: f(self.v), span: self.span }
+ }
- fn sub(self, rhs: Position) -> Position {
- if self.line == rhs.line {
- Position {
- line: 0,
- column: self.column - rhs.column
- }
- } else {
- Position {
- line: self.line - rhs.line,
- column: self.column,
- }
- }
+ /// Maps the span while keeping the value.
+ pub fn map_span<F>(mut self, f: F) -> Spanned<T> where F: FnOnce(Span) -> Span {
+ self.span = f(self.span);
+ self
}
}
-impl Debug for Position {
+impl<T: Debug> Debug for Spanned<T> {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
- write!(f, "{}:{}", self.line, self.column)
+ self.v.fmt(f)?;
+ if f.alternate() {
+ f.write_str(" ")?;
+ self.span.fmt(f)?;
+ }
+ Ok(())
}
}
@@ -97,6 +95,11 @@ impl Span {
Span { start: pos, end: pos }
}
+ /// Expand a span by merging it with another span.
+ pub fn expand(&mut self, other: Span) {
+ *self = Span::merge(*self, other)
+ }
+
/// Offset a span by a start position.
///
/// This is, for example, used to translate error spans from function local
@@ -111,56 +114,67 @@ impl Span {
impl Debug for Span {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
- write!(f, "({:?} -> {:?})", self.start, self.end)
+ write!(f, "<{:?}-{:?}>", self.start, self.end)
}
}
-/// A value with the span it corresponds to in the source code.
-#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)]
-pub struct Spanned<T> {
- /// The value.
- pub v: T,
- /// The corresponding span.
- pub span: Span,
+/// Zero-indexed line-column position in source code.
+#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize)]
+pub struct Position {
+ /// The zero-indexed line.
+ pub line: usize,
+ /// The zero-indexed column.
+ pub column: usize,
}
-impl<T> Spanned<T> {
- /// Create a new instance from a value and its span.
- pub fn new(v: T, span: Span) -> Spanned<T> {
- Spanned { v, span }
- }
+impl Position {
+ /// The line 0, column 0 position.
+ pub const ZERO: Position = Position { line: 0, column: 0 };
- /// Access the value.
- pub fn value(self) -> T {
- self.v
+ /// Create a new position from line and column.
+ pub fn new(line: usize, column: usize) -> Position {
+ Position { line, column }
}
+}
- /// Map the value using a function while keeping the span.
- pub fn map<V, F>(self, f: F) -> Spanned<V> where F: FnOnce(T) -> V {
- Spanned { v: f(self.v), span: self.span }
- }
+impl Add for Position {
+ type Output = Position;
- /// Maps the span while keeping the value.
- pub fn map_span<F>(mut self, f: F) -> Spanned<T> where F: FnOnce(Span) -> Span {
- self.span = f(self.span);
- self
+ fn add(self, rhs: Position) -> Position {
+ if rhs.line == 0 {
+ Position {
+ line: self.line,
+ column: self.column + rhs.column
+ }
+ } else {
+ Position {
+ line: self.line + rhs.line,
+ column: rhs.column,
+ }
+ }
}
}
-impl<T: Debug> Debug for Spanned<T> {
- fn fmt(&self, f: &mut Formatter) -> fmt::Result {
- self.v.fmt(f)?;
- f.write_str(" ")?;
- self.span.fmt(f)
+impl Sub for Position {
+ type Output = Position;
+
+ fn sub(self, rhs: Position) -> Position {
+ if self.line == rhs.line {
+ Position {
+ line: 0,
+ column: self.column - rhs.column
+ }
+ } else {
+ Position {
+ line: self.line - rhs.line,
+ column: self.column,
+ }
+ }
}
}
-/// A vector of spanned things.
-pub type SpanVec<T> = Vec<Spanned<T>>;
-
-/// [Offset](Span::offset) all spans in a vector of spanned things by a start
-/// position.
-pub fn offset_spans<T>(start: Position, vec: SpanVec<T>) -> impl Iterator<Item=Spanned<T>> {
- vec.into_iter()
- .map(move |s| s.map_span(|span| span.offset(start)))
+impl Debug for Position {
+ fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+ write!(f, "{}:{}", self.line, self.column)
+ }
}
diff --git a/src/syntax/test.rs b/src/syntax/test.rs
index 465f475a..7b1e0830 100644
--- a/src/syntax/test.rs
+++ b/src/syntax/test.rs
@@ -1,17 +1,14 @@
use std::fmt::Debug;
use super::func::FuncHeader;
+use super::span::Spanned;
use super::expr::{Expr, Tuple, NamedTuple, Object};
-use super::span::{Span, Spanned};
-use super::tokens::Token;
use super::*;
-
-/// Check whether the expected and found results for the given source code
-/// match by the comparison function, and print them out otherwise.
-pub fn check<T>(src: &str, exp: T, found: T, spans: bool)
+/// Check whether the expected and found results are the same.
+pub fn check<T>(src: &str, exp: T, found: T, cmp_spans: bool)
where T: Debug + PartialEq + SpanlessEq {
- let cmp = if spans { PartialEq::eq } else { SpanlessEq::spanless_eq };
+ let cmp = if cmp_spans { PartialEq::eq } else { SpanlessEq::spanless_eq };
if !cmp(&exp, &found) {
println!("source: {:?}", src);
println!("expected: {:#?}", exp);
@@ -23,16 +20,25 @@ where T: Debug + PartialEq + SpanlessEq {
/// Create a vector of optionally spanned expressions from a list description.
///
/// # Examples
-/// When you want to add span information to the items, the format is as
-/// follows.
/// ```
+/// // With spans
/// spanned![(0:0, 0:5, "hello"), (0:5, 0:3, "world")]
+///
+/// // Without spans: Implicit zero spans.
+/// spanned!["hello", "world"]
/// ```
-/// The span information can simply be omitted to create a vector with items
-/// that are spanned with zero spans.
-macro_rules! spanned {
- (item ($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)) => ({
- #[allow(unused_imports)]
+macro_rules! span_vec {
+ ($(($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)),* $(,)?) => {
+ (vec![$(span_item!(($sl:$sc, $el:$ec, $v))),*], true)
+ };
+
+ ($($v:expr),* $(,)?) => {
+ (vec![$(span_item!($v)),*], false)
+ };
+}
+
+macro_rules! span_item {
+ (($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)) => ({
use $crate::syntax::span::{Position, Span, Spanned};
Spanned {
span: Span::new(
@@ -43,24 +49,11 @@ macro_rules! spanned {
}
});
- (item $v:expr) => {
- $crate::syntax::test::zspan($v)
- };
-
- (vec $(($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)),* $(,)?) => {
- (vec![$(spanned![item ($sl:$sc, $el:$ec, $v)]),*], true)
- };
-
- (vec $($v:expr),* $(,)?) => {
- (vec![$($crate::syntax::test::zspan($v)),*], false)
+ ($v:expr) => {
+ $crate::syntax::span::Spanned::zero($v)
};
}
-/// Span an element with a zero span.
-pub fn zspan<T>(v: T) -> Spanned<T> {
- Spanned { v, span: Span::ZERO }
-}
-
function! {
/// Most functions in the tests are parsed into the debug function for easy
/// inspection of arguments and body.
@@ -70,13 +63,13 @@ function! {
pub body: Option<SyntaxModel>,
}
- parse(header, body, ctx, f) {
+ parse(header, body, state, f) {
let cloned = header.clone();
header.args.pos.items.clear();
header.args.key.pairs.clear();
DebugFn {
header: cloned,
- body: body!(opt: body, ctx, f),
+ body: body!(opt: body, state, f),
}
}
@@ -120,8 +113,8 @@ impl SpanlessEq for DebugFn {
impl SpanlessEq for Expr {
fn spanless_eq(&self, other: &Expr) -> bool {
match (self, other) {
- (Expr::NamedTuple(a), Expr::NamedTuple(b)) => a.spanless_eq(b),
(Expr::Tuple(a), Expr::Tuple(b)) => a.spanless_eq(b),
+ (Expr::NamedTuple(a), Expr::NamedTuple(b)) => a.spanless_eq(b),
(Expr::Object(a), Expr::Object(b)) => a.spanless_eq(b),
(Expr::Neg(a), Expr::Neg(b)) => a.spanless_eq(&b),
(Expr::Add(a1, a2), Expr::Add(b1, b2)) => a1.spanless_eq(&b1) && a2.spanless_eq(&b2),
@@ -175,8 +168,7 @@ impl<T: SpanlessEq> SpanlessEq for Box<T> {
}
}
-/// Implement `SpanlessEq` by just forwarding to `PartialEq`.
-macro_rules! forward {
+macro_rules! impl_through_partial_eq {
($type:ty) => {
impl SpanlessEq for $type {
fn spanless_eq(&self, other: &$type) -> bool {
@@ -186,6 +178,8 @@ macro_rules! forward {
};
}
-forward!(String);
-forward!(Token<'_>);
-forward!(Decoration);
+impl_through_partial_eq!(Token<'_>);
+
+// Implement for string and decoration to be able to compare feedback.
+impl_through_partial_eq!(String);
+impl_through_partial_eq!(Decoration);
diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs
index 3b34019d..10200708 100644
--- a/src/syntax/tokens.rs
+++ b/src/syntax/tokens.rs
@@ -5,9 +5,8 @@ use unicode_xid::UnicodeXID;
use crate::size::Size;
use super::span::{Position, Span, Spanned};
-use self::Token::*;
-use self::TokenizationMode::*;
-
+use Token::*;
+use TokenMode::*;
/// A minimal semantic entity of source code.
#[derive(Debug, Copy, Clone, PartialEq)]
@@ -152,7 +151,7 @@ impl<'s> Token<'s> {
#[derive(Debug)]
pub struct Tokens<'s> {
src: &'s str,
- mode: TokenizationMode,
+ mode: TokenMode,
iter: Peekable<Chars<'s>>,
position: Position,
index: usize,
@@ -163,20 +162,22 @@ pub struct Tokens<'s> {
/// backtick tokens.
#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
#[allow(missing_docs)]
-pub enum TokenizationMode {
+pub enum TokenMode {
Header,
Body,
}
impl<'s> Tokens<'s> {
- /// Create a new token iterator with the given mode where the first token
- /// span starts an the given `start` position.
- pub fn new(start: Position, src: &'s str, mode: TokenizationMode) -> Tokens<'s> {
+ /// Create a new token iterator with the given mode.
+ ///
+ /// The first token's span starts an the given `offset` position instead of
+ /// the zero position.
+ pub fn new(src: &'s str, offset: Position, mode: TokenMode) -> Tokens<'s> {
Tokens {
src,
mode,
iter: src.chars().peekable(),
- position: start,
+ position: offset,
index: 0,
}
}
@@ -188,7 +189,7 @@ impl<'s> Tokens<'s> {
}
/// The line-colunn position in the source at which the last token ends and
- /// next token will start. This position is
+ /// next token will start.
pub fn pos(&self) -> Position {
self.position
}
@@ -204,15 +205,15 @@ impl<'s> Iterator for Tokens<'s> {
let token = match first {
// Comments.
- '/' if self.peek() == Some('/') => self.parse_line_comment(),
- '/' if self.peek() == Some('*') => self.parse_block_comment(),
+ '/' if self.peek() == Some('/') => self.read_line_comment(),
+ '/' if self.peek() == Some('*') => self.read_block_comment(),
'*' if self.peek() == Some('/') => { self.eat(); Invalid("*/") }
// Whitespace.
- c if c.is_whitespace() => self.parse_whitespace(start),
+ c if c.is_whitespace() => self.read_whitespace(start),
// Functions.
- '[' => self.parse_function(start),
+ '[' => self.read_function(start),
']' => Invalid("]"),
// Syntactic elements in function headers.
@@ -230,7 +231,7 @@ impl<'s> Iterator for Tokens<'s> {
'/' if self.mode == Header => Slash,
// String values.
- '"' if self.mode == Header => self.parse_string(),
+ '"' if self.mode == Header => self.read_string(),
// Star serves a double purpose as a style modifier
// and a expression operator in the header.
@@ -238,13 +239,13 @@ impl<'s> Iterator for Tokens<'s> {
// Style toggles.
'_' if self.mode == Body => Underscore,
- '`' if self.mode == Body => self.parse_raw(),
+ '`' if self.mode == Body => self.read_raw(),
// An escaped thing.
- '\\' if self.mode == Body => self.parse_escaped(),
+ '\\' if self.mode == Body => self.read_escaped(),
// A hex expression.
- '#' if self.mode == Header => self.parse_hex_value(),
+ '#' if self.mode == Header => self.read_hex(),
// Expressions or just strings.
c => {
@@ -267,7 +268,7 @@ impl<'s> Iterator for Tokens<'s> {
}, false, -(c.len_utf8() as isize), 0).0;
if self.mode == Header {
- self.parse_expr(text)
+ self.read_expr(text)
} else {
Text(text)
}
@@ -282,11 +283,11 @@ impl<'s> Iterator for Tokens<'s> {
}
impl<'s> Tokens<'s> {
- fn parse_line_comment(&mut self) -> Token<'s> {
+ fn read_line_comment(&mut self) -> Token<'s> {
LineComment(self.read_string_until(is_newline_char, false, 1, 0).0)
}
- fn parse_block_comment(&mut self) -> Token<'s> {
+ fn read_block_comment(&mut self) -> Token<'s> {
enum Last { Slash, Star, Other }
self.eat();
@@ -314,14 +315,14 @@ impl<'s> Tokens<'s> {
}, true, 0, -2).0)
}
- fn parse_whitespace(&mut self, start: Position) -> Token<'s> {
+ fn read_whitespace(&mut self, start: Position) -> Token<'s> {
self.read_string_until(|n| !n.is_whitespace(), false, 0, 0);
let end = self.pos();
Space(end.line - start.line)
}
- fn parse_function(&mut self, start: Position) -> Token<'s> {
+ fn read_function(&mut self, start: Position) -> Token<'s> {
let (header, terminated) = self.read_function_part(Header);
self.eat();
@@ -341,7 +342,7 @@ impl<'s> Tokens<'s> {
Function { header, body: Some(Spanned { v: body, span }), terminated }
}
- fn read_function_part(&mut self, mode: TokenizationMode) -> (&'s str, bool) {
+ fn read_function_part(&mut self, mode: TokenMode) -> (&'s str, bool) {
let start = self.index();
let mut terminated = false;
@@ -353,11 +354,11 @@ impl<'s> Tokens<'s> {
self.eat();
match n {
- '[' => { self.parse_function(Position::ZERO); }
- '/' if self.peek() == Some('/') => { self.parse_line_comment(); }
- '/' if self.peek() == Some('*') => { self.parse_block_comment(); }
- '"' if mode == Header => { self.parse_string(); }
- '`' if mode == Body => { self.parse_raw(); }
+ '[' => { self.read_function(Position::ZERO); }
+ '/' if self.peek() == Some('/') => { self.read_line_comment(); }
+ '/' if self.peek() == Some('*') => { self.read_block_comment(); }
+ '"' if mode == Header => { self.read_string(); }
+ '`' if mode == Body => { self.read_raw(); }
'\\' => { self.eat(); }
_ => {}
}
@@ -367,12 +368,12 @@ impl<'s> Tokens<'s> {
(&self.src[start .. end], terminated)
}
- fn parse_string(&mut self) -> Token<'s> {
+ fn read_string(&mut self) -> Token<'s> {
let (string, terminated) = self.read_until_unescaped('"');
ExprStr { string, terminated }
}
- fn parse_raw(&mut self) -> Token<'s> {
+ fn read_raw(&mut self) -> Token<'s> {
let (raw, terminated) = self.read_until_unescaped('`');
Raw { raw, terminated }
}
@@ -390,7 +391,7 @@ impl<'s> Tokens<'s> {
}, true, 0, -1)
}
- fn parse_escaped(&mut self) -> Token<'s> {
+ fn read_escaped(&mut self) -> Token<'s> {
fn is_escapable(c: char) -> bool {
match c {
'[' | ']' | '\\' | '/' | '*' | '_' | '`' | '"' => true,
@@ -410,7 +411,7 @@ impl<'s> Tokens<'s> {
}
}
- fn parse_hex_value(&mut self) -> Token<'s> {
+ fn read_hex(&mut self) -> Token<'s> {
// This will parse more than the permissable 0-9, a-f, A-F character
// ranges to provide nicer error messages later.
ExprHex(self.read_string_until(
@@ -419,7 +420,7 @@ impl<'s> Tokens<'s> {
).0)
}
- fn parse_expr(&mut self, text: &'s str) -> Token<'s> {
+ fn read_expr(&mut self, text: &'s str) -> Token<'s> {
if let Ok(b) = text.parse::<bool>() {
ExprBool(b)
} else if let Ok(num) = text.parse::<f64>() {
@@ -435,8 +436,11 @@ impl<'s> Tokens<'s> {
}
}
- /// Will read the input stream until the argument F evaluates to `true`
- /// for the current character.
+ /// Will read the input stream until `f` evaluates to `true`. When
+ /// `eat_match` is true, the token for which `f` was true is consumed.
+ /// Returns the string from the index where this was called offset by
+ /// `offset_start` to the end offset by `offset_end`. The end is before or
+ /// after the match depending on `eat_match`.
fn read_string_until<F>(
&mut self,
mut f: F,
@@ -527,8 +531,8 @@ pub fn is_identifier(string: &str) -> bool {
true
}
-
#[cfg(test)]
+#[allow(non_snake_case)]
mod tests {
use super::super::test::check;
use super::*;
@@ -549,31 +553,23 @@ mod tests {
Slash,
};
- #[allow(non_snake_case)]
- fn Str(string: &'static str, terminated: bool) -> Token<'static> {
- Token::ExprStr { string, terminated }
- }
-
- #[allow(non_snake_case)]
- fn Raw(raw: &'static str, terminated: bool) -> Token<'static> {
- Token::Raw { raw, terminated }
- }
-
/// Test whether the given string tokenizes into the given list of tokens.
macro_rules! t {
($mode:expr, $source:expr => [$($tokens:tt)*]) => {
- let (exp, spans) = spanned![vec $($tokens)*];
- let found = Tokens::new(Position::ZERO, $source, $mode).collect::<Vec<_>>();
+ let (exp, spans) = span_vec![$($tokens)*];
+ let found = Tokens::new($source, Position::ZERO, $mode).collect::<Vec<_>>();
check($source, exp, found, spans);
}
}
- /// Write down a function token compactly.
+ fn Str(string: &str, terminated: bool) -> Token { Token::ExprStr { string, terminated } }
+ fn Raw(raw: &str, terminated: bool) -> Token { Token::Raw { raw, terminated } }
+
macro_rules! func {
($header:expr, Some($($tokens:tt)*), $terminated:expr) => {
Function {
header: $header,
- body: Some(spanned![item $($tokens)*]),
+ body: Some(span_item!(($($tokens)*))),
terminated: $terminated,
}
};
@@ -674,12 +670,12 @@ mod tests {
fn tokenize_functions() {
t!(Body, "a[f]" => [T("a"), func!("f", None, true)]);
t!(Body, "[f]a" => [func!("f", None, true), T("a")]);
- t!(Body, "\n\n[f][ ]" => [S(2), func!("f", Some((0:4, 0:5, " ")), true)]);
- t!(Body, "abc [f][ ]a" => [T("abc"), S(0), func!("f", Some((0:4, 0:5, " ")), true), T("a")]);
+ t!(Body, "\n\n[f][ ]" => [S(2), func!("f", Some(0:4, 0:5, " "), true)]);
+ t!(Body, "abc [f][ ]a" => [T("abc"), S(0), func!("f", Some(0:4, 0:5, " "), true), T("a")]);
t!(Body, "[f: [=][*]]" => [func!("f: [=][*]", None, true)]);
- t!(Body, "[_][[,],]," => [func!("_", Some((0:4, 0:8, "[,],")), true), T(",")]);
- t!(Body, "[=][=][=]" => [func!("=", Some((0:4, 0:5, "=")), true), func!("=", None, true)]);
- t!(Body, "[=][[=][=][=]]" => [func!("=", Some((0:4, 0:13, "[=][=][=]")), true)]);
+ t!(Body, "[_][[,],]," => [func!("_", Some(0:4, 0:8, "[,],"), true), T(",")]);
+ t!(Body, "[=][=][=]" => [func!("=", Some(0:4, 0:5, "="), true), func!("=", None, true)]);
+ t!(Body, "[=][[=][=][=]]" => [func!("=", Some(0:4, 0:13, "[=][=][=]"), true)]);
t!(Header, "[" => [func!("", None, false)]);
t!(Header, "]" => [Invalid("]")]);
}
@@ -693,25 +689,25 @@ mod tests {
t!(Body, "[f: `]" => [func!("f: `", None, true)]);
// End of function with strings and carets in bodies
- t!(Body, "[f][\"]" => [func!("f", Some((0:4, 0:5, "\"")), true)]);
- t!(Body, r#"[f][\"]"# => [func!("f", Some((0:4, 0:6, r#"\""#)), true)]);
- t!(Body, "[f][`]" => [func!("f", Some((0:4, 0:6, "`]")), false)]);
- t!(Body, "[f][\\`]" => [func!("f", Some((0:4, 0:6, "\\`")), true)]);
- t!(Body, "[f][`raw`]" => [func!("f", Some((0:4, 0:9, "`raw`")), true)]);
- t!(Body, "[f][`raw]" => [func!("f", Some((0:4, 0:9, "`raw]")), false)]);
- t!(Body, "[f][`raw]`]" => [func!("f", Some((0:4, 0:10, "`raw]`")), true)]);
- t!(Body, "[f][`\\`]" => [func!("f", Some((0:4, 0:8, "`\\`]")), false)]);
- t!(Body, "[f][`\\\\`]" => [func!("f", Some((0:4, 0:8, "`\\\\`")), true)]);
+ t!(Body, "[f][\"]" => [func!("f", Some(0:4, 0:5, "\""), true)]);
+ t!(Body, r#"[f][\"]"# => [func!("f", Some(0:4, 0:6, r#"\""#), true)]);
+ t!(Body, "[f][`]" => [func!("f", Some(0:4, 0:6, "`]"), false)]);
+ t!(Body, "[f][\\`]" => [func!("f", Some(0:4, 0:6, "\\`"), true)]);
+ t!(Body, "[f][`raw`]" => [func!("f", Some(0:4, 0:9, "`raw`"), true)]);
+ t!(Body, "[f][`raw]" => [func!("f", Some(0:4, 0:9, "`raw]"), false)]);
+ t!(Body, "[f][`raw]`]" => [func!("f", Some(0:4, 0:10, "`raw]`"), true)]);
+ t!(Body, "[f][`\\`]" => [func!("f", Some(0:4, 0:8, "`\\`]"), false)]);
+ t!(Body, "[f][`\\\\`]" => [func!("f", Some(0:4, 0:8, "`\\\\`"), true)]);
// End of function with comments
- t!(Body, "[f][/*]" => [func!("f", Some((0:4, 0:7, "/*]")), false)]);
- t!(Body, "[f][/*`*/]" => [func!("f", Some((0:4, 0:9, "/*`*/")), true)]);
+ t!(Body, "[f][/*]" => [func!("f", Some(0:4, 0:7, "/*]"), false)]);
+ t!(Body, "[f][/*`*/]" => [func!("f", Some(0:4, 0:9, "/*`*/"), true)]);
t!(Body, "[f: //]\n]" => [func!("f: //]\n", None, true)]);
t!(Body, "[f: \"//]\n]" => [func!("f: \"//]\n]", None, false)]);
// End of function with escaped brackets
- t!(Body, "[f][\\]]" => [func!("f", Some((0:4, 0:6, "\\]")), true)]);
- t!(Body, "[f][\\[]" => [func!("f", Some((0:4, 0:6, "\\[")), true)]);
+ t!(Body, "[f][\\]]" => [func!("f", Some(0:4, 0:6, "\\]"), true)]);
+ t!(Body, "[f][\\[]" => [func!("f", Some(0:4, 0:6, "\\["), true)]);
}
#[test]