summaryrefslogtreecommitdiff
path: root/src/syntax/parsing.rs
diff options
context:
space:
mode:
authorLaurenz <laurmaedje@gmail.com>2020-07-29 17:46:57 +0200
committerGitHub <noreply@github.com>2020-07-29 17:46:57 +0200
commitf34ba3dcda182d9b9c14cc94fdb48810bf18bef0 (patch)
tree667a7aba2f26996c7ada8ce85952c384a1dbd5a1 /src/syntax/parsing.rs
parente7ffdde43d09f60238590723c2829554806e23d5 (diff)
parent9672d4320052d08b67d497febed4a0ad78bf9252 (diff)
Merge pull request #7 from typst/parser-update
Parser update
Diffstat (limited to 'src/syntax/parsing.rs')
-rw-r--r--src/syntax/parsing.rs1162
1 files changed, 547 insertions, 615 deletions
diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs
index 09405d7f..a0d9c4e4 100644
--- a/src/syntax/parsing.rs
+++ b/src/syntax/parsing.rs
@@ -1,52 +1,45 @@
//! Parsing of source code into syntax models.
-use std::iter::FromIterator;
use std::str::FromStr;
-use crate::{Pass, Feedback};
-use super::func::{FuncHeader, FuncArgs, FuncArg};
use super::expr::*;
-use super::scope::Scope;
+use super::func::{FuncCall, FuncHeader, FuncArgs, FuncArg};
use super::span::{Position, Span, Spanned};
-use super::tokens::{Token, Tokens, TokenizationMode};
use super::*;
-
-/// The context for parsing.
-#[derive(Debug, Copy, Clone)]
-pub struct ParseContext<'a> {
- /// The scope containing function definitions.
- pub scope: &'a Scope,
+/// The state which can influence how a string of source code is parsed.
+///
+/// Parsing is pure - when passed in the same state and source code, the output
+/// must be the same.
+pub struct ParseState {
+ /// The scope containing all function definitions.
+ pub scope: Scope,
}
-/// Parse source code into a syntax model.
+/// Parse a string of source code.
///
-/// All errors and decorations are offset by the `start` position.
-pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Pass<SyntaxModel> {
+/// All spans in the resulting model and feedback are offset by the given
+/// `offset` position. This is used to make spans of a function body relative to
+/// the start of the function as a whole as opposed to the start of the
+/// function's body.
+pub fn parse(src: &str, offset: Position, state: &ParseState) -> Pass<SyntaxModel> {
let mut model = SyntaxModel::new();
let mut feedback = Feedback::new();
- // We always start in body mode. The header tokenization mode is only used
- // in the `FuncParser`.
- let mut tokens = Tokens::new(start, src, TokenizationMode::Body);
-
- while let Some(token) = tokens.next() {
+ for token in Tokens::new(src, offset, TokenMode::Body) {
let span = token.span;
-
let node = match token.v {
- Token::LineComment(_) | Token::BlockComment(_) => continue,
-
- // Only at least two newlines mean a _real_ newline indicating a
- // paragraph break.
+ // Starting from two newlines counts as a paragraph break, a single
+ // newline does not.
Token::Space(newlines) => if newlines >= 2 {
Node::Parbreak
} else {
Node::Space
- },
+ }
Token::Function { header, body, terminated } => {
- let parsed = FuncParser::new(header, body, ctx).parse();
- feedback.extend_offset(span.start, parsed.feedback);
+ let parsed = FuncParser::new(header, body, state).parse();
+ feedback.extend_offset(parsed.feedback, span.start);
if !terminated {
error!(@feedback, Span::at(span.end), "expected closing bracket");
@@ -55,9 +48,9 @@ pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Pass<SyntaxModel>
parsed.output
}
- Token::Star => Node::ToggleBolder,
+ Token::Star => Node::ToggleBolder,
Token::Underscore => Node::ToggleItalic,
- Token::Backslash => Node::Linebreak,
+ Token::Backslash => Node::Linebreak,
Token::Raw { raw, terminated } => {
if !terminated {
@@ -69,103 +62,101 @@ pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Pass<SyntaxModel>
Token::Text(text) => Node::Text(text.to_string()),
- other => {
- error!(@feedback, span, "unexpected {}", other.name());
+ Token::LineComment(_) | Token::BlockComment(_) => continue,
+ unexpected => {
+ error!(@feedback, span, "unexpected {}", unexpected.name());
continue;
}
};
- model.add(Spanned { v: node, span: token.span });
+ model.add(Spanned::new(node, span));
}
Pass::new(model, feedback)
}
-/// Performs the function parsing.
struct FuncParser<'s> {
- ctx: ParseContext<'s>,
- feedback: Feedback,
-
+ state: &'s ParseState,
/// ```typst
/// [tokens][body]
/// ^^^^^^
/// ```
tokens: Tokens<'s>,
peeked: Option<Option<Spanned<Token<'s>>>>,
-
/// The spanned body string if there is a body.
/// ```typst
/// [tokens][body]
/// ^^^^
/// ```
body: Option<Spanned<&'s str>>,
+ feedback: Feedback,
}
impl<'s> FuncParser<'s> {
- /// Create a new function parser.
fn new(
header: &'s str,
body: Option<Spanned<&'s str>>,
- ctx: ParseContext<'s>
+ state: &'s ParseState,
) -> FuncParser<'s> {
FuncParser {
- ctx,
- feedback: Feedback::new(),
- tokens: Tokens::new(Position::new(0, 1), header, TokenizationMode::Header),
+ state,
+ // Start at column 1 because the opening bracket is also part of
+ // the function, but not part of the `header` string.
+ tokens: Tokens::new(header, Position::new(0, 1), TokenMode::Header),
peeked: None,
body,
+ feedback: Feedback::new(),
}
}
- /// Do the parsing.
fn parse(mut self) -> Pass<Node> {
- let parsed = if let Some(header) = self.parse_func_header() {
+ let (parser, header) = if let Some(header) = self.parse_func_header() {
let name = header.name.v.as_str();
- let (parser, deco) = match self.ctx.scope.get_parser(name) {
- // A valid function.
- Ok(parser) => (parser, Decoration::ValidFuncName),
-
- // The fallback parser was returned. Invalid function.
- Err(parser) => {
+ let (parser, deco) = match self.state.scope.get_parser(name) {
+ // The function exists in the scope.
+ Some(parser) => (parser, Decoration::ValidFuncName),
+
+ // The function does not exist in the scope. The parser that is
+ // returned here is a fallback parser which exists to make sure
+ // the content of the function is not totally dropped (on a best
+ // effort basis).
+ None => {
error!(@self.feedback, header.name.span, "unknown function");
+ let parser = self.state.scope.get_fallback_parser();
(parser, Decoration::InvalidFuncName)
}
};
self.feedback.decos.push(Spanned::new(deco, header.name.span));
-
- parser(header, self.body, self.ctx)
+ (parser, header)
} else {
- let default = FuncHeader {
- name: Spanned::new(Ident("".to_string()), Span::ZERO),
+ // Parse the body with the fallback parser even when the header is
+ // completely unparsable.
+ let parser = self.state.scope.get_fallback_parser();
+ let header = FuncHeader {
+ name: Spanned::new(Ident(String::new()), Span::ZERO),
args: FuncArgs::new(),
};
-
- // Use the fallback function such that the body is still rendered
- // even if the header is completely unparsable.
- self.ctx.scope.get_fallback_parser()(default, self.body, self.ctx)
+ (parser, header)
};
- self.feedback.extend(parsed.feedback);
+ let call = FuncCall { header, body: self.body };
+ let parsed = parser(call, self.state);
+ self.feedback.extend(parsed.feedback);
Pass::new(Node::Model(parsed.output), self.feedback)
}
- /// Parse the header tokens.
fn parse_func_header(&mut self) -> Option<FuncHeader> {
- let start = self.pos();
- self.skip_whitespace();
-
- let name = match self.parse_ident() {
- Some(ident) => ident,
- None => {
- let other = self.eat();
- self.expected_found_or_at("identifier", other, start);
- return None;
- }
- };
+ let after_bracket = self.pos();
- self.skip_whitespace();
+ self.skip_white();
+ let name = try_opt_or!(self.parse_ident(), {
+ self.expected_found_or_at("function name", after_bracket);
+ return None;
+ });
+
+ self.skip_white();
let args = match self.eat().map(Spanned::value) {
Some(Token::Colon) => self.parse_func_args(),
Some(_) => {
@@ -178,58 +169,74 @@ impl<'s> FuncParser<'s> {
Some(FuncHeader { name, args })
}
- /// Parse the argument list between colons and end of the header.
fn parse_func_args(&mut self) -> FuncArgs {
- // Parse a collection until the token is `None`, that is, the end of the
- // header.
- self.parse_collection(None, |p| {
- // If we have an identifier we might have a keyword argument,
- // otherwise its for sure a postional argument.
- if let Some(ident) = p.parse_ident() {
- // This could still be a named tuple
- if let Some(Token::LeftParen) = p.peekv() {
- let tuple = p.parse_named_tuple(ident);
- return Ok(tuple.map(|t| FuncArg::Pos(Expr::NamedTuple(t))));
- }
-
- p.skip_whitespace();
+ let mut args = FuncArgs::new();
+ loop {
+ self.skip_white();
+ if self.eof() {
+ break;
+ }
- if let Some(Token::Equals) = p.peekv() {
- p.eat();
- p.skip_whitespace();
+ let arg = if let Some(ident) = self.parse_ident() {
+ self.skip_white();
- // Semantic highlighting for argument keys.
- p.feedback.decos.push(
- Spanned::new(Decoration::ArgumentKey, ident.span));
+ // This could be a keyword argument, or a positional argument of
+ // type named tuple or identifier.
+ if self.check_eat(Token::Equals).is_some() {
+ self.skip_white();
- let value = p.parse_expr().ok_or(("value", None))?;
+ let key = ident;
+ self.feedback.decos.push(
+ Spanned::new(Decoration::ArgumentKey, key.span)
+ );
- // Add a keyword argument.
- let span = Span::merge(ident.span, value.span);
- let pair = Pair { key: ident, value };
- Ok(Spanned::new(FuncArg::Key(pair), span))
+ let value = try_opt_or!(self.parse_expr(), {
+ self.expected("value");
+ continue;
+ });
+
+ let span = Span::merge(key.span, value.span);
+ let arg = FuncArg::Key(Pair { key, value });
+ Spanned::new(arg, span)
+ } else if self.check(Token::LeftParen) {
+ let tuple = self.parse_named_tuple(ident);
+ tuple.map(|tup| FuncArg::Pos(Expr::NamedTuple(tup)))
} else {
- // Add a positional argument because there was no equals
- // sign after the identifier that could have been a key.
- Ok(ident.map(|id| FuncArg::Pos(Expr::Ident(id))))
+ ident.map(|id| FuncArg::Pos(Expr::Ident(id)))
}
} else {
- // Add a positional argument because we haven't got an
- // identifier that could be an argument key.
- let value = p.parse_expr().ok_or(("argument", None))?;
- Ok(value.map(|expr| FuncArg::Pos(expr)))
+ // It's a positional argument.
+ try_opt_or!(self.parse_expr(), {
+ self.expected("argument");
+ continue;
+ }).map(|expr| FuncArg::Pos(expr))
+ };
+
+ let behind_arg = arg.span.end;
+ args.add(arg);
+
+ self.skip_white();
+ if self.eof() {
+ break;
}
- }).v
+
+ self.expect_at(Token::Comma, behind_arg);
+ }
+ args
+ }
+}
+
+// Parsing expressions and values
+impl FuncParser<'_> {
+ fn parse_ident(&mut self) -> Option<Spanned<Ident>> {
+ self.peek().and_then(|token| match token.v {
+ Token::ExprIdent(id) => self.eat_span(Ident(id.to_string())),
+ _ => None,
+ })
}
- /// Parse an expression which may contain math operands. For this, this
- /// method looks for operators in descending order of associativity, i.e. we
- /// first drill down to find all negations, brackets and tuples, the next
- /// level, we look for multiplication and division and here finally, for
- /// addition and subtraction.
fn parse_expr(&mut self) -> Option<Spanned<Expr>> {
- let o1 = self.parse_term()?;
- self.parse_binop(o1, "summand", Self::parse_expr, |token| match token {
+ self.parse_binops("summand", Self::parse_term, |token| match token {
Token::Plus => Some(Expr::Add),
Token::Hyphen => Some(Expr::Sub),
_ => None,
@@ -237,60 +244,57 @@ impl<'s> FuncParser<'s> {
}
fn parse_term(&mut self) -> Option<Spanned<Expr>> {
- let o1 = self.parse_factor()?;
- self.parse_binop(o1, "factor", Self::parse_term, |token| match token {
+ self.parse_binops("factor", Self::parse_factor, |token| match token {
Token::Star => Some(Expr::Mul),
Token::Slash => Some(Expr::Div),
_ => None,
})
}
- fn parse_binop<F, G>(
+ /// Parse expression of the form `<operand> (<op> <operand>)*`.
+ fn parse_binops(
&mut self,
- o1: Spanned<Expr>,
operand_name: &str,
- parse_operand: F,
- parse_op: G,
- ) -> Option<Spanned<Expr>>
- where
- F: FnOnce(&mut Self) -> Option<Spanned<Expr>>,
- G: FnOnce(Token) -> Option<fn(Box<Spanned<Expr>>, Box<Spanned<Expr>>) -> Expr>,
- {
- self.skip_whitespace();
-
- if let Some(next) = self.peek() {
- if let Some(binop) = parse_op(next.v) {
+ mut parse_operand: impl FnMut(&mut Self) -> Option<Spanned<Expr>>,
+ mut parse_op: impl FnMut(Token) -> Option<
+ fn(Box<Spanned<Expr>>, Box<Spanned<Expr>>) -> Expr
+ >,
+ ) -> Option<Spanned<Expr>> {
+ let mut left = parse_operand(self)?;
+
+ self.skip_white();
+ while let Some(token) = self.peek() {
+ if let Some(op) = parse_op(token.v) {
self.eat();
- self.skip_whitespace();
-
- if let Some(o2) = parse_operand(self) {
- let span = Span::merge(o1.span, o2.span);
- let expr = binop(Box::new(o1), Box::new(o2));
- return Some(Spanned::new(expr, span));
- } else {
- error!(
- @self.feedback, Span::merge(next.span, o1.span),
- "missing right {}", operand_name,
- );
+ self.skip_white();
+
+ if let Some(right) = parse_operand(self) {
+ let span = Span::merge(left.span, right.span);
+ let v = op(Box::new(left), Box::new(right));
+ left = Spanned::new(v, span);
+ self.skip_white();
+ continue;
}
+
+ error!(
+ @self.feedback, Span::merge(left.span, token.span),
+ "missing right {}", operand_name,
+ );
}
+ break;
}
- Some(o1)
+ Some(left)
}
- /// Parse expressions that are of the form value or -value.
fn parse_factor(&mut self) -> Option<Spanned<Expr>> {
- let first = self.peek()?;
- if first.v == Token::Hyphen {
- self.eat();
- self.skip_whitespace();
-
- if let Some(factor) = self.parse_value() {
- let span = Span::merge(first.span, factor.span);
- Some(Spanned::new(Expr::Neg(Box::new(factor)), span))
+ if let Some(hyph) = self.check_eat(Token::Hyphen) {
+ self.skip_white();
+ if let Some(value) = self.parse_value() {
+ let span = Span::merge(hyph.span, value.span);
+ Some(Spanned::new(Expr::Neg(Box::new(value)), span))
} else {
- error!(@self.feedback, first.span, "dangling minus");
+ error!(@self.feedback, hyph.span, "dangling minus");
None
}
} else {
@@ -299,283 +303,247 @@ impl<'s> FuncParser<'s> {
}
fn parse_value(&mut self) -> Option<Spanned<Expr>> {
- let first = self.peek()?;
- macro_rules! take {
- ($v:expr) => ({ self.eat(); Spanned { v: $v, span: first.span } });
- }
-
- Some(match first.v {
- Token::ExprIdent(i) => {
- let name = take!(Ident(i.to_string()));
-
- // This could be a named tuple or an identifier
- if let Some(Token::LeftParen) = self.peekv() {
- self.parse_named_tuple(name).map(|t| Expr::NamedTuple(t))
+ let Spanned { v: token, span } = self.peek()?;
+ match token {
+ // This could be a named tuple or an identifier.
+ Token::ExprIdent(id) => {
+ let name = Spanned::new(Ident(id.to_string()), span);
+ self.eat();
+ self.skip_white();
+ Some(if self.check(Token::LeftParen) {
+ self.parse_named_tuple(name).map(|tup| Expr::NamedTuple(tup))
} else {
- name.map(|i| Expr::Ident(i))
- }
- },
+ name.map(|id| Expr::Ident(id))
+ })
+ }
+
Token::ExprStr { string, terminated } => {
if !terminated {
- self.expected_at("quote", first.span.end);
+ self.expected_at("quote", span.end);
}
-
- take!(Expr::Str(unescape_string(string)))
+ self.eat_span(Expr::Str(unescape_string(string)))
}
- Token::ExprNumber(n) => take!(Expr::Number(n)),
- Token::ExprSize(s) => take!(Expr::Size(s)),
- Token::ExprBool(b) => take!(Expr::Bool(b)),
+ Token::ExprNumber(n) => self.eat_span(Expr::Number(n)),
+ Token::ExprSize(s) => self.eat_span(Expr::Size(s)),
+ Token::ExprBool(b) => self.eat_span(Expr::Bool(b)),
Token::ExprHex(s) => {
if let Ok(color) = RgbaColor::from_str(s) {
- take!(Expr::Color(color))
+ self.eat_span(Expr::Color(color))
} else {
- // Heal color by assuming black
- error!(@self.feedback, first.span, "invalid color");
- take!(Expr::Color(RgbaColor::new_healed(0, 0, 0, 255)))
+ // Heal color by assuming black.
+ error!(@self.feedback, span, "invalid color");
+ let healed = RgbaColor::new_healed(0, 0, 0, 255);
+ self.eat_span(Expr::Color(healed))
}
},
+ // This could be a tuple or a parenthesized expression. We parse as
+ // a tuple in any case and coerce the tuple into a value if it is
+ // coercable (length 1 and no trailing comma).
Token::LeftParen => {
- let (mut tuple, can_be_coerced) = self.parse_tuple();
- // Coerce 1-tuple into value
- if can_be_coerced && tuple.v.items.len() > 0 {
- tuple.v.items.pop().expect("length is at least one")
+ let (mut tuple, coercable) = self.parse_tuple();
+ Some(if coercable {
+ tuple.v.items.pop().expect("tuple is coercable")
} else {
- tuple.map(|t| Expr::Tuple(t))
- }
- },
- Token::LeftBrace => self.parse_object().map(|o| Expr::Object(o)),
-
- _ => return None,
- })
- }
-
- /// Parse a tuple expression: `(<expr>, ...)`. The boolean in the return
- /// values showes whether the tuple can be coerced into a single value.
- fn parse_tuple(&mut self) -> (Spanned<Tuple>, bool) {
- let token = self.eat();
- debug_assert_eq!(token.map(Spanned::value), Some(Token::LeftParen));
+ tuple.map(|tup| Expr::Tuple(tup))
+ })
+ }
+ Token::LeftBrace => {
+ Some(self.parse_object().map(|obj| Expr::Object(obj)))
+ }
- // Parse a collection until a right paren appears and complain about
- // missing a `value` when an invalid token is encoutered.
- self.parse_collection_comma_aware(Some(Token::RightParen),
- |p| p.parse_expr().ok_or(("value", None)))
+ _ => None,
+ }
}
- /// Parse a tuple expression: `name(<expr>, ...)` with a given identifier.
fn parse_named_tuple(&mut self, name: Spanned<Ident>) -> Spanned<NamedTuple> {
let tuple = self.parse_tuple().0;
let span = Span::merge(name.span, tuple.span);
Spanned::new(NamedTuple::new(name, tuple), span)
}
- /// Parse an object expression: `{ <key>: <value>, ... }`.
- fn parse_object(&mut self) -> Spanned<Object> {
- let token = self.eat();
- debug_assert_eq!(token.map(Spanned::value), Some(Token::LeftBrace));
-
- // Parse a collection until a right brace appears.
- self.parse_collection(Some(Token::RightBrace), |p| {
- // Expect an identifier as the key.
- let key = p.parse_ident().ok_or(("key", None))?;
-
- // Expect a colon behind the key (only separated by whitespace).
- let behind_key = p.pos();
- p.skip_whitespace();
- if p.peekv() != Some(Token::Colon) {
- return Err(("colon", Some(behind_key)));
+ /// The boolean tells you whether the tuple can be coerced into a value
+ /// (this is the case when it's length 1 and has no trailing comma).
+ fn parse_tuple(&mut self) -> (Spanned<Tuple>, bool) {
+ let start = self.pos();
+ self.assert(Token::LeftParen);
+
+ let mut tuple = Tuple::new();
+ let mut commaless = true;
+ loop {
+ self.skip_white();
+ if self.eof() || self.check(Token::RightParen) {
+ break;
}
- p.eat();
- p.skip_whitespace();
+ let expr = try_opt_or!(self.parse_expr(), {
+ self.expected("value");
+ continue;
+ });
- // Semantic highlighting for object keys.
- p.feedback.decos.push(
- Spanned::new(Decoration::ObjectKey, key.span));
+ let behind_expr = expr.span.end;
+ tuple.add(expr);
- let value = p.parse_expr().ok_or(("value", None))?;
+ self.skip_white();
+ if self.eof() || self.check(Token::RightParen) {
+ break;
+ }
- let span = Span::merge(key.span, value.span);
- Ok(Spanned::new(Pair { key, value }, span))
- })
- }
+ self.expect_at(Token::Comma, behind_expr);
+ commaless = false;
+ }
- /// Parse a comma-separated collection where each item is parsed through
- /// `parse_item` until the `end` token is met.
- fn parse_collection<C, I, F>(
- &mut self,
- end: Option<Token>,
- parse_item: F
- ) -> Spanned<C>
- where
- C: FromIterator<Spanned<I>>,
- F: FnMut(&mut Self) -> Result<Spanned<I>, (&'static str, Option<Position>)>,
- {
- self.parse_collection_comma_aware(end, parse_item).0
+ self.expect(Token::RightParen);
+ let end = self.pos();
+ let coercable = commaless && !tuple.items.is_empty();
+
+ (Spanned::new(tuple, Span::new(start, end)), coercable)
}
- /// Parse a comma-separated collection where each item is parsed through
- /// `parse_item` until the `end` token is met. The first item in the return
- /// tuple is the collection, the second item indicates whether the
- /// collection can be coerced into a single item (i.e. no comma appeared).
- fn parse_collection_comma_aware<C, I, F>(
- &mut self,
- end: Option<Token>,
- mut parse_item: F
- ) -> (Spanned<C>, bool)
- where
- C: FromIterator<Spanned<I>>,
- F: FnMut(&mut Self) -> Result<Spanned<I>, (&'static str, Option<Position>)>,
- {
+ fn parse_object(&mut self) -> Spanned<Object> {
let start = self.pos();
- let mut can_be_coerced = true;
+ self.assert(Token::LeftBrace);
- // Parse the comma separated items.
- let collection = std::iter::from_fn(|| {
- self.skip_whitespace();
- let peeked = self.peekv();
-
- // We finished as expected.
- if peeked == end {
- self.eat();
- return None;
+ let mut object = Object::new();
+ loop {
+ self.skip_white();
+ if self.eof() || self.check(Token::RightBrace) {
+ break;
}
- // We finished without the expected end token (which has to be a
- // `Some` value at this point since otherwise we would have already
- // returned in the previous case).
- if peeked == None {
- self.eat();
- self.expected_at(end.unwrap().name(), self.pos());
- return None;
+ let key = try_opt_or!(self.parse_ident(), {
+ self.expected("key");
+ continue;
+ });
+
+ let after_key = self.pos();
+ self.skip_white();
+ if !self.expect_at(Token::Colon, after_key) {
+ continue;
}
- // Try to parse a collection item.
- match parse_item(self) {
- Ok(item) => {
- // Expect a comma behind the item (only separated by
- // whitespace).
- self.skip_whitespace();
- match self.peekv() {
- Some(Token::Comma) => {
- can_be_coerced = false;
- self.eat();
- }
- t @ Some(_) if t != end => {
- can_be_coerced = false;
- self.expected_at("comma", item.span.end);
- },
- _ => {}
- }
-
- return Some(Some(item));
- }
+ self.feedback.decos.push(
+ Spanned::new(Decoration::ObjectKey, key.span)
+ );
- // The item parser expected something different at either some
- // given position or instead of the currently peekable token.
- Err((expected, Some(pos))) => self.expected_at(expected, pos),
- Err((expected, None)) => {
- let token = self.peek();
- if token.map(Spanned::value) != end {
- self.eat();
- }
- self.expected_found_or_at(expected, token, self.pos());
- }
+ self.skip_white();
+ let value = try_opt_or!(self.parse_expr(), {
+ self.expected("value");
+ continue;
+ });
+
+ let behind_value = value.span.end;
+ let span = Span::merge(key.span, value.span);
+ object.add(Spanned::new(Pair { key, value }, span));
+
+ self.skip_white();
+ if self.eof() || self.check(Token::RightBrace) {
+ break;
}
- Some(None)
- }).filter_map(|x| x).collect();
+ self.expect_at(Token::Comma, behind_value);
+ }
+ self.expect(Token::RightBrace);
let end = self.pos();
- (Spanned::new(collection, Span { start, end }), can_be_coerced)
+
+ Spanned::new(object, Span::new(start, end))
}
+}
- /// Try to parse an identifier and do nothing if the peekable token is no
- /// identifier.
- fn parse_ident(&mut self) -> Option<Spanned<Ident>> {
- match self.peek() {
- Some(Spanned { v: Token::ExprIdent(s), span }) => {
- self.eat();
- Some(Spanned { v: Ident(s.to_string()), span })
- }
- _ => None
+// Error handling
+impl FuncParser<'_> {
+ fn expect(&mut self, token: Token<'_>) -> bool {
+ if self.check(token) {
+ self.eat();
+ true
+ } else {
+ self.expected(token.name());
+ false
}
}
- /// Skip all whitespace/comment tokens.
- fn skip_whitespace(&mut self) {
- self.eat_until(|t| match t {
- Token::Space(_) | Token::LineComment(_) |
- Token::BlockComment(_) => false,
- _ => true,
- }, false)
+ fn expect_at(&mut self, token: Token<'_>, pos: Position) -> bool {
+ if self.check(token) {
+ self.eat();
+ true
+ } else {
+ self.expected_at(token.name(), pos);
+ false
+ }
}
- /// Add an error about an expected `thing` which was not found, showing
- /// what was found instead.
- fn expected_found(&mut self, thing: &str, found: Spanned<Token>) {
- error!(
- @self.feedback, found.span,
- "expected {}, found {}", thing, found.v.name(),
- );
+ fn expected(&mut self, thing: &str) {
+ if let Some(found) = self.eat() {
+ error!(
+ @self.feedback, found.span,
+ "expected {}, found {}", thing, found.v.name(),
+ );
+ } else {
+ error!(@self.feedback, Span::at(self.pos()), "expected {}", thing);
+ }
}
- /// Add an error about an `thing` which was expected but not found at the
- /// given position.
fn expected_at(&mut self, thing: &str, pos: Position) {
error!(@self.feedback, Span::at(pos), "expected {}", thing);
}
- /// Add a expected-found-error if `found` is `Some` and an expected-error
- /// otherwise.
- fn expected_found_or_at(
- &mut self,
- thing: &str,
- found: Option<Spanned<Token>>,
- pos: Position
- ) {
- match found {
- Some(found) => self.expected_found(thing, found),
- None => self.expected_at(thing, pos),
+ fn expected_found_or_at(&mut self, thing: &str, pos: Position) {
+ if self.eof() {
+ self.expected_at(thing, pos)
+ } else {
+ self.expected(thing);
}
}
+}
- /// Consume tokens until the function returns true and only consume the last
- /// token if instructed to so by `eat_match`.
- fn eat_until<F>(&mut self, mut f: F, eat_match: bool)
- where F: FnMut(Token<'s>) -> bool {
- while let Some(token) = self.peek() {
- if f(token.v) {
- if eat_match {
- self.eat();
- }
- break;
+// Parsing primitives
+impl<'s> FuncParser<'s> {
+ fn skip_white(&mut self) {
+ loop {
+ match self.peek().map(Spanned::value) {
+ Some(Token::Space(_))
+ | Some(Token::LineComment(_))
+ | Some(Token::BlockComment(_)) => { self.eat(); }
+ _ => break,
}
-
- self.eat();
}
}
- /// Consume and return the next token.
fn eat(&mut self) -> Option<Spanned<Token<'s>>> {
- self.peeked.take()
- .unwrap_or_else(|| self.tokens.next())
+ self.peeked.take().unwrap_or_else(|| self.tokens.next())
+ }
+
+ fn eat_span<T>(&mut self, v: T) -> Option<Spanned<T>> {
+ self.eat().map(|spanned| spanned.map(|_| v))
}
- /// Peek at the next token without consuming it.
fn peek(&mut self) -> Option<Spanned<Token<'s>>> {
- let iter = &mut self.tokens;
- *self.peeked.get_or_insert_with(|| iter.next())
+ let tokens = &mut self.tokens;
+ *self.peeked.get_or_insert_with(|| tokens.next())
+ }
+
+ fn assert(&mut self, token: Token<'_>) {
+ assert!(self.check_eat(token).is_some());
+ }
+
+ fn check(&mut self, token: Token<'_>) -> bool {
+ self.peek().map(Spanned::value) == Some(token)
+ }
+
+ fn check_eat(&mut self, token: Token<'_>) -> Option<Spanned<Token<'s>>> {
+ if self.check(token) {
+ self.eat()
+ } else {
+ None
+ }
}
- /// Peek at the unspanned value of the next token.
- fn peekv(&mut self) -> Option<Token<'s>> {
- self.peek().map(Spanned::value)
+ fn eof(&mut self) -> bool {
+ self.peek().is_none()
}
- /// The position at the end of the last eaten token / start of the peekable
- /// token.
fn pos(&self) -> Position {
self.peeked.flatten()
.map(|s| s.span.start)
@@ -583,140 +551,71 @@ impl<'s> FuncParser<'s> {
}
}
-/// Unescape a string: `the string is \"this\"` => `the string is "this"`.
fn unescape_string(string: &str) -> String {
- let mut s = String::with_capacity(string.len());
let mut iter = string.chars();
+ let mut out = String::with_capacity(string.len());
while let Some(c) = iter.next() {
if c == '\\' {
match iter.next() {
- Some('\\') => s.push('\\'),
- Some('"') => s.push('"'),
- Some('n') => s.push('\n'),
- Some('t') => s.push('\t'),
- Some(c) => { s.push('\\'); s.push(c); }
- None => s.push('\\'),
+ Some('\\') => out.push('\\'),
+ Some('"') => out.push('"'),
+ Some('n') => out.push('\n'),
+ Some('t') => out.push('\t'),
+ Some(c) => { out.push('\\'); out.push(c); }
+ None => out.push('\\'),
}
} else {
- s.push(c);
+ out.push(c);
}
}
- s
+ out
}
-/// Unescape raw markup into lines.
+/// Unescape raw markup and split it into into lines.
fn unescape_raw(raw: &str) -> Vec<String> {
- let mut lines = Vec::new();
- let mut s = String::new();
let mut iter = raw.chars().peekable();
+ let mut line = String::new();
+ let mut lines = Vec::new();
while let Some(c) = iter.next() {
if c == '\\' {
match iter.next() {
- Some('`') => s.push('`'),
- Some(c) => { s.push('\\'); s.push(c); }
- None => s.push('\\'),
+ Some('`') => line.push('`'),
+ Some(c) => { line.push('\\'); line.push(c); }
+ None => line.push('\\'),
}
} else if is_newline_char(c) {
if c == '\r' && iter.peek() == Some(&'\n') {
iter.next();
}
- lines.push(std::mem::replace(&mut s, String::new()));
+ lines.push(std::mem::take(&mut line));
} else {
- s.push(c);
+ line.push(c);
}
}
- lines.push(s);
+ lines.push(line);
lines
}
-
#[cfg(test)]
#[allow(non_snake_case)]
mod tests {
use crate::size::Size;
- use crate::syntax::test::{DebugFn, check, zspan};
- use crate::syntax::func::Value;
+ use super::super::test::{check, DebugFn};
+ use super::super::func::Value;
use super::*;
use Decoration::*;
+ use Expr::{Number as Num, Size as Sz, Bool};
use Node::{
Space as S, ToggleItalic as Italic, ToggleBolder as Bold,
Parbreak, Linebreak,
};
- use Expr::{Number as Num, Size as Sz, Bool};
- fn Id(text: &str) -> Expr { Expr::Ident(Ident(text.to_string())) }
- fn Str(text: &str) -> Expr { Expr::Str(text.to_string()) }
- fn Pt(points: f32) -> Expr { Expr::Size(Size::pt(points)) }
- fn Neg(e1: Expr) -> Expr { Expr::Neg(Box::new(zspan(e1))) }
- fn Add(e1: Expr, e2: Expr) -> Expr {
- Expr::Add(Box::new(zspan(e1)), Box::new(zspan(e2)))
- }
- fn Sub(e1: Expr, e2: Expr) -> Expr {
- Expr::Sub(Box::new(zspan(e1)), Box::new(zspan(e2)))
- }
- fn Mul(e1: Expr, e2: Expr) -> Expr {
- Expr::Mul(Box::new(zspan(e1)), Box::new(zspan(e2)))
- }
- fn Div(e1: Expr, e2: Expr) -> Expr {
- Expr::Div(Box::new(zspan(e1)), Box::new(zspan(e2)))
- }
-
- fn Clr(r: u8, g: u8, b: u8, a: u8) -> Expr {
- Expr::Color(RgbaColor::new(r, g, b, a))
- }
- fn ClrStr(color: &str) -> Expr {
- Expr::Color(RgbaColor::from_str(color).expect("invalid test color"))
- }
- fn ClrStrHealed() -> Expr {
- let mut c = RgbaColor::from_str("000f").expect("invalid test color");
- c.healed = true;
- Expr::Color(c)
- }
-
- fn T(text: &str) -> Node { Node::Text(text.to_string()) }
-
- /// Create a raw text node.
- macro_rules! raw {
- ($($line:expr),* $(,)?) => {
- Node::Raw(vec![$($line.to_string()),*])
- };
- }
-
- /// Create a tuple expression.
- macro_rules! tuple {
- ($($items:expr),* $(,)?) => {
- Expr::Tuple(Tuple { items: spanned![vec $($items),*].0 })
- };
- }
-
- /// Create a named tuple expression.
- macro_rules! named_tuple {
- ($name:expr $(, $items:expr)* $(,)?) => {
- Expr::NamedTuple(NamedTuple::new(
- zspan(Ident($name.to_string())),
- zspan(Tuple { items: spanned![vec $($items),*].0 })
- ))
- };
- }
-
- /// Create an object expression.
- macro_rules! object {
- ($($key:expr => $value:expr),* $(,)?) => {
- Expr::Object(Object {
- pairs: vec![$(zspan(Pair {
- key: zspan(Ident($key.to_string())),
- value: zspan($value),
- })),*]
- })
- };
- }
-
/// Test whether the given string parses into
/// - the given node list (required).
/// - the given error list (optional, if omitted checks against empty list).
@@ -733,15 +632,15 @@ mod tests {
scope.add::<DebugFn>("box");
scope.add::<DebugFn>("val");
- let ctx = ParseContext { scope: &scope };
- let pass = parse(Position::ZERO, $source, ctx);
+ let state = ParseState { scope };
+ let pass = parse($source, Position::ZERO, &state);
- // Test model
- let (exp, cmp) = spanned![vec $($model)*];
+ // Test model.
+ let (exp, cmp) = span_vec![$($model)*];
check($source, exp, pass.output.nodes, cmp);
- // Test problems
- let (exp, cmp) = spanned![vec $($problems)*];
+ // Test problems.
+ let (exp, cmp) = span_vec![$($problems)*];
let exp = exp.into_iter()
.map(|s: Spanned<&str>| s.map(|e| e.to_string()))
.collect::<Vec<_>>();
@@ -750,42 +649,90 @@ mod tests {
.collect::<Vec<_>>();
check($source, exp, found, cmp);
- // Test decos
- $(let (exp, cmp) = spanned![vec $($decos)*];
+ // Test decos.
+ $(let (exp, cmp) = span_vec![$($decos)*];
check($source, exp, pass.feedback.decos, cmp);)?
};
}
- /// Write down a `DebugFn` function model compactly.
+ /// Shorthand for `p!("[val: ...]" => func!("val", ...))`.
+ macro_rules! pval {
+ ($header:expr => $($tts:tt)*) => {
+ p!(concat!("[val: ", $header, "]") => [func!("val": $($tts)*)]);
+ }
+ }
+
+ fn Id(text: &str) -> Expr { Expr::Ident(Ident(text.to_string())) }
+ fn Str(text: &str) -> Expr { Expr::Str(text.to_string()) }
+ fn Pt(points: f32) -> Expr { Expr::Size(Size::pt(points)) }
+ fn Color(r: u8, g: u8, b: u8, a: u8) -> Expr { Expr::Color(RgbaColor::new(r, g, b, a)) }
+ fn ColorStr(color: &str) -> Expr { Expr::Color(RgbaColor::from_str(color).expect("invalid test color")) }
+ fn ColorHealed() -> Expr { Expr::Color(RgbaColor::new_healed(0, 0, 0, 255)) }
+ fn Neg(e1: Expr) -> Expr { Expr::Neg(Box::new(Z(e1))) }
+ fn Add(e1: Expr, e2: Expr) -> Expr { Expr::Add(Box::new(Z(e1)), Box::new(Z(e2))) }
+ fn Sub(e1: Expr, e2: Expr) -> Expr { Expr::Sub(Box::new(Z(e1)), Box::new(Z(e2))) }
+ fn Mul(e1: Expr, e2: Expr) -> Expr { Expr::Mul(Box::new(Z(e1)), Box::new(Z(e2))) }
+ fn Div(e1: Expr, e2: Expr) -> Expr { Expr::Div(Box::new(Z(e1)), Box::new(Z(e2))) }
+ fn T(text: &str) -> Node { Node::Text(text.to_string()) }
+ fn Z<T>(v: T) -> Spanned<T> { Spanned::zero(v) }
+
+ macro_rules! tuple {
+ ($($items:expr),* $(,)?) => {
+ Expr::Tuple(Tuple { items: span_vec![$($items),*].0 })
+ };
+ }
+
+ macro_rules! named_tuple {
+ ($name:expr $(, $items:expr)* $(,)?) => {
+ Expr::NamedTuple(NamedTuple::new(
+ Z(Ident($name.to_string())),
+ Z(Tuple { items: span_vec![$($items),*].0 })
+ ))
+ };
+ }
+
+ macro_rules! object {
+ ($($key:expr => $value:expr),* $(,)?) => {
+ Expr::Object(Object {
+ pairs: vec![$(Z(Pair {
+ key: Z(Ident($key.to_string())),
+ value: Z($value),
+ })),*]
+ })
+ };
+ }
+
+ macro_rules! raw {
+ ($($line:expr),* $(,)?) => {
+ Node::Raw(vec![$($line.to_string()),*])
+ };
+ }
+
macro_rules! func {
- ($name:tt $(: ($($pos:tt)*), { $($key:tt)* } )? $(; $($body:tt)*)?) => ({
+ ($name:tt $(: ($($pos:tt)*) $(, { $($key:tt)* })? )? $(; $($body:tt)*)?) => {{
#[allow(unused_mut)]
let mut args = FuncArgs::new();
- $(args.pos = Tuple::parse(zspan(tuple!($($pos)*))).unwrap();)?
- $(args.key = Object::parse(zspan(object! { $($key)* })).unwrap();)?
-
+ $(args.pos = Tuple::parse(Z(tuple!($($pos)*))).unwrap();
+ $(args.key = Object::parse(Z(object! { $($key)* })).unwrap();)?)?
Node::Model(Box::new(DebugFn {
header: FuncHeader {
- name: spanned!(item $name).map(|s| Ident(s.to_string())),
+ name: span_item!($name).map(|s| Ident(s.to_string())),
args,
},
body: func!(@body $($($body)*)?),
}))
- });
-
- (@body [$($body:tt)*]) => ({
- Some(SyntaxModel { nodes: spanned![vec $($body)*].0 })
- });
- (@body) => (None);
+ }};
+ (@body [$($body:tt)*]) => { Some(SyntaxModel { nodes: span_vec![$($body)*].0 }) };
+ (@body) => { None };
}
#[test]
fn parse_color_strings() {
- assert_eq!(Clr(0xf6, 0x12, 0x43, 0xff), ClrStr("f61243ff"));
- assert_eq!(Clr(0xb3, 0xd8, 0xb3, 0xff), ClrStr("b3d8b3"));
- assert_eq!(Clr(0xfc, 0xd2, 0xa9, 0xad), ClrStr("fCd2a9AD"));
- assert_eq!(Clr(0x22, 0x33, 0x33, 0xff), ClrStr("233"));
- assert_eq!(Clr(0x11, 0x11, 0x11, 0xbb), ClrStr("111b"));
+ assert_eq!(Color(0xf6, 0x12, 0x43, 0xff), ColorStr("f61243ff"));
+ assert_eq!(Color(0xb3, 0xd8, 0xb3, 0xff), ColorStr("b3d8b3"));
+ assert_eq!(Color(0xfc, 0xd2, 0xa9, 0xad), ColorStr("fCd2a9AD"));
+ assert_eq!(Color(0x22, 0x33, 0x33, 0xff), ColorStr("233"));
+ assert_eq!(Color(0x11, 0x11, 0x11, 0xbb), ColorStr("111b"));
}
#[test]
@@ -824,7 +771,7 @@ mod tests {
#[test]
fn parse_basic_nodes() {
- // Basic nodes
+ // Basic nodes.
p!("" => []);
p!("hi" => [T("hi")]);
p!("*hi" => [Bold, T("hi")]);
@@ -838,13 +785,13 @@ mod tests {
p!(r"a\ b" => [T("a"), Linebreak, S, T("b")]);
p!("πŸ’œ\n\n 🌍" => [T("πŸ’œ"), Parbreak, T("🌍")]);
- // Raw markup
+ // Raw markup.
p!("`py`" => [raw!["py"]]);
p!("[val][`hi]`]" => [func!("val"; [raw!["hi]"]])]);
p!("`hi\nyou" => [raw!["hi", "you"]], [(1:3, 1:3, "expected backtick")]);
p!("`hi\\`du`" => [raw!["hi`du"]]);
- // Spanned nodes
+ // Spanned nodes.
p!("Hi" => [(0:0, 0:2, T("Hi"))]);
p!("*Hi*" => [(0:0, 0:1, Bold), (0:1, 0:3, T("Hi")), (0:3, 0:4, Bold)]);
p!("🌎\n*/[n]" =>
@@ -856,45 +803,51 @@ mod tests {
#[test]
fn parse_function_names() {
- // No closing bracket
+ // No closing bracket.
p!("[" => [func!("")], [
- (0:1, 0:1, "expected identifier"),
+ (0:1, 0:1, "expected function name"),
(0:1, 0:1, "expected closing bracket")
]);
- // No name
- p!("[]" => [func!("")], [(0:1, 0:1, "expected identifier")]);
+ // No name.
+ p!("[]" => [func!("")], [(0:1, 0:1, "expected function name")]);
p!("[\"]" => [func!("")], [
- (0:1, 0:3, "expected identifier, found string"),
+ (0:1, 0:3, "expected function name, found string"),
(0:3, 0:3, "expected closing bracket"),
]);
- // An unknown name
+ // An unknown name.
p!("[hi]" =>
[func!("hi")],
[(0:1, 0:3, "unknown function")],
[(0:1, 0:3, InvalidFuncName)],
);
- // A valid name
+ // A valid name.
p!("[f]" => [func!("f")], [], [(0:1, 0:2, ValidFuncName)]);
p!("[ f]" => [func!("f")], [], [(0:3, 0:4, ValidFuncName)]);
- // An invalid token for a name
- p!("[12]" => [func!("")], [(0:1, 0:3, "expected identifier, found number")], []);
- p!("[🌎]" => [func!("")], [(0:1, 0:2, "expected identifier, found invalid token")], []);
- p!("[ 🌎]" => [func!("")], [(0:3, 0:4, "expected identifier, found invalid token")], []);
+ // An invalid token for a name.
+ p!("[12]" => [func!("")], [(0:1, 0:3, "expected function name, found number")], []);
+ p!("[🌎]" => [func!("")], [(0:1, 0:2, "expected function name, found invalid token")], []);
+ p!("[ 🌎]" => [func!("")], [(0:3, 0:4, "expected function name, found invalid token")], []);
}
#[test]
fn parse_colon_starting_function_arguments() {
- // No colon before arg
+ // Valid.
+ p!("[val: true]" =>
+ [func!["val": (Bool(true))]], [],
+ [(0:1, 0:4, ValidFuncName)],
+ );
+
+ // No colon before arg.
p!("[val\"s\"]" => [func!("val")], [(0:4, 0:4, "expected colon")]);
- // No colon before valid, but wrong token
+ // No colon before valid, but wrong token.
p!("[val=]" => [func!("val")], [(0:4, 0:4, "expected colon")]);
- // No colon before invalid tokens, which are ignored
+ // No colon before invalid tokens, which are ignored.
p!("[val/🌎:$]" =>
[func!("val")],
[(0:4, 0:4, "expected colon")],
@@ -909,36 +862,38 @@ mod tests {
(0:7, 0:7, "expected closing bracket"),
]);
- // Just colon without args
+ // Just colon without args.
p!("[val:]" => [func!("val")]);
p!("[val:/*12pt*/]" => [func!("val")]);
- // Whitespace / comments around colon
- p!("[val\n:\ntrue]" => [func!("val": (Bool(true)), {})]);
- p!("[val/*:*/://\ntrue]" => [func!("val": (Bool(true)), {})]);
+ // Whitespace / comments around colon.
+ p!("[val\n:\ntrue]" => [func!("val": (Bool(true)))]);
+ p!("[val/*:*/://\ntrue]" => [func!("val": (Bool(true)))]);
}
#[test]
fn parse_one_positional_argument() {
- // Different expressions
- p!("[val: true]" =>
- [func!("val": (Bool(true)), {})], [],
- [(0:1, 0:4, ValidFuncName)],
- );
- p!("[val: _]" => [func!("val": (Id("_")), {})]);
- p!("[val: name]" => [func!("val": (Id("name")), {})]);
- p!("[val: \"hi\"]" => [func!("val": (Str("hi")), {})]);
- p!("[val: \"a\n[]\\\"string\"]" => [func!("val": (Str("a\n[]\"string")), {})]);
- p!("[val: 3.14]" => [func!("val": (Num(3.14)), {})]);
- p!("[val: 4.5cm]" => [func!("val": (Sz(Size::cm(4.5))), {})]);
- p!("[val: 12e1pt]" => [func!("val": (Pt(12e1)), {})]);
- p!("[val: #f7a20500]" => [func!("val": (ClrStr("f7a20500")), {})]);
-
- // Math
- p!("[val: 3.2in + 6pt]" => [func!("val": (Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0)))), {})]);
- p!("[val: 5 - 0.01]" => [func!("val": (Sub(Num(5.0), Num(0.01))), {})]);
- p!("[val: (3mm * 2)]" => [func!("val": (Mul(Sz(Size::mm(3.0)), Num(2.0))), {})]);
- p!("[val: 12e-3cm/1pt]" => [func!("val": (Div(Sz(Size::cm(12e-3)), Sz(Size::pt(1.0)))), {})]);
+ // Different expressions.
+ pval!("_" => (Id("_")));
+ pval!("name" => (Id("name")));
+ pval!("\"hi\"" => (Str("hi")));
+ pval!("3.14" => (Num(3.14)));
+ pval!("4.5cm" => (Sz(Size::cm(4.5))));
+ pval!("12e1pt" => (Pt(12e1)));
+ pval!("#f7a20500" => (ColorStr("f7a20500")));
+ pval!("\"a\n[]\\\"string\"" => (Str("a\n[]\"string")));
+
+ // Trailing comma.
+ pval!("a," => (Id("a")));
+
+ // Simple coerced tuple.
+ pval!("(hi)" => (Id("hi")));
+
+ // Math.
+ pval!("3.2in + 6pt" => (Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0)))));
+ pval!("5 - 0.01" => (Sub(Num(5.0), Num(0.01))));
+ pval!("(3mm * 2)" => (Mul(Sz(Size::mm(3.0)), Num(2.0))));
+ pval!("12e-3cm/1pt" => (Div(Sz(Size::cm(12e-3)), Sz(Size::pt(1.0)))));
// Unclosed string.
p!("[val: \"hello]" => [func!("val": (Str("hello]")), {})], [
@@ -946,112 +901,91 @@ mod tests {
(0:13, 0:13, "expected closing bracket"),
]);
- //Invalid colors
- p!("[val: #12345]" => [func!("val": (ClrStrHealed()), {})], [
- (0:6, 0:12, "invalid color"),
- ]);
- p!("[val: #a5]" => [func!("val": (ClrStrHealed()), {})], [
- (0:6, 0:9, "invalid color"),
- ]);
- p!("[val: #14b2ah]" => [func!("val": (ClrStrHealed()), {})], [
- (0:6, 0:13, "invalid color"),
- ]);
- p!("[val: #f075ff011]" => [func!("val": (ClrStrHealed()), {})], [
- (0:6, 0:16, "invalid color"),
- ]);
+ // Invalid, healed colors.
+ p!("[val: #12345]" => [func!("val": (ColorHealed()))], [(0:6, 0:12, "invalid color")]);
+ p!("[val: #a5]" => [func!("val": (ColorHealed()))], [(0:6, 0:9, "invalid color")]);
+ p!("[val: #14b2ah]" => [func!("val": (ColorHealed()))], [(0:6, 0:13, "invalid color")]);
+ p!("[val: #f075ff011]" => [func!("val": (ColorHealed()))], [(0:6, 0:16, "invalid color")]);
}
#[test]
fn parse_complex_mathematical_expressions() {
- p!("[val: (3.2in + 6pt)*(5/2-1)]" => [func!("val": (
- Mul(
- Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0))),
- Sub(Div(Num(5.0), Num(2.0)), Num(1.0))
- )
- ), {})]);
- p!("[val: (6.3E+2+4* - 3.2pt)/2]" => [func!("val": (
- Div(Add(Num(6.3e2),Mul(Num(4.0), Neg(Pt(3.2)))), Num(2.0))
- ), {})]);
- p!("[val: 4pt--]" =>
- [func!("val": (Pt(4.0)), {})],
- [
- (0:10, 0:11, "dangling minus"),
- (0:6, 0:10, "missing right summand")
- ],
+ // Valid expressions.
+ pval!("(3.2in + 6pt)*(5/2-1)" => (Mul(
+ Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0))),
+ Sub(Div(Num(5.0), Num(2.0)), Num(1.0))
+ )));
+ pval!("(6.3E+2+4* - 3.2pt)/2" => (Div(
+ Add(Num(6.3e2),Mul(Num(4.0), Neg(Pt(3.2)))),
+ Num(2.0)
+ )));
+
+ // Associativity of multiplication and division.
+ p!("[val: 3/4*5]" =>
+ [func!("val": (Mul(Div(Num(3.0), Num(4.0)), Num(5.0))), {})]
);
+
+ // Invalid expressions.
+ p!("[val: 4pt--]" => [func!("val": (Pt(4.0)))], [
+ (0:10, 0:11, "dangling minus"),
+ (0:6, 0:10, "missing right summand")
+ ]);
p!("[val: 3mm+4pt*]" =>
- [func!("val": (Add(Sz(Size::mm(3.0)), Pt(4.0))), {})],
+ [func!("val": (Add(Sz(Size::mm(3.0)), Pt(4.0))))],
[(0:10, 0:14, "missing right factor")],
);
}
#[test]
fn parse_tuples() {
- // Empty tuple
- p!("[val: ()]" => [func!("val": (tuple!()), {})]);
- p!("[val: empty()]" => [func!("val": (named_tuple!("empty")), {})]);
+ // Empty tuple.
+ pval!("()" => (tuple!()));
+ pval!("empty()" => (named_tuple!("empty")));
+
+ // Space between name and tuple.
+ pval!("add ( 1 , 2 )" => (named_tuple!("add", Num(1.0), Num(2.0))));
+ pval!("num = add ( 1 , 2 )" => (), {
+ "num" => named_tuple!("add", Num(1.0), Num(2.0))
+ });
- // Invalid value
- p!("[val: (🌎)]" =>
- [func!("val": (tuple!()), {})],
- [(0:7, 0:8, "expected value, found invalid token")],
- );
+ // Invalid value.
p!("[val: sound(\x07)]" =>
[func!("val": (named_tuple!("sound")), {})],
[(0:12, 0:13, "expected value, found invalid token")],
);
- // Invalid tuple name
+ // Invalid tuple name.
p!("[val: πŸ‘ (\"abc\", 13e-5)]" =>
[func!("val": (tuple!(Str("abc"), Num(13.0e-5))), {})],
[(0:6, 0:7, "expected argument, found invalid token")],
);
- // Unclosed tuple
- p!("[val: (hello,]" =>
- [func!("val": (tuple!(Id("hello"),)), {})],
- [(0:13, 0:13, "expected closing paren")],
- );
+ // Unclosed tuple.
p!("[val: lang(δΈ­ζ–‡]" =>
[func!("val": (named_tuple!("lang", Id("δΈ­ζ–‡"))), {})],
[(0:13, 0:13, "expected closing paren")],
);
- // Valid values
- p!("[val: (1, 2)]" => [func!("val": (tuple!(Num(1.0), Num(2.0))), {})]);
- p!("[val: (\"s\",)]" => [func!("val": (tuple!(Str("s"))), {})]);
- p!("[val: cmyk(1, 46, 0, 0)]" =>
- [func!("val": (named_tuple!(
- "cmyk", Num(1.0), Num(46.0), Num(0.0), Num(0.0)
- )), {})]
- );
- p!("[val: items(\"fire\", #f93a6d)]" =>
- [func!("val": (named_tuple!(
- "items", Str("fire"), ClrStr("f93a6d")
- )), {})]
- );
-
- // Nested tuples
- p!("[val: (1, (2, 3))]" =>
- [func!("val": (tuple!(Num(1.0), tuple!(Num(2.0), Num(3.0)))), {})]
- );
- p!("[val: css(1pt, rgb(90, 102, 254), \"solid\")]" =>
- [func!("val": (named_tuple!(
- "css", Pt(1.0), named_tuple!(
- "rgb", Num(90.0), Num(102.0), Num(254.0)
- ), Str("solid")
- )), {})]
- );
-
- // Invalid commas
+ // Valid values.
+ pval!("(1, 2)" => (tuple!(Num(1.0), Num(2.0))));
+ pval!("(\"s\",)" => (tuple!(Str("s"))));
+ pval!("items(\"fire\", #f93a6d)" => (
+ named_tuple!("items", Str("fire"), ColorStr("f93a6d")
+ )));
+
+ // Nested tuples.
+ pval!("css(1pt, rgb(90, 102, 254), \"solid\")" => (named_tuple!(
+ "css",
+ Pt(1.0),
+ named_tuple!("rgb", Num(90.0), Num(102.0), Num(254.0)),
+ Str("solid"),
+ )));
+
+ // Invalid commas.
p!("[val: (,)]" =>
[func!("val": (tuple!()), {})],
[(0:7, 0:8, "expected value, found comma")],
);
- p!("[val: nose(,)]" =>
- [func!("val": (named_tuple!("nose")), {})],
- [(0:11, 0:12, "expected value, found comma")],
- );
p!("[val: (true false)]" =>
[func!("val": (tuple!(Bool(true), Bool(false))), {})],
[(0:11, 0:11, "expected comma")],
@@ -1060,14 +994,13 @@ mod tests {
#[test]
fn parse_objects() {
- let f = || func!("val": (object! {}), {});
+ let val = || func!("val": (object! {}), {});
- // Okay objects
- p!("[val: {}]" => [f()]);
- p!("[val: { key: value }]" =>
- [func!("val": (object! { "key" => Id("value") }), {})]);
+ // Okay objects.
+ pval!("{}" => (object! {}));
+ pval!("{ key: value }" => (object! { "key" => Id("value") }));
- // Unclosed object
+ // Unclosed object.
p!("[val: {hello: world]" =>
[func!("val": (object! { "hello" => Id("world") }), {})],
[(0:19, 0:19, "expected closing brace")],
@@ -1077,14 +1010,14 @@ mod tests {
[(0:9, 0:9, "expected colon"), (0:9, 0:9, "expected closing brace")],
);
- // Missing key
- p!("[val: {,}]" => [f()], [(0:7, 0:8, "expected key, found comma")]);
- p!("[val: { 12pt }]" => [f()], [(0:8, 0:12, "expected key, found size")]);
- p!("[val: { : }]" => [f()], [(0:8, 0:9, "expected key, found colon")]);
+ // Missing key.
+ p!("[val: {,}]" => [val()], [(0:7, 0:8, "expected key, found comma")]);
+ p!("[val: { 12pt }]" => [val()], [(0:8, 0:12, "expected key, found size")]);
+ p!("[val: { : }]" => [val()], [(0:8, 0:9, "expected key, found colon")]);
- // Missing colon
- p!("[val: { key }]" => [f()], [(0:11, 0:11, "expected colon")]);
- p!("[val: { key false }]" => [f()], [
+ // Missing colon.
+ p!("[val: { key }]" => [val()], [(0:11, 0:11, "expected colon")]);
+ p!("[val: { key false }]" => [val()], [
(0:11, 0:11, "expected colon"),
(0:12, 0:17, "expected key, found bool"),
]);
@@ -1093,14 +1026,14 @@ mod tests {
[(0:9, 0:9, "expected colon")],
);
- // Missing value
- p!("[val: { key: : }]" => [f()], [(0:13, 0:14, "expected value, found colon")]);
+ // Missing value.
+ p!("[val: { key: : }]" => [val()], [(0:13, 0:14, "expected value, found colon")]);
p!("[val: { key: , k: \"s\" }]" =>
[func!("val": (object! { "k" => Str("s") }), {})],
[(0:13, 0:14, "expected value, found comma")],
);
- // Missing comma, invalid token
+ // Missing comma, invalid token.
p!("[val: left={ a: 2, b: false 🌎 }]" =>
[func!("val": (), {
"left" => object! {
@@ -1115,7 +1048,7 @@ mod tests {
#[test]
fn parse_nested_tuples_and_objects() {
- p!("[val: (1, { ab: (), d: (3, 14pt) }), false]" => [func!("val": (
+ pval!("(1, { ab: (), d: (3, 14pt) }), false" => (
tuple!(
Num(1.0),
object!(
@@ -1124,7 +1057,7 @@ mod tests {
),
),
Bool(false),
- ), {})]);
+ ));
}
#[test]
@@ -1151,12 +1084,11 @@ mod tests {
#[test]
fn parse_multiple_mixed_arguments() {
- p!("[val: a,]" => [func!("val": (Id("a")), {})]);
p!("[val: 12pt, key=value]" =>
[func!("val": (Pt(12.0)), { "key" => Id("value") })], [],
[(0:12, 0:15, ArgumentKey), (0:1, 0:4, ValidFuncName)],
);
- p!("[val: a , \"b\" , c]" => [func!("val": (Id("a"), Str("b"), Id("c")), {})]);
+ pval!("a , x=\"b\" , c" => (Id("a"), Id("c")), { "x" => Str("b"), });
}
#[test]
@@ -1177,7 +1109,7 @@ mod tests {
#[test]
fn parse_invalid_key_value_pairs() {
- // Invalid keys
+ // Invalid keys.
p!("[val: true=you]" =>
[func!("val": (Bool(true), Id("you")), {})],
[(0:10, 0:10, "expected comma"),
@@ -1185,13 +1117,14 @@ mod tests {
[(0:1, 0:4, ValidFuncName)],
);
+ // Unexpected equals.
p!("[box: z=y=4]" =>
[func!("box": (Num(4.0)), { "z" => Id("y") })],
[(0:9, 0:9, "expected comma"),
(0:9, 0:10, "expected argument, found equals sign")],
);
- // Invalid colon after keyable positional argument
+ // Invalid colon after keyable positional argument.
p!("[val: key:12]" =>
[func!("val": (Id("key"), Num(12.0)), {})],
[(0:9, 0:9, "expected comma"),
@@ -1199,7 +1132,7 @@ mod tests {
[(0:1, 0:4, ValidFuncName)],
);
- // Invalid colon after non-keyable positional argument
+ // Invalid colon after unkeyable positional argument.
p!("[val: true:12]" => [func!("val": (Bool(true), Num(12.0)), {})],
[(0:10, 0:10, "expected comma"),
(0:10, 0:11, "expected argument, found colon")],
@@ -1209,7 +1142,7 @@ mod tests {
#[test]
fn parse_invalid_commas() {
- // Missing commas
+ // Missing commas.
p!("[val: 1pt 1]" =>
[func!("val": (Pt(1.0), Num(1.0)), {})],
[(0:9, 0:9, "expected comma")],
@@ -1219,7 +1152,7 @@ mod tests {
[(0:7, 0:7, "expected comma")],
);
- // Unexpected commas
+ // Unexpected commas.
p!("[val:,]" => [func!("val")], [(0:5, 0:6, "expected argument, found comma")]);
p!("[val: key=,]" => [func!("val")], [(0:10, 0:11, "expected value, found comma")]);
p!("[val:, true]" =>
@@ -1231,13 +1164,10 @@ mod tests {
#[test]
fn parse_bodies() {
p!("[val][Hi]" => [func!("val"; [T("Hi")])]);
-
- // Body nodes in bodies.
p!("[val:*][*Hi*]" =>
[func!("val"; [Bold, T("Hi"), Bold])],
[(0:5, 0:6, "expected argument, found star")],
);
-
// Errors in bodies.
p!(" [val][ */ ]" =>
[S, func!("val"; [S, S])],
@@ -1267,7 +1197,8 @@ mod tests {
(0:6, 0:18, func!((0:1, 0:4, "val"); [(0:6, 0:11, T("world"))])),
(0:18, 0:19, S),
(0:19, 0:20, T("🌎"))
- ], [],
+ ],
+ [],
[(0:7, 0:10, ValidFuncName)],
);
@@ -1281,7 +1212,8 @@ mod tests {
(1:4, 1:10, func!((0:2, 0:5, "box"))),
(1:10, 2:1, S),
]))
- ], [],
+ ],
+ [],
[(0:2, 0:5, ValidFuncName), (1:6, 1:9, ValidFuncName)],
);
}