diff options
Diffstat (limited to 'src/parse')
| -rw-r--r-- | src/parse/mod.rs | 87 | ||||
| -rw-r--r-- | src/parse/tokens.rs | 102 |
2 files changed, 78 insertions, 111 deletions
diff --git a/src/parse/mod.rs b/src/parse/mod.rs index 773f642c..ce992834 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -138,8 +138,7 @@ fn markup_node(p: &mut Parser, at_start: &mut bool) { NodeKind::LeftBracket => template(p), // Comments. - NodeKind::LineComment | NodeKind::BlockComment => p.eat(), - NodeKind::Error(t, e) if t != &ErrorPosition::Full || e.contains(' ') => p.eat(), + NodeKind::LineComment | NodeKind::BlockComment | NodeKind::Error(_, _) => p.eat(), _ => { *at_start = false; @@ -319,7 +318,7 @@ fn primary(p: &mut Parser, atomic: bool) { Some(NodeKind::Import) => import_expr(p), Some(NodeKind::Include) => include_expr(p), - Some(NodeKind::Error(t, e)) if t != &ErrorPosition::Full || e.contains(' ') => { + Some(NodeKind::Error(_, _)) => { p.eat(); } @@ -333,28 +332,26 @@ fn primary(p: &mut Parser, atomic: bool) { /// Parse a literal. fn literal(p: &mut Parser) -> bool { - let peeked = match p.peek() { - Some(x) => x.clone(), - None => return false, - }; - - match peeked { + match p.peek() { // Basic values. - NodeKind::None - | NodeKind::Auto - | NodeKind::Int(_) - | NodeKind::Float(_) - | NodeKind::Bool(_) - | NodeKind::Fraction(_) - | NodeKind::Length(_, _) - | NodeKind::Angle(_, _) - | NodeKind::Percentage(_) - | NodeKind::Str(_) => p.eat(), + Some( + NodeKind::None + | NodeKind::Auto + | NodeKind::Int(_) + | NodeKind::Float(_) + | NodeKind::Bool(_) + | NodeKind::Fraction(_) + | NodeKind::Length(_, _) + | NodeKind::Angle(_, _) + | NodeKind::Percentage(_) + | NodeKind::Str(_), + ) => { + p.eat(); + true + } - _ => return false, + _ => false, } - - true } /// Parse something that starts with a parenthesis, which can be either of: @@ -395,11 +392,11 @@ fn parenthesized(p: &mut Parser) { // Find out which kind of collection this is. match kind { CollectionKind::Group => p.end(NodeKind::Group), - CollectionKind::PositionalCollection => { + CollectionKind::Positional => { p.lift(); array(p, token_count); } - CollectionKind::NamedCollection => { + CollectionKind::Named => { p.lift(); dict(p, token_count); } @@ -413,9 +410,9 @@ enum CollectionKind { Group, /// The collection starts with a positional and has more items or a trailing /// comma. - PositionalCollection, + Positional, /// The collection starts with a named item. - NamedCollection, + Named, } /// Parse a collection. @@ -424,20 +421,19 @@ enum CollectionKind { /// commas. fn collection(p: &mut Parser) -> (CollectionKind, usize) { let mut items = 0; - let mut kind = CollectionKind::PositionalCollection; - let mut seen_spread = false; + let mut kind = CollectionKind::Positional; let mut has_comma = false; let mut missing_coma = None; while !p.eof() { let item_kind = item(p); if p.success() { - if items == 0 && item_kind == CollectionItemKind::Named { - kind = CollectionKind::NamedCollection; + if items == 0 && item_kind == NodeKind::Named { + kind = CollectionKind::Named; } - if item_kind == CollectionItemKind::ParameterSink { - seen_spread = true; + if item_kind == NodeKind::ParameterSink { + has_comma = true; } items += 1; @@ -458,42 +454,27 @@ fn collection(p: &mut Parser) -> (CollectionKind, usize) { } } - if !has_comma - && items == 1 - && !seen_spread - && kind == CollectionKind::PositionalCollection - { + if !has_comma && items == 1 && kind == CollectionKind::Positional { kind = CollectionKind::Group; } (kind, items) } -/// What kind of item is this? -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -enum CollectionItemKind { - /// A named item. - Named, - /// An unnamed item. - Unnamed, - /// A parameter sink. - ParameterSink, -} - /// Parse an expression or a named pair. Returns if this is a named pair. -fn item(p: &mut Parser) -> CollectionItemKind { +fn item(p: &mut Parser) -> NodeKind { p.start(); if p.eat_if(&NodeKind::Dots) { expr(p); p.end_or_abort(NodeKind::ParameterSink); - return CollectionItemKind::ParameterSink; + return NodeKind::ParameterSink; } expr(p); if p.may_lift_abort() { - return CollectionItemKind::Unnamed; + return NodeKind::None; } if p.eat_if(&NodeKind::Colon) { @@ -512,10 +493,10 @@ fn item(p: &mut Parser) -> CollectionItemKind { p.unsuccessful(); } - CollectionItemKind::Named + NodeKind::Named } else { p.lift(); - CollectionItemKind::Unnamed + p.last_child().unwrap().kind().clone() } } diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs index 8a480b02..7c500ce7 100644 --- a/src/parse/tokens.rs +++ b/src/parse/tokens.rs @@ -91,7 +91,7 @@ impl<'s> Iterator for Tokens<'s> { '/' if self.s.eat_if('*') => self.block_comment(), '/' if !self.maybe_in_url() && self.s.eat_if('/') => self.line_comment(), '*' if self.s.eat_if('/') => { - NodeKind::Error(ErrorPosition::Full, self.s.eaten_from(start).into()) + NodeKind::Unknown(self.s.eaten_from(start).into()) } // Other things. @@ -173,7 +173,7 @@ impl<'s> Tokens<'s> { // Strings. '"' => self.string(), - _ => NodeKind::Error(ErrorPosition::Full, self.s.eaten_from(start).into()), + _ => NodeKind::Unknown(self.s.eaten_from(start).into()), } } @@ -398,10 +398,10 @@ impl<'s> Tokens<'s> { } else { NodeKind::Error( ErrorPosition::End, - if display { + if !display || (!escaped && dollar) { "expected closing dollar sign" } else { - "expected display math closure sequence" + "expected closing bracket and dollar sign" } .into(), ) @@ -466,11 +466,11 @@ impl<'s> Tokens<'s> { "deg" => NodeKind::Angle(f, AngularUnit::Deg), "rad" => NodeKind::Angle(f, AngularUnit::Rad), _ => { - return NodeKind::Error(ErrorPosition::Full, all.into()); + return NodeKind::Unknown(all.into()); } } } else { - NodeKind::Error(ErrorPosition::Full, all.into()) + NodeKind::Unknown(all.into()) } } @@ -575,45 +575,31 @@ mod tests { text: &str, lang: Option<&str>, backticks_left: u8, - backticks_right: u8, + err_msg: Option<&str>, block: bool, ) -> NodeKind { - if backticks_left == backticks_right { - NodeKind::Raw(Rc::new(RawToken { + match err_msg { + None => NodeKind::Raw(Rc::new(RawToken { text: text.into(), lang: lang.map(Into::into), backticks: backticks_left, block, - })) - } else { - let remaining = backticks_left - backticks_right; - let noun = if remaining == 1 { "backtick" } else { "backticks" }; - - NodeKind::Error( - ErrorPosition::End, - if backticks_right == 0 { - format!("expected {} {}", remaining, noun) - } else { - format!("expected {} more {}", remaining, noun) - } - .into(), - ) + })), + Some(msg) => { + NodeKind::Error(ErrorPosition::End, format!("expected {}", msg).into()) + } } } - fn Math(formula: &str, display: bool, terminated: bool) -> NodeKind { - if terminated { - NodeKind::Math(Rc::new(MathToken { formula: formula.into(), display })) - } else { - NodeKind::Error( + fn Math(formula: &str, display: bool, err_msg: Option<&str>) -> NodeKind { + match err_msg { + None => { + NodeKind::Math(Rc::new(MathToken { formula: formula.into(), display })) + } + Some(msg) => NodeKind::Error( ErrorPosition::End, - if display { - "expected closing dollar sign" - } else { - "expected display math closure sequence" - } - .into(), - ) + format!("expected closing {}", msg).into(), + ), } } @@ -634,7 +620,7 @@ mod tests { } fn Invalid(invalid: &str) -> NodeKind { - NodeKind::Error(ErrorPosition::Full, invalid.into()) + NodeKind::Unknown(invalid.into()) } /// Building blocks for suffix testing. @@ -687,7 +673,7 @@ mod tests { ('/', None, "//", LineComment), ('/', None, "/**/", BlockComment), ('/', Some(Markup), "*", Strong), - ('/', Some(Markup), "$ $", Math(" ", false, true)), + ('/', Some(Markup), "$ $", Math(" ", false, None)), ('/', Some(Markup), r"\\", Text("\\")), ('/', Some(Markup), "#let", Let), ('/', Some(Code), "(", LeftParen), @@ -908,42 +894,42 @@ mod tests { #[test] fn test_tokenize_raw_blocks() { // Test basic raw block. - t!(Markup: "``" => Raw("", None, 1, 1, false)); - t!(Markup: "`raw`" => Raw("raw", None, 1, 1, false)); - t!(Markup[""]: "`]" => Raw("]", None, 1, 0, false)); + t!(Markup: "``" => Raw("", None, 1, None, false)); + t!(Markup: "`raw`" => Raw("raw", None, 1, None, false)); + t!(Markup[""]: "`]" => Raw("]", None, 1, Some("1 backtick"), false)); // Test special symbols in raw block. - t!(Markup: "`[brackets]`" => Raw("[brackets]", None, 1, 1, false)); - t!(Markup[""]: r"`\`` " => Raw(r"\", None, 1, 1, false), Raw(" ", None, 1, 0, false)); + t!(Markup: "`[brackets]`" => Raw("[brackets]", None, 1, None, false)); + t!(Markup[""]: r"`\`` " => Raw(r"\", None, 1, None, false), Raw(" ", None, 1, Some("1 backtick"), false)); // Test separated closing backticks. - t!(Markup: "```not `y`e`t```" => Raw("`y`e`t", Some("not"), 3, 3, false)); + t!(Markup: "```not `y`e`t```" => Raw("`y`e`t", Some("not"), 3, None, false)); // Test more backticks. - t!(Markup: "``nope``" => Raw("", None, 1, 1, false), Text("nope"), Raw("", None, 1, 1, false)); - t!(Markup: "````🚀````" => Raw("", Some("🚀"), 4, 4, false)); - t!(Markup[""]: "`````👩🚀````noend" => Raw("````noend", Some("👩🚀"), 5, 0, false)); - t!(Markup[""]: "````raw``````" => Raw("", Some("raw"), 4, 4, false), Raw("", None, 1, 1, false)); + t!(Markup: "``nope``" => Raw("", None, 1, None, false), Text("nope"), Raw("", None, 1, None, false)); + t!(Markup: "````🚀````" => Raw("", Some("🚀"), 4, None, false)); + t!(Markup[""]: "`````👩🚀````noend" => Raw("````noend", Some("👩🚀"), 5, Some("5 backticks"), false)); + t!(Markup[""]: "````raw``````" => Raw("", Some("raw"), 4, None, false), Raw("", None, 1, None, false)); } #[test] fn test_tokenize_math_formulas() { // Test basic formula. - t!(Markup: "$$" => Math("", false, true)); - t!(Markup: "$x$" => Math("x", false, true)); - t!(Markup: r"$\\$" => Math(r"\\", false, true)); - t!(Markup: "$[x + y]$" => Math("x + y", true, true)); - t!(Markup: r"$[\\]$" => Math(r"\\", true, true)); + t!(Markup: "$$" => Math("", false, None)); + t!(Markup: "$x$" => Math("x", false, None)); + t!(Markup: r"$\\$" => Math(r"\\", false, None)); + t!(Markup: "$[x + y]$" => Math("x + y", true, None)); + t!(Markup: r"$[\\]$" => Math(r"\\", true, None)); // Test unterminated. - t!(Markup[""]: "$x" => Math("x", false, false)); - t!(Markup[""]: "$[x" => Math("x", true, false)); - t!(Markup[""]: "$[x]\n$" => Math("x]\n$", true, false)); + t!(Markup[""]: "$x" => Math("x", false, Some("dollar sign"))); + t!(Markup[""]: "$[x" => Math("x", true, Some("bracket and dollar sign"))); + t!(Markup[""]: "$[x]\n$" => Math("x]\n$", true, Some("bracket and dollar sign"))); // Test escape sequences. - t!(Markup: r"$\$x$" => Math(r"\$x", false, true)); - t!(Markup: r"$[\\\]$]$" => Math(r"\\\]$", true, true)); - t!(Markup[""]: r"$[ ]\\$" => Math(r" ]\\$", true, false)); + t!(Markup: r"$\$x$" => Math(r"\$x", false, None)); + t!(Markup: r"$[\\\]$]$" => Math(r"\\\]$", true, None)); + t!(Markup[""]: r"$[ ]\\$" => Math(r" ]\\$", true, Some("bracket and dollar sign"))); } #[test] |