diff options
Diffstat (limited to 'src/parse')
| -rw-r--r-- | src/parse/mod.rs | 1088 | ||||
| -rw-r--r-- | src/parse/parser.rs | 415 | ||||
| -rw-r--r-- | src/parse/resolve.rs | 40 | ||||
| -rw-r--r-- | src/parse/tokens.rs | 519 |
4 files changed, 1205 insertions, 857 deletions
diff --git a/src/parse/mod.rs b/src/parse/mod.rs index 30787423..dc769183 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -12,215 +12,213 @@ pub use tokens::*; use std::rc::Rc; -use crate::diag::TypResult; use crate::source::SourceFile; use crate::syntax::*; use crate::util::EcoString; /// Parse a source file. -pub fn parse(source: &SourceFile) -> TypResult<Markup> { +pub fn parse(source: &SourceFile) -> Rc<GreenNode> { let mut p = Parser::new(source); - let markup = markup(&mut p); - let errors = p.finish(); - if errors.is_empty() { - Ok(markup) - } else { - Err(Box::new(errors)) - } + markup(&mut p); + p.finish() } /// Parse markup. -fn markup(p: &mut Parser) -> Markup { +fn markup(p: &mut Parser) { markup_while(p, true, &mut |_| true) } -/// Parse markup that stays equal or right of the given column. -fn markup_indented(p: &mut Parser, column: usize) -> Markup { +/// Parse markup that stays right of the given column. +fn markup_indented(p: &mut Parser, column: usize) { + // TODO this is broken p.eat_while(|t| match t { - Token::Space(n) => n == 0, - Token::LineComment(_) | Token::BlockComment(_) => true, + NodeKind::Space(n) => n == 0, + NodeKind::LineComment | NodeKind::BlockComment => true, _ => false, }); markup_while(p, false, &mut |p| match p.peek() { - Some(Token::Space(n)) if n >= 1 => p.column(p.next_end()) >= column, + Some(NodeKind::Space(n)) if n >= 1 => p.column(p.next_end()) >= column, _ => true, }) } -/// Parse a syntax tree while the peeked token satisifies a condition. +/// Parse a syntax tree while the peeked NodeKind satisifies a condition. /// /// If `at_start` is true, things like headings that may only appear at the /// beginning of a line or template are allowed. -fn markup_while<F>(p: &mut Parser, mut at_start: bool, f: &mut F) -> Markup +fn markup_while<F>(p: &mut Parser, mut at_start: bool, f: &mut F) where F: FnMut(&mut Parser) -> bool, { - let mut tree = vec![]; + p.start(); while !p.eof() && f(p) { - if let Some(node) = markup_node(p, &mut at_start) { - at_start &= matches!(node, MarkupNode::Space | MarkupNode::Parbreak(_)); - tree.push(node); + markup_node(p, &mut at_start); + if let Some(node) = p.last_child() { + at_start &= matches!(node.kind(), &NodeKind::Space(_) | &NodeKind::Parbreak | &NodeKind::LineComment | &NodeKind::BlockComment); } } - tree + p.end(NodeKind::Markup); } /// Parse a markup node. -fn markup_node(p: &mut Parser, at_start: &mut bool) -> Option<MarkupNode> { - let token = p.peek()?; - let span = p.peek_span(); - let node = match token { - // Whitespace. - Token::Space(newlines) => { - *at_start |= newlines > 0; - if newlines < 2 { - MarkupNode::Space - } else { - MarkupNode::Parbreak(span) +fn markup_node(p: &mut Parser, at_start: &mut bool) { + if let Some(token) = p.peek() { + match token { + // Whitespace. + NodeKind::Space(newlines) => { + *at_start |= newlines > 0; + + if newlines < 2 { + p.eat(); + } else { + p.convert(NodeKind::Parbreak); + } } - } - // Text. - Token::Text(text) => MarkupNode::Text(text.into()), - Token::Tilde => MarkupNode::Text("\u{00A0}".into()), - Token::HyphHyph => MarkupNode::Text("\u{2013}".into()), - Token::HyphHyphHyph => MarkupNode::Text("\u{2014}".into()), - Token::UnicodeEscape(t) => MarkupNode::Text(unicode_escape(p, t)), - - // Markup. - Token::Backslash => MarkupNode::Linebreak(span), - Token::Star => MarkupNode::Strong(span), - Token::Underscore => MarkupNode::Emph(span), - Token::Raw(t) => raw(p, t), - Token::Eq if *at_start => return Some(heading(p)), - Token::Hyph if *at_start => return Some(list_node(p)), - Token::Numbering(number) if *at_start => return Some(enum_node(p, number)), - - // Line-based markup that is not currently at the start of the line. - Token::Eq | Token::Hyph | Token::Numbering(_) => { - MarkupNode::Text(p.peek_src().into()) - } + // Text. + NodeKind::UnicodeEscape(u) => { + if !u.terminated { + p.convert(NodeKind::Error( + ErrorPosition::End, + "expected closing brace".into(), + )); + p.unsuccessful(); + return; + } + + if u.character.is_none() { + let src = p.peek_src(); + p.convert(NodeKind::Error( + ErrorPosition::Full, + "invalid unicode escape sequence".into(), + )); + p.start(); + p.end(NodeKind::Text(src.into())); + return; + } - // Hashtag + keyword / identifier. - Token::Ident(_) - | Token::Let - | Token::If - | Token::While - | Token::For - | Token::Import - | Token::Include => { - let stmt = matches!(token, Token::Let | Token::Import); - let group = if stmt { Group::Stmt } else { Group::Expr }; - - p.start_group(group, TokenMode::Code); - let expr = expr_with(p, true, 0); - if stmt && expr.is_some() && !p.eof() { - p.expected_at(p.prev_end(), "semicolon or line break"); + p.eat(); } - p.end_group(); + NodeKind::Raw(r) => { + if !r.terminated { + p.convert(NodeKind::Error( + ErrorPosition::End, + "expected backtick(s)".into(), + )); + p.unsuccessful(); + return; + } - return expr.map(MarkupNode::Expr); - } + p.eat(); + } + NodeKind::Text(_) + | NodeKind::EnDash + | NodeKind::EmDash + | NodeKind::NonBreakingSpace => { + p.eat(); + } - // Block and template. - Token::LeftBrace => return Some(MarkupNode::Expr(block(p))), - Token::LeftBracket => return Some(MarkupNode::Expr(template(p))), + // Markup. + NodeKind::Emph | NodeKind::Strong | NodeKind::Linebreak => { + p.eat(); + } - // Comments. - Token::LineComment(_) | Token::BlockComment(_) => { - p.eat(); - return None; - } + NodeKind::Eq if *at_start => heading(p), + NodeKind::ListBullet if *at_start => list_node(p), + NodeKind::EnumNumbering(_) if *at_start => enum_node(p), - _ => { - *at_start = false; - p.unexpected(); - return None; - } - }; - p.eat(); - Some(node) -} + // Line-based markup that is not currently at the start of the line. + NodeKind::Eq | NodeKind::ListBullet | NodeKind::EnumNumbering(_) => { + p.convert(NodeKind::Text(p.peek_src().into())) + } -/// Handle a unicode escape sequence. -fn unicode_escape(p: &mut Parser, token: UnicodeEscapeToken) -> EcoString { - let span = p.peek_span(); - let text = if let Some(c) = resolve::resolve_hex(token.sequence) { - c.into() - } else { - // Print out the escape sequence verbatim if it is invalid. - p.error(span, "invalid unicode escape sequence"); - p.peek_src().into() - }; + // Hashtag + keyword / identifier. + NodeKind::Ident(_) + | NodeKind::Let + | NodeKind::If + | NodeKind::While + | NodeKind::For + | NodeKind::Import + | NodeKind::Include => { + let stmt = matches!(token, NodeKind::Let | NodeKind::Import); + let group = if stmt { Group::Stmt } else { Group::Expr }; + + p.start_group(group, TokenMode::Code); + expr_with(p, true, 0); + if stmt && p.success() && !p.eof() { + p.expected_at("semicolon or line break"); + } + p.end_group(); + } - if !token.terminated { - p.error(span.end, "expected closing brace"); - } + // Block and template. + NodeKind::LeftBrace => { + block(p); + } + NodeKind::LeftBracket => { + template(p); + } - text -} + // Comments. + NodeKind::LineComment | NodeKind::BlockComment => { + p.eat(); + } -/// Handle a raw block. -fn raw(p: &mut Parser, token: RawToken) -> MarkupNode { - let column = p.column(p.next_start()); - let span = p.peek_span(); - let raw = resolve::resolve_raw(span, column, token.backticks, token.text); - if !token.terminated { - p.error(span.end, "expected backtick(s)"); + _ => { + *at_start = false; + p.unexpected(); + } + }; } - MarkupNode::Raw(Box::new(raw)) } /// Parse a heading. -fn heading(p: &mut Parser) -> MarkupNode { - let start = p.next_start(); - p.eat_assert(Token::Eq); +fn heading(p: &mut Parser) { + p.start(); + p.start(); + p.eat_assert(NodeKind::Eq); // Count depth. let mut level: usize = 1; - while p.eat_if(Token::Eq) { + while p.eat_if(NodeKind::Eq) { level += 1; } if level > 6 { - return MarkupNode::Text(p.get(start .. p.prev_end()).into()); + p.lift(); + p.end(NodeKind::Text(EcoString::from('=').repeat(level))); + } else { + p.end(NodeKind::HeadingLevel(level as u8)); + let column = p.column(p.prev_end()); + markup_indented(p, column); + p.end(NodeKind::Heading); } - - let column = p.column(p.prev_end()); - let body = markup_indented(p, column); - MarkupNode::Heading(Box::new(HeadingNode { - span: p.span_from(start), - level, - body, - })) } /// Parse a single list item. -fn list_node(p: &mut Parser) -> MarkupNode { - let start = p.next_start(); - p.eat_assert(Token::Hyph); +fn list_node(p: &mut Parser) { + p.start(); + p.eat_assert(NodeKind::ListBullet); let column = p.column(p.prev_end()); - let body = markup_indented(p, column); - MarkupNode::List(Box::new(ListNode { span: p.span_from(start), body })) + markup_indented(p, column); + p.end(NodeKind::List); } /// Parse a single enum item. -fn enum_node(p: &mut Parser, number: Option<usize>) -> MarkupNode { - let start = p.next_start(); - p.eat_assert(Token::Numbering(number)); +fn enum_node(p: &mut Parser) { + p.start(); + if !matches!(p.eat(), Some(NodeKind::EnumNumbering(_))) { + panic!("enum item does not start with numbering") + }; let column = p.column(p.prev_end()); - let body = markup_indented(p, column); - MarkupNode::Enum(Box::new(EnumNode { - span: p.span_from(start), - number, - body, - })) + markup_indented(p, column); + p.end(NodeKind::Enum); } /// Parse an expression. -fn expr(p: &mut Parser) -> Option<Expr> { +fn expr(p: &mut Parser) { expr_with(p, false, 0) } @@ -231,134 +229,167 @@ fn expr(p: &mut Parser) -> Option<Expr> { /// in markup. /// /// Stops parsing at operations with lower precedence than `min_prec`, -fn expr_with(p: &mut Parser, atomic: bool, min_prec: usize) -> Option<Expr> { - let start = p.next_start(); - let mut lhs = match p.eat_map(UnOp::from_token) { +fn expr_with(p: &mut Parser, atomic: bool, min_prec: usize) { + p.start(); + let mut offset = p.child_count(); + // Start the unary expression. + match p.eat_map(|x| UnOp::from_token(&x)) { Some(op) => { let prec = op.precedence(); - let expr = expr_with(p, atomic, prec)?; - Expr::Unary(Box::new(UnaryExpr { span: p.span_from(start), op, expr })) + expr_with(p, atomic, prec); + + if p.may_lift_abort() { + return; + } + + p.end_and_start_with(NodeKind::Unary); + } + None => { + primary(p, atomic); + if p.may_lift_abort() { + return; + } } - None => primary(p, atomic)?, }; loop { // Exclamation mark, parenthesis or bracket means this is a function // call. - if matches!(p.peek_direct(), Some(Token::LeftParen | Token::LeftBracket)) { - lhs = call(p, lhs)?; + if matches!( + p.peek_direct(), + Some(NodeKind::LeftParen | NodeKind::LeftBracket) + ) { + call(p, p.child_count() - offset); continue; } - if p.eat_if(Token::With) { - lhs = with_expr(p, lhs)?; + if p.peek() == Some(NodeKind::With) { + with_expr(p, p.child_count() - offset); + + if p.may_lift_abort() { + return; + } } if atomic { + p.lift(); break; } - let op = match p.peek().and_then(BinOp::from_token) { + let op = match p.peek().as_ref().and_then(BinOp::from_token) { Some(binop) => binop, - None => break, + None => { + p.lift(); + break; + } }; let mut prec = op.precedence(); if prec < min_prec { - break; + { + p.lift(); + break; + }; } p.eat(); + match op.associativity() { Associativity::Left => prec += 1, Associativity::Right => {} } - let rhs = match expr_with(p, atomic, prec) { - Some(rhs) => rhs, - None => break, - }; + expr_with(p, atomic, prec); - let span = lhs.span().join(rhs.span()); - lhs = Expr::Binary(Box::new(BinaryExpr { span, lhs, op, rhs })); - } + if !p.success() { + p.lift(); + break; + } - Some(lhs) + offset = p.end_and_start_with(NodeKind::Binary).0; + } } /// Parse a primary expression. -fn primary(p: &mut Parser, atomic: bool) -> Option<Expr> { - if let Some(expr) = literal(p) { - return Some(expr); +fn primary(p: &mut Parser, atomic: bool) { + if literal(p) { + return; } match p.peek() { // Things that start with an identifier. - Some(Token::Ident(string)) => { - let ident = Ident { - span: p.eat_span(), - string: string.into(), - }; + Some(NodeKind::Ident(_)) => { + // Start closure params. + p.start(); + p.eat(); // Arrow means this is a closure's lone parameter. - Some(if !atomic && p.eat_if(Token::Arrow) { - let body = expr(p)?; - Expr::Closure(Box::new(ClosureExpr { - span: ident.span.join(body.span()), - name: None, - params: vec![ClosureParam::Pos(ident)], - body: Rc::new(body), - })) + if !atomic && p.peek() == Some(NodeKind::Arrow) { + p.end_and_start_with(NodeKind::ClosureParams); + p.eat(); + + expr(p); + + p.end_or_abort(NodeKind::Closure); } else { - Expr::Ident(Box::new(ident)) - }) + p.lift(); + } } // Structures. - Some(Token::LeftParen) => parenthesized(p), - Some(Token::LeftBracket) => Some(template(p)), - Some(Token::LeftBrace) => Some(block(p)), + Some(NodeKind::LeftParen) => parenthesized(p), + Some(NodeKind::LeftBracket) => template(p), + Some(NodeKind::LeftBrace) => block(p), // Keywords. - Some(Token::Let) => let_expr(p), - Some(Token::If) => if_expr(p), - Some(Token::While) => while_expr(p), - Some(Token::For) => for_expr(p), - Some(Token::Import) => import_expr(p), - Some(Token::Include) => include_expr(p), + Some(NodeKind::Let) => let_expr(p), + Some(NodeKind::If) => if_expr(p), + Some(NodeKind::While) => while_expr(p), + Some(NodeKind::For) => for_expr(p), + Some(NodeKind::Import) => import_expr(p), + Some(NodeKind::Include) => include_expr(p), // Nothing. _ => { p.expected("expression"); - None + p.unsuccessful(); } } } /// Parse a literal. -fn literal(p: &mut Parser) -> Option<Expr> { - let span = p.peek_span(); - let lit = match p.peek()? { +fn literal(p: &mut Parser) -> bool { + let peeked = if let Some(p) = p.peek() { + p + } else { + return false; + }; + + match peeked { // Basic values. - Token::None => Lit::None(span), - Token::Auto => Lit::Auto(span), - Token::Bool(b) => Lit::Bool(span, b), - Token::Int(i) => Lit::Int(span, i), - Token::Float(f) => Lit::Float(span, f), - Token::Length(val, unit) => Lit::Length(span, val, unit), - Token::Angle(val, unit) => Lit::Angle(span, val, unit), - Token::Percent(p) => Lit::Percent(span, p), - Token::Fraction(p) => Lit::Fractional(span, p), - Token::Str(token) => Lit::Str(span, { - if !token.terminated { - p.expected_at(span.end, "quote"); + NodeKind::None + | NodeKind::Auto + | NodeKind::Int(_) + | NodeKind::Float(_) + | NodeKind::Bool(_) + | NodeKind::Fraction(_) + | NodeKind::Length(_, _) + | NodeKind::Angle(_, _) + | NodeKind::Percentage(_) => { + p.eat(); + } + NodeKind::Str(s) => { + p.eat(); + if !s.terminated { + p.expected_at("quote"); } - resolve::resolve_string(token.string) - }), - _ => return None, - }; - p.eat(); - Some(Expr::Lit(Box::new(lit))) + } + _ => { + return false; + } + } + + true } /// Parse something that starts with a parenthesis, which can be either of: @@ -366,433 +397,508 @@ fn literal(p: &mut Parser) -> Option<Expr> { /// - Dictionary literal /// - Parenthesized expression /// - Parameter list of closure expression -fn parenthesized(p: &mut Parser) -> Option<Expr> { +fn parenthesized(p: &mut Parser) { + let offset = p.child_count(); + p.start(); p.start_group(Group::Paren, TokenMode::Code); - let colon = p.eat_if(Token::Colon); - let (items, has_comma) = collection(p); - let span = p.end_group(); + let colon = p.eat_if(NodeKind::Colon); + let kind = collection(p).0; + p.end_group(); + let token_count = p.child_count() - offset; - // Leading colon makes this a dictionary. + // Leading colon makes this a (empty) dictionary. if colon { - return Some(dict(p, items, span)); + p.lift(); + dict(p, token_count); + return; } // Arrow means this is a closure's parameter list. - if p.eat_if(Token::Arrow) { - let params = params(p, items); - let body = expr(p)?; - return Some(Expr::Closure(Box::new(ClosureExpr { - span: span.join(body.span()), - name: None, - params, - body: Rc::new(body), - }))); + if p.peek() == Some(NodeKind::Arrow) { + p.start_with(token_count); + params(p, 0, true); + p.end(NodeKind::ClosureParams); + + p.eat_assert(NodeKind::Arrow); + + expr(p); + + p.end_or_abort(NodeKind::Closure); + return; } // Find out which kind of collection this is. - Some(match items.as_slice() { - [] => array(p, items, span), - [CallArg::Pos(_)] if !has_comma => match items.into_iter().next() { - Some(CallArg::Pos(expr)) => Expr::Group(Box::new(GroupExpr { span, expr })), - _ => unreachable!(), - }, - [CallArg::Pos(_), ..] => array(p, items, span), - [CallArg::Named(_), ..] => dict(p, items, span), - [CallArg::Spread(expr), ..] => { - p.error(expr.span(), "spreading is not allowed here"); - return None; + match kind { + CollectionKind::Group => p.end(NodeKind::Group), + CollectionKind::PositionalCollection => { + p.lift(); + array(p, token_count); } - }) + CollectionKind::NamedCollection => { + p.lift(); + dict(p, token_count); + } + } +} + +/// The type of a collection. +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +enum CollectionKind { + /// The collection is only one item and has no comma. + Group, + /// The collection starts with a positional and has more items or a trailing + /// comma. + PositionalCollection, + /// The collection starts with a named item. + NamedCollection, } /// Parse a collection. /// -/// Returns whether the literal contained any commas. -fn collection(p: &mut Parser) -> (Vec<CallArg>, bool) { - let mut items = vec![]; +/// Returns the length of the collection and whether the literal contained any +/// commas. +fn collection(p: &mut Parser) -> (CollectionKind, usize) { + let mut items = 0; + let mut kind = CollectionKind::PositionalCollection; + let mut seen_spread = false; let mut has_comma = false; let mut missing_coma = None; while !p.eof() { - if let Some(arg) = item(p) { - items.push(arg); + let item_kind = item(p); + if p.success() { + if items == 0 && item_kind == CollectionItemKind::Named { + kind = CollectionKind::NamedCollection; + } + + if item_kind == CollectionItemKind::ParameterSink { + seen_spread = true; + } + + items += 1; if let Some(pos) = missing_coma.take() { - p.expected_at(pos, "comma"); + p.expected_at_child(pos, "comma"); } if p.eof() { break; } - let behind = p.prev_end(); - if p.eat_if(Token::Comma) { + if p.eat_if(NodeKind::Comma) { has_comma = true; } else { - missing_coma = Some(behind); + missing_coma = Some(p.child_count()); } } } - (items, has_comma) + if !has_comma + && items == 1 + && !seen_spread + && kind == CollectionKind::PositionalCollection + { + kind = CollectionKind::Group; + } + + (kind, items) } -/// Parse an expression or a named pair. -fn item(p: &mut Parser) -> Option<CallArg> { - if p.eat_if(Token::Dots) { - return expr(p).map(CallArg::Spread); +/// What kind of item is this? +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +enum CollectionItemKind { + /// A named item. + Named, + /// An unnamed item. + Unnamed, + /// A parameter sink. + ParameterSink, +} + +/// Parse an expression or a named pair. Returns if this is a named pair. +fn item(p: &mut Parser) -> CollectionItemKind { + p.start(); + if p.eat_if(NodeKind::Dots) { + expr(p); + + p.end_or_abort(NodeKind::ParameterSink); + return CollectionItemKind::ParameterSink; + } + + expr(p); + + if p.may_lift_abort() { + return CollectionItemKind::Unnamed; } - let first = expr(p)?; - if p.eat_if(Token::Colon) { - if let Expr::Ident(name) = first { - Some(CallArg::Named(Named { name: *name, expr: expr(p)? })) + if p.eat_if(NodeKind::Colon) { + let child = p.child(1).unwrap(); + if matches!(child.kind(), &NodeKind::Ident(_)) { + expr(p); + p.end_or_abort(NodeKind::Named); } else { - p.error(first.span(), "expected identifier"); + p.wrap( + 1, + NodeKind::Error(ErrorPosition::Full, "expected identifier".into()), + ); + expr(p); - None + p.end(NodeKind::Named); + p.unsuccessful(); } + + CollectionItemKind::Named } else { - Some(CallArg::Pos(first)) + p.lift(); + CollectionItemKind::Unnamed } } /// Convert a collection into an array, producing errors for anything other than /// expressions. -fn array(p: &mut Parser, items: Vec<CallArg>, span: Span) -> Expr { - let iter = items.into_iter().filter_map(|item| match item { - CallArg::Pos(expr) => Some(expr), - CallArg::Named(_) => { - p.error(item.span(), "expected expression, found named pair"); - None - } - CallArg::Spread(_) => { - p.error(item.span(), "spreading is not allowed here"); - None - } - }); - Expr::Array(Box::new(ArrayExpr { span, items: iter.collect() })) +fn array(p: &mut Parser, items: usize) { + p.start_with(items); + p.filter_children( + 0, + |x| match x.kind() { + NodeKind::Named | NodeKind::ParameterSink => false, + _ => true, + }, + |kind| match kind { + NodeKind::Named => ( + ErrorPosition::Full, + "expected expression, found named pair".into(), + ), + NodeKind::ParameterSink => { + (ErrorPosition::Full, "spreading is not allowed here".into()) + } + _ => unreachable!(), + }, + ); + + p.end(NodeKind::Array) } /// Convert a collection into a dictionary, producing errors for anything other /// than named pairs. -fn dict(p: &mut Parser, items: Vec<CallArg>, span: Span) -> Expr { - let iter = items.into_iter().filter_map(|item| match item { - CallArg::Named(named) => Some(named), - CallArg::Pos(_) => { - p.error(item.span(), "expected named pair, found expression"); - None - } - CallArg::Spread(_) => { - p.error(item.span(), "spreading is not allowed here"); - None - } - }); - Expr::Dict(Box::new(DictExpr { span, items: iter.collect() })) +fn dict(p: &mut Parser, items: usize) { + p.start_with(items); + p.filter_children( + 0, + |x| { + x.kind() == &NodeKind::Named + || x.kind().is_parenthesis() + || x.kind() == &NodeKind::Comma + || x.kind() == &NodeKind::Colon + }, + |kind| match kind { + NodeKind::ParameterSink => { + (ErrorPosition::Full, "spreading is not allowed here".into()) + } + _ => ( + ErrorPosition::Full, + "expected named pair, found expression".into(), + ), + }, + ); + p.end(NodeKind::Dict); } /// Convert a collection into a list of parameters, producing errors for /// anything other than identifiers, spread operations and named pairs. -fn params(p: &mut Parser, items: Vec<CallArg>) -> Vec<ClosureParam> { - let iter = items.into_iter().filter_map(|item| match item { - CallArg::Pos(Expr::Ident(ident)) => Some(ClosureParam::Pos(*ident)), - CallArg::Named(named) => Some(ClosureParam::Named(named)), - CallArg::Spread(Expr::Ident(ident)) => Some(ClosureParam::Sink(*ident)), - _ => { - p.error(item.span(), "expected identifier"); - None - } - }); - iter.collect() -} - -/// Convert a collection into a list of identifiers, producing errors for -/// anything other than identifiers. -fn idents(p: &mut Parser, items: Vec<CallArg>) -> Vec<Ident> { - let iter = items.into_iter().filter_map(|item| match item { - CallArg::Pos(Expr::Ident(ident)) => Some(*ident), - _ => { - p.error(item.span(), "expected identifier"); - None - } - }); - iter.collect() +fn params(p: &mut Parser, count: usize, allow_parens: bool) { + p.filter_children( + count, + |x| match x.kind() { + NodeKind::Named | NodeKind::Comma | NodeKind::Ident(_) => true, + NodeKind::ParameterSink => matches!( + x.children().last().map(|x| x.kind()), + Some(&NodeKind::Ident(_)) + ), + _ => false, + } + || (allow_parens && x.kind().is_parenthesis()), + |_| (ErrorPosition::Full, "expected identifier".into()), + ); } // Parse a template block: `[...]`. -fn template(p: &mut Parser) -> Expr { +fn template(p: &mut Parser) { + p.start(); p.start_group(Group::Bracket, TokenMode::Markup); - let tree = markup(p); - let span = p.end_group(); - Expr::Template(Box::new(TemplateExpr { span, body: tree })) + markup(p); + p.end_group(); + p.end(NodeKind::Template); } /// Parse a code block: `{...}`. -fn block(p: &mut Parser) -> Expr { +fn block(p: &mut Parser) { + p.start(); p.start_group(Group::Brace, TokenMode::Code); - let mut exprs = vec![]; while !p.eof() { p.start_group(Group::Stmt, TokenMode::Code); - if let Some(expr) = expr(p) { - exprs.push(expr); + expr(p); + if p.success() { if !p.eof() { - p.expected_at(p.prev_end(), "semicolon or line break"); + p.expected_at("semicolon or line break"); } } p.end_group(); // Forcefully skip over newlines since the group's contents can't. - p.eat_while(|t| matches!(t, Token::Space(_))); + p.eat_while(|t| matches!(t, NodeKind::Space(_))); } - let span = p.end_group(); - Expr::Block(Box::new(BlockExpr { span, exprs })) + p.end_group(); + p.end(NodeKind::Block); } /// Parse a function call. -fn call(p: &mut Parser, callee: Expr) -> Option<Expr> { - let mut args = match p.peek_direct() { - Some(Token::LeftParen) => args(p), - Some(Token::LeftBracket) => CallArgs { - span: Span::at(p.id(), callee.span().end), - items: vec![], - }, +fn call(p: &mut Parser, callee: usize) { + p.start_with(callee); + match p.peek_direct() { + Some(NodeKind::LeftParen) | Some(NodeKind::LeftBracket) => args(p, true), _ => { - p.expected_at(p.prev_end(), "argument list"); - return None; + p.expected_at("argument list"); + p.may_end_abort(NodeKind::Call); + return; } }; - while p.peek_direct() == Some(Token::LeftBracket) { - let body = template(p); - args.items.push(CallArg::Pos(body)); - } - - Some(Expr::Call(Box::new(CallExpr { - span: p.span_from(callee.span().start), - callee, - args, - }))) + p.end(NodeKind::Call); } /// Parse the arguments to a function call. -fn args(p: &mut Parser) -> CallArgs { - p.start_group(Group::Paren, TokenMode::Code); - let items = collection(p).0; - let span = p.end_group(); - CallArgs { span, items } +fn args(p: &mut Parser, allow_template: bool) { + p.start(); + if !allow_template || p.peek_direct() == Some(&NodeKind::LeftParen) { + p.start_group(Group::Paren, TokenMode::Code); + collection(p); + p.end_group(); + } + + while allow_template && p.peek_direct() == Some(&NodeKind::LeftBracket) { + template(p); + } + + p.end(NodeKind::CallArgs); } /// Parse a with expression. -fn with_expr(p: &mut Parser, callee: Expr) -> Option<Expr> { - if p.peek() == Some(Token::LeftParen) { - Some(Expr::With(Box::new(WithExpr { - span: p.span_from(callee.span().start), - callee, - args: args(p), - }))) +fn with_expr(p: &mut Parser, preserve: usize) { + p.start_with(preserve); + p.eat_assert(NodeKind::With); + + if p.peek() == Some(NodeKind::LeftParen) { + args(p, false); + p.end(NodeKind::WithExpr); } else { p.expected("argument list"); - None + p.may_end_abort(NodeKind::WithExpr); } } /// Parse a let expression. -fn let_expr(p: &mut Parser) -> Option<Expr> { - let start = p.next_start(); - p.eat_assert(Token::Let); - - let mut output = None; - if let Some(binding) = ident(p) { - let mut init = None; +fn let_expr(p: &mut Parser) { + p.start(); + p.eat_assert(NodeKind::Let); + + let offset = p.child_count(); + ident(p); + if p.may_end_abort(NodeKind::LetExpr) { + return; + } - if p.eat_if(Token::With) { - init = with_expr(p, Expr::Ident(Box::new(binding.clone()))); + if p.peek() == Some(NodeKind::With) { + with_expr(p, p.child_count() - offset); + } else { + // If a parenthesis follows, this is a function definition. + let has_params = if p.peek_direct() == Some(&NodeKind::LeftParen) { + p.start(); + p.start_group(Group::Paren, TokenMode::Code); + let offset = p.child_count(); + collection(p); + params(p, offset, true); + p.end_group(); + p.end(NodeKind::ClosureParams); + true } else { - // If a parenthesis follows, this is a function definition. - let mut maybe_params = None; - if p.peek_direct() == Some(Token::LeftParen) { - p.start_group(Group::Paren, TokenMode::Code); - let items = collection(p).0; - maybe_params = Some(params(p, items)); - p.end_group(); - } + false + }; - if p.eat_if(Token::Eq) { - init = expr(p); - } else if maybe_params.is_some() { - // Function definitions must have a body. - p.expected_at(p.prev_end(), "body"); - } + if p.eat_if(NodeKind::Eq) { + expr(p); + } else if has_params { + // Function definitions must have a body. + p.expected_at("body"); + } - // Rewrite into a closure expression if it's a function definition. - if let Some(params) = maybe_params { - let body = init?; - init = Some(Expr::Closure(Box::new(ClosureExpr { - span: binding.span.join(body.span()), - name: Some(binding.clone()), - params, - body: Rc::new(body), - }))); + // Rewrite into a closure expression if it's a function definition. + if has_params { + if p.may_end_abort(NodeKind::LetExpr) { + return; } - } - output = Some(Expr::Let(Box::new(LetExpr { - span: p.span_from(start), - binding, - init, - }))); + p.start_with(p.child_count() - offset); + p.end(NodeKind::Closure) + } } - output + p.end(NodeKind::LetExpr); } /// Parse an if expresion. -fn if_expr(p: &mut Parser) -> Option<Expr> { - let start = p.next_start(); - p.eat_assert(Token::If); - - let mut output = None; - if let Some(condition) = expr(p) { - if let Some(if_body) = body(p) { - let mut else_body = None; - if p.eat_if(Token::Else) { - if p.peek() == Some(Token::If) { - else_body = if_expr(p); - } else { - else_body = body(p); - } - } +fn if_expr(p: &mut Parser) { + p.start(); + p.eat_assert(NodeKind::If); - output = Some(Expr::If(Box::new(IfExpr { - span: p.span_from(start), - condition, - if_body, - else_body, - }))); + expr(p); + if p.may_end_abort(NodeKind::IfExpr) { + return; + } + + body(p); + if p.may_end_abort(NodeKind::IfExpr) { + // Expected function body. + return; + } + + if p.eat_if(NodeKind::Else) { + if p.peek() == Some(NodeKind::If) { + if_expr(p); + } else { + body(p); } } - output + p.end(NodeKind::IfExpr); } /// Parse a while expresion. -fn while_expr(p: &mut Parser) -> Option<Expr> { - let start = p.next_start(); - p.eat_assert(Token::While); - - let mut output = None; - if let Some(condition) = expr(p) { - if let Some(body) = body(p) { - output = Some(Expr::While(Box::new(WhileExpr { - span: p.span_from(start), - condition, - body, - }))); - } +fn while_expr(p: &mut Parser) { + p.start(); + p.eat_assert(NodeKind::While); + + expr(p); + + if p.may_end_abort(NodeKind::WhileExpr) { + return; } - output + body(p); + if !p.may_end_abort(NodeKind::WhileExpr) { + p.end(NodeKind::WhileExpr); + } } /// Parse a for expression. -fn for_expr(p: &mut Parser) -> Option<Expr> { - let start = p.next_start(); - p.eat_assert(Token::For); - - let mut output = None; - if let Some(pattern) = for_pattern(p) { - if p.eat_expect(Token::In) { - if let Some(iter) = expr(p) { - if let Some(body) = body(p) { - output = Some(Expr::For(Box::new(ForExpr { - span: p.span_from(start), - pattern, - iter, - body, - }))); - } - } - } +fn for_expr(p: &mut Parser) { + p.start(); + p.eat_assert(NodeKind::For); + + for_pattern(p); + + if p.may_end_abort(NodeKind::ForExpr) { + return; } - output + if p.eat_expect(NodeKind::In) { + expr(p); + + if p.may_end_abort(NodeKind::ForExpr) { + return; + } + + body(p); + + if !p.may_end_abort(NodeKind::ForExpr) { + p.end(NodeKind::ForExpr); + } + } else { + p.unsuccessful(); + p.may_end_abort(NodeKind::ForExpr); + } } /// Parse a for loop pattern. -fn for_pattern(p: &mut Parser) -> Option<ForPattern> { - let first = ident(p)?; - if p.eat_if(Token::Comma) { - if let Some(second) = ident(p) { - return Some(ForPattern::KeyValue(first, second)); +fn for_pattern(p: &mut Parser) { + p.start(); + ident(p); + + if p.may_end_abort(NodeKind::ForPattern) { + return; + } + + if p.peek() == Some(NodeKind::Comma) { + p.eat(); + + ident(p); + + if p.may_end_abort(NodeKind::ForPattern) { + return; } } - Some(ForPattern::Value(first)) + + p.end(NodeKind::ForPattern); } /// Parse an import expression. -fn import_expr(p: &mut Parser) -> Option<Expr> { - let start = p.next_start(); - p.eat_assert(Token::Import); +fn import_expr(p: &mut Parser) { + p.start(); + p.eat_assert(NodeKind::Import); - let imports = if p.eat_if(Token::Star) { - // This is the wildcard scenario. - Imports::Wildcard - } else { + if !p.eat_if(NodeKind::Star) { // This is the list of identifiers scenario. + p.start(); p.start_group(Group::Imports, TokenMode::Code); - let items = collection(p).0; - if items.is_empty() { - p.expected_at(p.prev_end(), "import items"); + let offset = p.child_count(); + let items = collection(p).1; + if items == 0 { + p.expected_at("import items"); } p.end_group(); - Imports::Idents(idents(p, items)) + + p.filter_children( + offset, + |n| matches!(n.kind(), NodeKind::Ident(_) | NodeKind::Comma), + |_| (ErrorPosition::Full, "expected identifier".into()), + ); + p.end(NodeKind::ImportItems); }; - let mut output = None; - if p.eat_expect(Token::From) { - if let Some(path) = expr(p) { - output = Some(Expr::Import(Box::new(ImportExpr { - span: p.span_from(start), - imports, - path, - }))); - } + if p.eat_expect(NodeKind::From) { + expr(p); } - output + p.end(NodeKind::ImportExpr); } /// Parse an include expression. -fn include_expr(p: &mut Parser) -> Option<Expr> { - let start = p.next_start(); - p.eat_assert(Token::Include); +fn include_expr(p: &mut Parser) { + p.start(); + p.eat_assert(NodeKind::Include); - expr(p).map(|path| { - Expr::Include(Box::new(IncludeExpr { span: p.span_from(start), path })) - }) + expr(p); + p.end(NodeKind::IncludeExpr); } /// Parse an identifier. -fn ident(p: &mut Parser) -> Option<Ident> { - if let Some(Token::Ident(string)) = p.peek() { - Some(Ident { - span: p.eat_span(), - string: string.into(), - }) +fn ident(p: &mut Parser) { + if let Some(NodeKind::Ident(_)) = p.peek() { + p.eat(); } else { p.expected("identifier"); - None + p.unsuccessful(); } } /// Parse a control flow body. -fn body(p: &mut Parser) -> Option<Expr> { +fn body(p: &mut Parser) { match p.peek() { - Some(Token::LeftBracket) => Some(template(p)), - Some(Token::LeftBrace) => Some(block(p)), + Some(NodeKind::LeftBracket) => template(p), + Some(NodeKind::LeftBrace) => block(p), _ => { - p.expected_at(p.prev_end(), "body"); - None + p.expected_at("body"); + p.unsuccessful(); } } } diff --git a/src/parse/parser.rs b/src/parse/parser.rs index 347d6f71..f62e882a 100644 --- a/src/parse/parser.rs +++ b/src/parse/parser.rs @@ -1,29 +1,34 @@ use std::ops::Range; +use std::rc::Rc; use super::{TokenMode, Tokens}; -use crate::diag::Error; use crate::source::{SourceFile, SourceId}; -use crate::syntax::{IntoSpan, Pos, Span, Token}; +use crate::syntax::{ErrorPosition, Green, GreenData, GreenNode, NodeKind}; +use crate::util::EcoString; /// A convenient token-based parser. pub struct Parser<'s> { /// The parsed file. source: &'s SourceFile, - /// Parsing errors. - errors: Vec<Error>, /// An iterator over the source tokens. tokens: Tokens<'s>, /// The stack of open groups. groups: Vec<GroupEntry>, /// The next token. - next: Option<Token<'s>>, + next: Option<NodeKind>, /// The peeked token. /// (Same as `next` except if we are at the end of group, then `None`). - peeked: Option<Token<'s>>, + peeked: Option<NodeKind>, /// The end index of the last (non-whitespace if in code mode) token. prev_end: usize, /// The start index of the peeked token. next_start: usize, + /// A stack of outer children vectors. + stack: Vec<Vec<Green>>, + /// The children of the currently built node. + children: Vec<Green>, + /// Whether the last parsing step was successful. + success: bool, } /// A logical group of tokens, e.g. `[...]`. @@ -32,9 +37,6 @@ struct GroupEntry { /// For example, a [`Group::Paren`] will be ended by /// [`Token::RightParen`]. pub kind: Group, - /// The start index of the group. Used by `Parser::end_group` to return the - /// group's full span. - pub start: usize, /// The mode the parser was in _before_ the group started (to which we go /// back once the group ends). pub prev_mode: TokenMode, @@ -60,51 +62,204 @@ pub enum Group { impl<'s> Parser<'s> { /// Create a new parser for the source string. pub fn new(source: &'s SourceFile) -> Self { - let mut tokens = Tokens::new(source.src(), TokenMode::Markup); + let mut tokens = Tokens::new(source, TokenMode::Markup); let next = tokens.next(); Self { source, - errors: vec![], tokens, groups: vec![], - next, + next: next.clone(), peeked: next, prev_end: 0, next_start: 0, + stack: vec![], + children: vec![], + success: true, } } - /// Finish parsing and return all errors. - pub fn finish(self) -> Vec<Error> { - self.errors - } - /// The id of the parsed source file. pub fn id(&self) -> SourceId { self.source.id() } + /// Start a nested node. + /// + /// Each start call has to be matched with a call to `end`, + /// `end_with_custom_children`, `lift`, `abort`, or `end_or_abort`. + pub fn start(&mut self) { + self.stack.push(std::mem::take(&mut self.children)); + } + + /// Start a nested node, preserving a number of the current children. + pub fn start_with(&mut self, preserve: usize) { + let preserved = self.children.drain(self.children.len() - preserve ..).collect(); + self.stack.push(std::mem::replace(&mut self.children, preserved)); + } + + /// Filter the last children using the given predicate. + pub fn filter_children<F, G>(&mut self, count: usize, f: F, error: G) + where + F: Fn(&Green) -> bool, + G: Fn(&NodeKind) -> (ErrorPosition, EcoString), + { + for child in &mut self.children[count ..] { + if !((self.tokens.mode() != TokenMode::Code + || Self::skip_type_ext(child.kind(), false)) + || child.kind().is_error() + || f(&child)) + { + let (pos, msg) = error(child.kind()); + let inner = std::mem::take(child); + *child = + GreenNode::with_child(NodeKind::Error(pos, msg), inner.len(), inner) + .into(); + } + } + } + + pub fn child(&self, child: usize) -> Option<&Green> { + self.node_index_from_back(child).map(|i| &self.children[i]) + } + + fn node_index_from_back(&self, child: usize) -> Option<usize> { + let len = self.children.len(); + let code = self.tokens.mode() == TokenMode::Code; + let mut seen = 0; + for x in (0 .. len).rev() { + if self.skip_type(self.children[x].kind()) && code { + continue; + } + if seen == child { + return Some(x); + } + seen += 1; + } + + None + } + + /// End the current node as a node of given `kind`. + pub fn end(&mut self, kind: NodeKind) { + let outer = self.stack.pop().unwrap(); + let mut children = std::mem::replace(&mut self.children, outer); + + // have trailing whitespace continue to sit in self.children in code + // mode. + let mut remains = vec![]; + if self.tokens.mode() == TokenMode::Code { + let len = children.len(); + for n in (0 .. len).rev() { + if !self.skip_type(&children[n].kind()) { + break; + } + + remains.push(children.pop().unwrap()); + } + remains.reverse(); + } + + let len = children.iter().map(|c| c.len()).sum(); + self.children + .push(GreenNode::with_children(kind, len, children.into_iter()).into()); + self.children.extend(remains); + self.success = true; + } + + /// End the current node as a node of given `kind`, and start a new node + /// with the ended node as a first child. The function returns how many + /// children the stack frame had before and how many were appended (accounts + /// for trivia). + pub fn end_and_start_with(&mut self, kind: NodeKind) -> (usize, usize) { + let stack_offset = self.stack.last().unwrap().len(); + self.end(kind); + let diff = self.children.len() - stack_offset; + self.start_with(diff); + (stack_offset, diff) + } + + pub fn wrap(&mut self, index: usize, kind: NodeKind) { + let index = self.node_index_from_back(index).unwrap(); + let child = std::mem::take(&mut self.children[index]); + let item = GreenNode::with_child(kind, child.len(), child); + self.children[index] = item.into(); + } + + pub fn convert(&mut self, kind: NodeKind) { + self.start(); + self.eat(); + self.end(kind); + } + + /// End the current node and undo its existence, inling all accumulated + /// children into its parent. + pub fn lift(&mut self) { + let outer = self.stack.pop().unwrap(); + let children = std::mem::replace(&mut self.children, outer); + self.children.extend(children); + self.success = true; + } + + /// End the current node and undo its existence, deleting all accumulated + /// children. + pub fn abort(&mut self, msg: impl Into<String>) { + self.end(NodeKind::Error(ErrorPosition::Full, msg.into().into())); + self.success = false; + } + + pub fn may_lift_abort(&mut self) -> bool { + if !self.success { + self.lift(); + self.success = false; + true + } else { + false + } + } + + pub fn may_end_abort(&mut self, kind: NodeKind) -> bool { + if !self.success { + self.end(kind); + self.success = false; + true + } else { + false + } + } + + /// End the current node as a node of given `kind` if the last parse was + /// successful, otherwise, abort. + pub fn end_or_abort(&mut self, kind: NodeKind) -> bool { + if self.success { + self.end(kind); + true + } else { + self.may_end_abort(kind); + false + } + } + + pub fn finish(&mut self) -> Rc<GreenNode> { + if let Green::Node(n) = self.children.pop().unwrap() { + n + } else { + panic!() + } + } + /// Whether the end of the source string or group is reached. pub fn eof(&self) -> bool { self.peek().is_none() } - /// Consume the next token. - pub fn eat(&mut self) -> Option<Token<'s>> { + pub fn eat(&mut self) -> Option<NodeKind> { let token = self.peek()?; self.bump(); Some(token) } - /// Eat the next token and return its source range. - pub fn eat_span(&mut self) -> Span { - let start = self.next_start(); - self.eat(); - Span::new(self.id(), start, self.prev_end()) - } - /// Consume the next token if it is the given one. - pub fn eat_if(&mut self, t: Token) -> bool { + pub fn eat_if(&mut self, t: NodeKind) -> bool { if self.peek() == Some(t) { self.bump(); true @@ -116,7 +271,7 @@ impl<'s> Parser<'s> { /// Consume the next token if the closure maps it a to `Some`-variant. pub fn eat_map<T, F>(&mut self, f: F) -> Option<T> where - F: FnOnce(Token<'s>) -> Option<T>, + F: FnOnce(NodeKind) -> Option<T>, { let token = self.peek()?; let mapped = f(token); @@ -128,16 +283,16 @@ impl<'s> Parser<'s> { /// Consume the next token if it is the given one and produce an error if /// not. - pub fn eat_expect(&mut self, t: Token) -> bool { - let eaten = self.eat_if(t); + pub fn eat_expect(&mut self, t: NodeKind) -> bool { + let eaten = self.eat_if(t.clone()); if !eaten { - self.expected_at(self.prev_end(), t.name()); + self.expected_at(&t.to_string()); } eaten } /// Consume the next token, debug-asserting that it is one of the given ones. - pub fn eat_assert(&mut self, t: Token) { + pub fn eat_assert(&mut self, t: NodeKind) { let next = self.eat(); debug_assert_eq!(next, Some(t)); } @@ -145,7 +300,7 @@ impl<'s> Parser<'s> { /// Consume tokens while the condition is true. pub fn eat_while<F>(&mut self, mut f: F) where - F: FnMut(Token<'s>) -> bool, + F: FnMut(NodeKind) -> bool, { while self.peek().map_or(false, |t| f(t)) { self.eat(); @@ -153,42 +308,25 @@ impl<'s> Parser<'s> { } /// Peek at the next token without consuming it. - pub fn peek(&self) -> Option<Token<'s>> { - self.peeked + pub fn peek(&self) -> Option<NodeKind> { + self.peeked.clone() } /// Peek at the next token if it follows immediately after the last one /// without any whitespace in between. - pub fn peek_direct(&self) -> Option<Token<'s>> { + pub fn peek_direct(&self) -> Option<&NodeKind> { if self.next_start() == self.prev_end() { - self.peeked + self.peeked.as_ref() } else { None } } - /// Peek at the span of the next token. - /// - /// Has length zero if `peek()` returns `None`. - pub fn peek_span(&self) -> Span { - Span::new(self.id(), self.next_start(), self.next_end()) - } - /// Peek at the source of the next token. pub fn peek_src(&self) -> &'s str { self.get(self.next_start() .. self.next_end()) } - /// Checks whether the next token fulfills a condition. - /// - /// Returns `false` if there is no next token. - pub fn check<F>(&self, f: F) -> bool - where - F: FnOnce(Token<'s>) -> bool, - { - self.peek().map_or(false, f) - } - /// The byte index at which the last token ended. /// /// Refers to the end of the last _non-whitespace_ token in code mode. @@ -219,11 +357,6 @@ impl<'s> Parser<'s> { self.source.get(range).unwrap() } - /// The span from `start` to [`self.prev_end()`](Self::prev_end). - pub fn span_from(&self, start: impl Into<Pos>) -> Span { - Span::new(self.id(), start, self.prev_end()) - } - /// Continue parsing in a group. /// /// When the end delimiter of the group is reached, all subsequent calls to @@ -232,19 +365,15 @@ impl<'s> Parser<'s> { /// /// This panics if the next token does not start the given group. pub fn start_group(&mut self, kind: Group, mode: TokenMode) { - self.groups.push(GroupEntry { - kind, - start: self.next_start(), - prev_mode: self.tokens.mode(), - }); + self.groups.push(GroupEntry { kind, prev_mode: self.tokens.mode() }); self.tokens.set_mode(mode); self.repeek(); match kind { - Group::Paren => self.eat_assert(Token::LeftParen), - Group::Bracket => self.eat_assert(Token::LeftBracket), - Group::Brace => self.eat_assert(Token::LeftBrace), + Group::Paren => self.eat_assert(NodeKind::LeftParen), + Group::Bracket => self.eat_assert(NodeKind::LeftBracket), + Group::Brace => self.eat_assert(NodeKind::LeftBrace), Group::Stmt => {} Group::Expr => {} Group::Imports => {} @@ -254,7 +383,7 @@ impl<'s> Parser<'s> { /// End the parsing of a group. /// /// This panics if no group was started. - pub fn end_group(&mut self) -> Span { + pub fn end_group(&mut self) { let prev_mode = self.tokens.mode(); let group = self.groups.pop().expect("no started group"); self.tokens.set_mode(group.prev_mode); @@ -264,83 +393,125 @@ impl<'s> Parser<'s> { // Eat the end delimiter if there is one. if let Some((end, required)) = match group.kind { - Group::Paren => Some((Token::RightParen, true)), - Group::Bracket => Some((Token::RightBracket, true)), - Group::Brace => Some((Token::RightBrace, true)), - Group::Stmt => Some((Token::Semicolon, false)), + Group::Paren => Some((NodeKind::RightParen, true)), + Group::Bracket => Some((NodeKind::RightBracket, true)), + Group::Brace => Some((NodeKind::RightBrace, true)), + Group::Stmt => Some((NodeKind::Semicolon, false)), Group::Expr => None, Group::Imports => None, } { - if self.next == Some(end) { + if self.next == Some(end.clone()) { // Bump the delimeter and return. No need to rescan in this case. self.bump(); rescan = false; } else if required { - self.error( - self.next_start() .. self.next_start(), - format!("expected {}", end.name()), - ); + self.start(); + self.abort(format!("expected {}", end.to_string())); } } // Rescan the peeked token if the mode changed. if rescan { self.tokens.jump(self.prev_end()); - self.bump(); - } - Span::new(self.id(), group.start, self.prev_end()) - } + if prev_mode == TokenMode::Code { + let len = self.children.len(); + for n in (0 .. len).rev() { + if !self.skip_type(self.children[n].kind()) { + break; + } + + self.children.pop(); + } + } - /// Add an error with location and message. - pub fn error(&mut self, span: impl IntoSpan, message: impl Into<String>) { - self.errors.push(Error::new(span.into_span(self.id()), message)); + self.fast_forward(); + } } /// Add an error that `what` was expected at the given span. - pub fn expected_at(&mut self, span: impl IntoSpan, what: &str) { - self.error(span, format!("expected {}", what)); + pub fn expected_at(&mut self, what: &str) { + let mut found = self.children.len(); + for (i, node) in self.children.iter().enumerate().rev() { + if !self.skip_type(node.kind()) { + break; + } + found = i; + } + + self.expected_at_child(found, what); + } + + /// Add an error that `what` was expected at the given child index. + pub fn expected_at_child(&mut self, index: usize, what: &str) { + self.children.insert( + index, + GreenData::new( + NodeKind::Error(ErrorPosition::Full, format!("expected {}", what).into()), + 0, + ) + .into(), + ); } /// Eat the next token and add an error that it is not the expected `thing`. pub fn expected(&mut self, what: &str) { - let before = self.next_start(); + self.start(); if let Some(found) = self.eat() { - let after = self.prev_end(); - self.error( - before .. after, - format!("expected {}, found {}", what, found.name()), - ); + self.abort(format!("expected {}, found {}", what, found.to_string())) } else { - self.expected_at(self.next_start(), what); + self.lift(); + self.expected_at(what); } } /// Eat the next token and add an error that it is unexpected. pub fn unexpected(&mut self) { - let before = self.next_start(); + self.start(); if let Some(found) = self.eat() { - let after = self.prev_end(); - self.error(before .. after, format!("unexpected {}", found.name())); + self.abort(format!("unexpected {}", found.to_string())) + } else { + self.abort("unexpected end of file") } } + pub fn skip_type_ext(token: &NodeKind, stop_at_newline: bool) -> bool { + match token { + NodeKind::Space(n) => n < &1 || !stop_at_newline, + NodeKind::LineComment => true, + NodeKind::BlockComment => true, + _ => false, + } + } + + fn skip_type(&self, token: &NodeKind) -> bool { + Self::skip_type_ext(token, self.stop_at_newline()) + } + /// Move to the next token. fn bump(&mut self) { - self.prev_end = self.tokens.index().into(); + self.children.push( + GreenData::new( + self.next.clone().unwrap(), + self.tokens.index() - self.next_start, + ) + .into(), + ); + + self.fast_forward(); + } + + pub fn fast_forward(&mut self) { + if !self.next.as_ref().map_or(false, |x| self.skip_type(x)) { + self.prev_end = self.tokens.index().into(); + } self.next_start = self.tokens.index().into(); self.next = self.tokens.next(); if self.tokens.mode() == TokenMode::Code { // Skip whitespace and comments. - while match self.next { - Some(Token::Space(n)) => n < 1 || !self.stop_at_newline(), - Some(Token::LineComment(_)) => true, - Some(Token::BlockComment(_)) => true, - _ => false, - } { - self.next_start = self.tokens.index().into(); - self.next = self.tokens.next(); + while self.next.as_ref().map_or(false, |x| self.skip_type(x)) { + self.bump(); } } @@ -349,19 +520,19 @@ impl<'s> Parser<'s> { /// Take another look at the next token to recheck whether it ends a group. fn repeek(&mut self) { - self.peeked = self.next; - let token = match self.next { + self.peeked = self.next.clone(); + let token = match self.next.as_ref() { Some(token) => token, None => return, }; if match token { - Token::RightParen => self.inside(Group::Paren), - Token::RightBracket => self.inside(Group::Bracket), - Token::RightBrace => self.inside(Group::Brace), - Token::Semicolon => self.inside(Group::Stmt), - Token::From => self.inside(Group::Imports), - Token::Space(n) => n >= 1 && self.stop_at_newline(), + NodeKind::RightParen => self.inside(Group::Paren), + NodeKind::RightBracket => self.inside(Group::Bracket), + NodeKind::RightBrace => self.inside(Group::Brace), + NodeKind::Semicolon => self.inside(Group::Stmt), + NodeKind::From => self.inside(Group::Imports), + NodeKind::Space(n) => n > &0 && self.stop_at_newline(), _ => false, } { self.peeked = None; @@ -380,4 +551,22 @@ impl<'s> Parser<'s> { fn inside(&self, kind: Group) -> bool { self.groups.iter().any(|g| g.kind == kind) } + + pub fn last_child(&self) -> Option<&Green> { + self.children.last() + } + + pub fn success(&mut self) -> bool { + let s = self.success; + self.success = true; + s + } + + pub fn unsuccessful(&mut self) { + self.success = false; + } + + pub fn child_count(&self) -> usize { + self.children.len() + } } diff --git a/src/parse/resolve.rs b/src/parse/resolve.rs index 1b323847..c59c3bb1 100644 --- a/src/parse/resolve.rs +++ b/src/parse/resolve.rs @@ -1,5 +1,5 @@ use super::{is_newline, Scanner}; -use crate::syntax::{Ident, RawNode, Span}; +use crate::syntax::RawToken; use crate::util::EcoString; /// Resolve all escape sequences in a string. @@ -48,21 +48,28 @@ pub fn resolve_hex(sequence: &str) -> Option<char> { } /// Resolve the language tag and trims the raw text. -pub fn resolve_raw(span: Span, column: usize, backticks: usize, text: &str) -> RawNode { +pub fn resolve_raw( + column: usize, + backticks: u8, + text: &str, + terminated: bool, +) -> RawToken { if backticks > 1 { let (tag, inner) = split_at_lang_tag(text); let (text, block) = trim_and_split_raw(column, inner); - RawNode { - span, - lang: Ident::new(tag, span.with_end(span.start + tag.len())), + RawToken { + lang: Some(tag.into()), text: text.into(), + backticks, + terminated, block, } } else { - RawNode { - span, + RawToken { lang: None, text: split_lines(text).join("\n").into(), + backticks, + terminated, block: false, } } @@ -140,7 +147,6 @@ fn split_lines(text: &str) -> Vec<&str> { #[cfg(test)] #[rustfmt::skip] mod tests { - use crate::syntax::Span; use super::*; #[test] @@ -175,8 +181,8 @@ mod tests { test("typst\n it!", "typst", "\n it!"); test("typst\n it!", "typst", "\n it!"); test("abc`", "abc", "`"); - test(" hi", "", " hi"); - test("`", "", "`"); + test(" hi", "", " hi"); + test("`", "", "`"); } #[test] @@ -184,13 +190,13 @@ mod tests { #[track_caller] fn test( column: usize, - backticks: usize, + backticks: u8, raw: &str, lang: Option<&str>, text: &str, block: bool, ) { - let node = resolve_raw(Span::detached(), column, backticks, raw); + let node = resolve_raw(column, backticks, raw, true); assert_eq!(node.lang.as_deref(), lang); assert_eq!(node.text, text); assert_eq!(node.block, block); @@ -204,15 +210,15 @@ mod tests { // More than one backtick with lang tag. test(0, 2, "js alert()", Some("js"), "alert()", false); test(0, 3, "py quit(\n\n)", Some("py"), "quit(\n\n)", true); - test(0, 2, "♥", None, "", false); + test(0, 2, "♥", Some("♥"), "", false); // Trimming of whitespace (tested more thoroughly in separate test). - test(0, 2, " a", None, "a", false); - test(0, 2, " a", None, " a", false); - test(0, 2, " \na", None, "a", true); + test(0, 2, " a", Some(""), "a", false); + test(0, 2, " a", Some(""), " a", false); + test(0, 2, " \na", Some(""), "a", true); // Dedenting - test(2, 3, " def foo():\n bar()", None, "def foo():\n bar()", true); + test(2, 3, " def foo():\n bar()", Some(""), "def foo():\n bar()", true); } #[test] diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs index 5f969452..19d0d77b 100644 --- a/src/parse/tokens.rs +++ b/src/parse/tokens.rs @@ -1,9 +1,13 @@ -use super::{is_newline, Scanner}; +use super::{is_newline, resolve_raw, Scanner}; use crate::geom::{AngularUnit, LengthUnit}; +use crate::parse::resolve::{resolve_hex, resolve_string}; +use crate::source::SourceFile; use crate::syntax::*; +use crate::util::EcoString; /// An iterator over the tokens of a string of source code. pub struct Tokens<'s> { + source: &'s SourceFile, s: Scanner<'s>, mode: TokenMode, } @@ -20,8 +24,12 @@ pub enum TokenMode { impl<'s> Tokens<'s> { /// Create a new token iterator with the given mode. #[inline] - pub fn new(src: &'s str, mode: TokenMode) -> Self { - Self { s: Scanner::new(src), mode } + pub fn new(source: &'s SourceFile, mode: TokenMode) -> Self { + Self { + s: Scanner::new(source.src()), + source, + mode, + } } /// Get the current token mode. @@ -59,7 +67,7 @@ impl<'s> Tokens<'s> { } impl<'s> Iterator for Tokens<'s> { - type Item = Token<'s>; + type Item = NodeKind; /// Parse the next token in the source code. #[inline] @@ -68,19 +76,21 @@ impl<'s> Iterator for Tokens<'s> { let c = self.s.eat()?; Some(match c { // Blocks and templates. - '[' => Token::LeftBracket, - ']' => Token::RightBracket, - '{' => Token::LeftBrace, - '}' => Token::RightBrace, + '[' => NodeKind::LeftBracket, + ']' => NodeKind::RightBracket, + '{' => NodeKind::LeftBrace, + '}' => NodeKind::RightBrace, // Whitespace. - ' ' if self.s.check_or(true, |c| !c.is_whitespace()) => Token::Space(0), + ' ' if self.s.check_or(true, |c| !c.is_whitespace()) => NodeKind::Space(0), c if c.is_whitespace() => self.whitespace(), // Comments with special case for URLs. '/' if self.s.eat_if('*') => self.block_comment(), '/' if !self.maybe_in_url() && self.s.eat_if('/') => self.line_comment(), - '*' if self.s.eat_if('/') => Token::Invalid(self.s.eaten_from(start)), + '*' if self.s.eat_if('/') => { + NodeKind::Error(ErrorPosition::Full, self.s.eaten_from(start).into()) + } // Other things. _ => match self.mode { @@ -93,7 +103,7 @@ impl<'s> Iterator for Tokens<'s> { impl<'s> Tokens<'s> { #[inline] - fn markup(&mut self, start: usize, c: char) -> Token<'s> { + fn markup(&mut self, start: usize, c: char) -> NodeKind { match c { // Escape sequences. '\\' => self.backslash(), @@ -102,13 +112,15 @@ impl<'s> Tokens<'s> { '#' => self.hash(), // Markup. - '~' => Token::Tilde, - '*' => Token::Star, - '_' => Token::Underscore, + '~' => NodeKind::NonBreakingSpace, + '*' => NodeKind::Strong, + '_' => NodeKind::Emph, '`' => self.raw(), '$' => self.math(), - '-' => self.hyph(start), - '=' if self.s.check_or(true, |c| c == '=' || c.is_whitespace()) => Token::Eq, + '-' => self.hyph(), + '=' if self.s.check_or(true, |c| c == '=' || c.is_whitespace()) => { + NodeKind::Eq + } c if c == '.' || c.is_ascii_digit() => self.numbering(start, c), // Plain text. @@ -116,35 +128,35 @@ impl<'s> Tokens<'s> { } } - fn code(&mut self, start: usize, c: char) -> Token<'s> { + fn code(&mut self, start: usize, c: char) -> NodeKind { match c { // Parens. - '(' => Token::LeftParen, - ')' => Token::RightParen, + '(' => NodeKind::LeftParen, + ')' => NodeKind::RightParen, // Length two. - '=' if self.s.eat_if('=') => Token::EqEq, - '!' if self.s.eat_if('=') => Token::ExclEq, - '<' if self.s.eat_if('=') => Token::LtEq, - '>' if self.s.eat_if('=') => Token::GtEq, - '+' if self.s.eat_if('=') => Token::PlusEq, - '-' if self.s.eat_if('=') => Token::HyphEq, - '*' if self.s.eat_if('=') => Token::StarEq, - '/' if self.s.eat_if('=') => Token::SlashEq, - '.' if self.s.eat_if('.') => Token::Dots, - '=' if self.s.eat_if('>') => Token::Arrow, + '=' if self.s.eat_if('=') => NodeKind::EqEq, + '!' if self.s.eat_if('=') => NodeKind::ExclEq, + '<' if self.s.eat_if('=') => NodeKind::LtEq, + '>' if self.s.eat_if('=') => NodeKind::GtEq, + '+' if self.s.eat_if('=') => NodeKind::PlusEq, + '-' if self.s.eat_if('=') => NodeKind::HyphEq, + '*' if self.s.eat_if('=') => NodeKind::StarEq, + '/' if self.s.eat_if('=') => NodeKind::SlashEq, + '.' if self.s.eat_if('.') => NodeKind::Dots, + '=' if self.s.eat_if('>') => NodeKind::Arrow, // Length one. - ',' => Token::Comma, - ';' => Token::Semicolon, - ':' => Token::Colon, - '+' => Token::Plus, - '-' => Token::Hyph, - '*' => Token::Star, - '/' => Token::Slash, - '=' => Token::Eq, - '<' => Token::Lt, - '>' => Token::Gt, + ',' => NodeKind::Comma, + ';' => NodeKind::Semicolon, + ':' => NodeKind::Colon, + '+' => NodeKind::Plus, + '-' => NodeKind::Minus, + '*' => NodeKind::Star, + '/' => NodeKind::Slash, + '=' => NodeKind::Eq, + '<' => NodeKind::Lt, + '>' => NodeKind::Gt, // Identifiers. c if is_id_start(c) => self.ident(start), @@ -159,12 +171,12 @@ impl<'s> Tokens<'s> { // Strings. '"' => self.string(), - _ => Token::Invalid(self.s.eaten_from(start)), + _ => NodeKind::Error(ErrorPosition::Full, self.s.eaten_from(start).into()), } } #[inline] - fn text(&mut self, start: usize) -> Token<'s> { + fn text(&mut self, start: usize) -> NodeKind { macro_rules! table { ($($c:literal)|*) => {{ let mut t = [false; 128]; @@ -186,10 +198,10 @@ impl<'s> Tokens<'s> { TABLE.get(c as usize).copied().unwrap_or_else(|| c.is_whitespace()) }); - Token::Text(self.s.eaten_from(start)) + NodeKind::Text(resolve_string(self.s.eaten_from(start))) } - fn whitespace(&mut self) -> Token<'s> { + fn whitespace(&mut self) -> NodeKind { self.s.uneat(); // Count the number of newlines. @@ -208,10 +220,10 @@ impl<'s> Tokens<'s> { } } - Token::Space(newlines) + NodeKind::Space(newlines) } - fn backslash(&mut self) -> Token<'s> { + fn backslash(&mut self) -> NodeKind { if let Some(c) = self.s.peek() { match c { // Backslash and comments. @@ -220,61 +232,61 @@ impl<'s> Tokens<'s> { '[' | ']' | '{' | '}' | '#' | // Markup. '*' | '_' | '=' | '~' | '`' | '$' => { - let start = self.s.index(); self.s.eat_assert(c); - Token::Text(&self.s.eaten_from(start)) + NodeKind::Text(c.into()) } 'u' if self.s.rest().starts_with("u{") => { self.s.eat_assert('u'); self.s.eat_assert('{'); - Token::UnicodeEscape(UnicodeEscapeToken { - // Allow more than `ascii_hexdigit` for better error recovery. - sequence: self.s.eat_while(|c| c.is_ascii_alphanumeric()), - terminated: self.s.eat_if('}'), + let sequence: EcoString = self.s.eat_while(|c| c.is_ascii_alphanumeric()).into(); + NodeKind::UnicodeEscape(UnicodeEscapeToken { + character: resolve_hex(&sequence), + sequence, + terminated: self.s.eat_if('}') }) } - c if c.is_whitespace() => Token::Backslash, - _ => Token::Text("\\"), + c if c.is_whitespace() => NodeKind::Linebreak, + _ => NodeKind::Text("\\".into()), } } else { - Token::Backslash + NodeKind::Linebreak } } #[inline] - fn hash(&mut self) -> Token<'s> { + fn hash(&mut self) -> NodeKind { if self.s.check_or(false, is_id_start) { let read = self.s.eat_while(is_id_continue); if let Some(keyword) = keyword(read) { keyword } else { - Token::Ident(read) + NodeKind::Ident(read.into()) } } else { - Token::Text("#") + NodeKind::Text("#".into()) } } - fn hyph(&mut self, start: usize) -> Token<'s> { + fn hyph(&mut self) -> NodeKind { if self.s.eat_if('-') { if self.s.eat_if('-') { - Token::HyphHyphHyph + NodeKind::EmDash } else { - Token::HyphHyph + NodeKind::EnDash } } else if self.s.check_or(true, char::is_whitespace) { - Token::Hyph + NodeKind::ListBullet } else { - Token::Text(self.s.eaten_from(start)) + NodeKind::Text("-".into()) } } - fn numbering(&mut self, start: usize, c: char) -> Token<'s> { + fn numbering(&mut self, start: usize, c: char) -> NodeKind { let number = if c != '.' { self.s.eat_while(|c| c.is_ascii_digit()); let read = self.s.eaten_from(start); if !self.s.eat_if('.') { - return Token::Text(read); + return NodeKind::Text(self.s.eaten_from(start).into()); } read.parse().ok() } else { @@ -282,21 +294,28 @@ impl<'s> Tokens<'s> { }; if self.s.check_or(true, char::is_whitespace) { - Token::Numbering(number) + NodeKind::EnumNumbering(number) } else { - Token::Text(self.s.eaten_from(start)) + NodeKind::Text(self.s.eaten_from(start).into()) } } - fn raw(&mut self) -> Token<'s> { + fn raw(&mut self) -> NodeKind { + let column = self.source.byte_to_column(self.s.index() - 1).unwrap(); let mut backticks = 1; - while self.s.eat_if('`') { + while self.s.eat_if('`') && backticks < u8::MAX { backticks += 1; } // Special case for empty inline block. if backticks == 2 { - return Token::Raw(RawToken { text: "", backticks: 1, terminated: true }); + return NodeKind::Raw(RawToken { + text: EcoString::new(), + lang: None, + backticks: 1, + terminated: true, + block: false, + }); } let start = self.s.index(); @@ -311,16 +330,17 @@ impl<'s> Tokens<'s> { } let terminated = found == backticks; - let end = self.s.index() - if terminated { found } else { 0 }; + let end = self.s.index() - if terminated { found as usize } else { 0 }; - Token::Raw(RawToken { - text: self.s.get(start .. end), + NodeKind::Raw(resolve_raw( + column, backticks, + self.s.get(start .. end).into(), terminated, - }) + )) } - fn math(&mut self) -> Token<'s> { + fn math(&mut self) -> NodeKind { let mut display = false; if self.s.eat_if('[') { display = true; @@ -350,25 +370,25 @@ impl<'s> Tokens<'s> { (true, true) => 2, }; - Token::Math(MathToken { - formula: self.s.get(start .. end), + NodeKind::Math(MathToken { + formula: self.s.get(start .. end).into(), display, terminated, }) } - fn ident(&mut self, start: usize) -> Token<'s> { + fn ident(&mut self, start: usize) -> NodeKind { self.s.eat_while(is_id_continue); match self.s.eaten_from(start) { - "none" => Token::None, - "auto" => Token::Auto, - "true" => Token::Bool(true), - "false" => Token::Bool(false), - id => keyword(id).unwrap_or(Token::Ident(id)), + "none" => NodeKind::None, + "auto" => NodeKind::Auto, + "true" => NodeKind::Bool(true), + "false" => NodeKind::Bool(false), + id => keyword(id).unwrap_or(NodeKind::Ident(id.into())), } } - fn number(&mut self, start: usize, c: char) -> Token<'s> { + fn number(&mut self, start: usize, c: char) -> NodeKind { // Read the first part (integer or fractional depending on `first`). self.s.eat_while(|c| c.is_ascii_digit()); @@ -380,7 +400,9 @@ impl<'s> Tokens<'s> { // Read the exponent. if self.s.eat_if('e') || self.s.eat_if('E') { - let _ = self.s.eat_if('+') || self.s.eat_if('-'); + if !self.s.eat_if('+') { + self.s.eat_if('-'); + } self.s.eat_while(|c| c.is_ascii_digit()); } @@ -396,55 +418,53 @@ impl<'s> Tokens<'s> { // Find out whether it is a simple number. if suffix.is_empty() { - if let Ok(int) = number.parse::<i64>() { - return Token::Int(int); - } else if let Ok(float) = number.parse::<f64>() { - return Token::Float(float); + if let Ok(i) = number.parse::<i64>() { + return NodeKind::Int(i); } } - // Otherwise parse into the fitting numeric type. - let build = match suffix { - "%" => Token::Percent, - "fr" => Token::Fraction, - "pt" => |x| Token::Length(x, LengthUnit::Pt), - "mm" => |x| Token::Length(x, LengthUnit::Mm), - "cm" => |x| Token::Length(x, LengthUnit::Cm), - "in" => |x| Token::Length(x, LengthUnit::In), - "rad" => |x| Token::Angle(x, AngularUnit::Rad), - "deg" => |x| Token::Angle(x, AngularUnit::Deg), - _ => return Token::Invalid(all), - }; - - if let Ok(float) = number.parse::<f64>() { - build(float) + if let Ok(f) = number.parse::<f64>() { + match suffix { + "" => NodeKind::Float(f), + "%" => NodeKind::Percentage(f), + "fr" => NodeKind::Fraction(f), + "pt" => NodeKind::Length(f, LengthUnit::Pt), + "mm" => NodeKind::Length(f, LengthUnit::Mm), + "cm" => NodeKind::Length(f, LengthUnit::Cm), + "in" => NodeKind::Length(f, LengthUnit::In), + "deg" => NodeKind::Angle(f, AngularUnit::Deg), + "rad" => NodeKind::Angle(f, AngularUnit::Rad), + _ => { + return NodeKind::Error(ErrorPosition::Full, all.into()); + } + } } else { - Token::Invalid(all) + NodeKind::Error(ErrorPosition::Full, all.into()) } } - fn string(&mut self) -> Token<'s> { + + fn string(&mut self) -> NodeKind { let mut escaped = false; - Token::Str(StrToken { - string: self.s.eat_until(|c| { + NodeKind::Str(StrToken { + string: resolve_string(self.s.eat_until(|c| { if c == '"' && !escaped { true } else { escaped = c == '\\' && !escaped; false } - }), + })), terminated: self.s.eat_if('"'), }) } - fn line_comment(&mut self) -> Token<'s> { - Token::LineComment(self.s.eat_until(is_newline)) + fn line_comment(&mut self) -> NodeKind { + self.s.eat_until(is_newline); + NodeKind::LineComment } - fn block_comment(&mut self) -> Token<'s> { - let start = self.s.index(); - + fn block_comment(&mut self) -> NodeKind { let mut state = '_'; let mut depth = 1; @@ -466,10 +486,7 @@ impl<'s> Tokens<'s> { } } - let terminated = depth == 0; - let end = self.s.index() - if terminated { 2 } else { 0 }; - - Token::BlockComment(self.s.get(start .. end)) + NodeKind::BlockComment } fn maybe_in_url(&self) -> bool { @@ -477,24 +494,24 @@ impl<'s> Tokens<'s> { } } -fn keyword(ident: &str) -> Option<Token<'static>> { +fn keyword(ident: &str) -> Option<NodeKind> { Some(match ident { - "not" => Token::Not, - "and" => Token::And, - "or" => Token::Or, - "with" => Token::With, - "let" => Token::Let, - "if" => Token::If, - "else" => Token::Else, - "for" => Token::For, - "in" => Token::In, - "while" => Token::While, - "break" => Token::Break, - "continue" => Token::Continue, - "return" => Token::Return, - "import" => Token::Import, - "include" => Token::Include, - "from" => Token::From, + "not" => NodeKind::Not, + "and" => NodeKind::And, + "or" => NodeKind::Or, + "with" => NodeKind::With, + "let" => NodeKind::Let, + "if" => NodeKind::If, + "else" => NodeKind::Else, + "for" => NodeKind::For, + "in" => NodeKind::In, + "while" => NodeKind::While, + "break" => NodeKind::Break, + "continue" => NodeKind::Continue, + "return" => NodeKind::Return, + "import" => NodeKind::Import, + "include" => NodeKind::Include, + "from" => NodeKind::From, _ => return None, }) } @@ -506,24 +523,56 @@ mod tests { use super::*; + use NodeKind::*; use Option::None; - use Token::{Ident, *}; use TokenMode::{Code, Markup}; - const fn UnicodeEscape(sequence: &str, terminated: bool) -> Token { - Token::UnicodeEscape(UnicodeEscapeToken { sequence, terminated }) + fn UnicodeEscape(sequence: &str, terminated: bool) -> NodeKind { + NodeKind::UnicodeEscape(UnicodeEscapeToken { + character: resolve_hex(sequence), + sequence: sequence.into(), + terminated, + }) } - const fn Raw(text: &str, backticks: usize, terminated: bool) -> Token { - Token::Raw(RawToken { text, backticks, terminated }) + fn Raw( + text: &str, + lang: Option<&str>, + backticks: u8, + terminated: bool, + block: bool, + ) -> NodeKind { + NodeKind::Raw(RawToken { + text: text.into(), + lang: lang.map(Into::into), + backticks, + terminated, + block, + }) + } + + fn Math(formula: &str, display: bool, terminated: bool) -> NodeKind { + NodeKind::Math(MathToken { + formula: formula.into(), + display, + terminated, + }) } - const fn Math(formula: &str, display: bool, terminated: bool) -> Token { - Token::Math(MathToken { formula, display, terminated }) + fn Str(string: &str, terminated: bool) -> NodeKind { + NodeKind::Str(StrToken { string: string.into(), terminated }) } - const fn Str(string: &str, terminated: bool) -> Token { - Token::Str(StrToken { string, terminated }) + fn Text(string: &str) -> NodeKind { + NodeKind::Text(string.into()) + } + + fn Ident(ident: &str) -> NodeKind { + NodeKind::Ident(ident.into()) + } + + fn Invalid(invalid: &str) -> NodeKind { + NodeKind::Error(ErrorPosition::Full, invalid.into()) } /// Building blocks for suffix testing. @@ -541,40 +590,6 @@ mod tests { /// - '/': symbols const BLOCKS: &str = " a1/"; - /// Suffixes described by four-tuples of: - /// - /// - block the suffix is part of - /// - mode in which the suffix is applicable - /// - the suffix string - /// - the resulting suffix token - const SUFFIXES: &[(char, Option<TokenMode>, &str, Token)] = &[ - // Whitespace suffixes. - (' ', None, " ", Space(0)), - (' ', None, "\n", Space(1)), - (' ', None, "\r", Space(1)), - (' ', None, "\r\n", Space(1)), - // Letter suffixes. - ('a', Some(Markup), "hello", Text("hello")), - ('a', Some(Markup), "💚", Text("💚")), - ('a', Some(Code), "val", Ident("val")), - ('a', Some(Code), "α", Ident("α")), - ('a', Some(Code), "_", Ident("_")), - // Number suffixes. - ('1', Some(Code), "2", Int(2)), - ('1', Some(Code), ".2", Float(0.2)), - // Symbol suffixes. - ('/', None, "[", LeftBracket), - ('/', None, "//", LineComment("")), - ('/', None, "/**/", BlockComment("")), - ('/', Some(Markup), "*", Star), - ('/', Some(Markup), "$ $", Math(" ", false, true)), - ('/', Some(Markup), r"\\", Text(r"\")), - ('/', Some(Markup), "#let", Let), - ('/', Some(Code), "(", LeftParen), - ('/', Some(Code), ":", Colon), - ('/', Some(Code), "+=", PlusEq), - ]; - macro_rules! t { (Both $($tts:tt)*) => { t!(Markup $($tts)*); @@ -584,22 +599,56 @@ mod tests { // Test without suffix. t!(@$mode: $src => $($token),*); + // Suffixes described by four-tuples of: + // + // - block the suffix is part of + // - mode in which the suffix is applicable + // - the suffix string + // - the resulting suffix NodeKind + let suffixes: &[(char, Option<TokenMode>, &str, NodeKind)] = &[ + // Whitespace suffixes. + (' ', None, " ", Space(0)), + (' ', None, "\n", Space(1)), + (' ', None, "\r", Space(1)), + (' ', None, "\r\n", Space(1)), + // Letter suffixes. + ('a', Some(Markup), "hello", Text("hello")), + ('a', Some(Markup), "💚", Text("💚")), + ('a', Some(Code), "val", Ident("val")), + ('a', Some(Code), "α", Ident("α")), + ('a', Some(Code), "_", Ident("_")), + // Number suffixes. + ('1', Some(Code), "2", Int(2)), + ('1', Some(Code), ".2", Float(0.2)), + // Symbol suffixes. + ('/', None, "[", LeftBracket), + ('/', None, "//", LineComment), + ('/', None, "/**/", BlockComment), + ('/', Some(Markup), "*", Strong), + ('/', Some(Markup), "$ $", Math(" ", false, true)), + ('/', Some(Markup), r"\\", Text("\\")), + ('/', Some(Markup), "#let", Let), + ('/', Some(Code), "(", LeftParen), + ('/', Some(Code), ":", Colon), + ('/', Some(Code), "+=", PlusEq), + ]; + // Test with each applicable suffix. - for &(block, mode, suffix, token) in SUFFIXES { + for (block, mode, suffix, token) in suffixes { let src = $src; #[allow(unused_variables)] let blocks = BLOCKS; $(let blocks = $blocks;)? assert!(!blocks.contains(|c| !BLOCKS.contains(c))); - if (mode.is_none() || mode == Some($mode)) && blocks.contains(block) { + if (mode.is_none() || mode == &Some($mode)) && blocks.contains(*block) { t!(@$mode: format!("{}{}", src, suffix) => $($token,)* token); } } }}; (@$mode:ident: $src:expr => $($token:expr),*) => {{ let src = $src; - let found = Tokens::new(&src, $mode).collect::<Vec<_>>(); - let expected = vec![$($token),*]; + let found = Tokens::new(&SourceFile::detached(src.clone()), $mode).collect::<Vec<_>>(); + let expected = vec![$($token.clone()),*]; check(&src, found, expected); }}; } @@ -671,7 +720,7 @@ mod tests { // Test text ends. t!(Markup[""]: "hello " => Text("hello"), Space(0)); - t!(Markup[""]: "hello~" => Text("hello"), Tilde); + t!(Markup[""]: "hello~" => Text("hello"), NonBreakingSpace); } #[test] @@ -713,16 +762,16 @@ mod tests { #[test] fn test_tokenize_markup_symbols() { // Test markup tokens. - t!(Markup[" a1"]: "*" => Star); - t!(Markup: "_" => Underscore); + t!(Markup[" a1"]: "*" => Strong); + t!(Markup: "_" => Emph); t!(Markup[""]: "===" => Eq, Eq, Eq); t!(Markup["a1/"]: "= " => Eq, Space(0)); - t!(Markup: "~" => Tilde); - t!(Markup[" "]: r"\" => Backslash); - t!(Markup["a "]: r"a--" => Text("a"), HyphHyph); - t!(Markup["a1/"]: "- " => Hyph, Space(0)); - t!(Markup[" "]: "." => Numbering(None)); - t!(Markup[" "]: "1." => Numbering(Some(1))); + t!(Markup: "~" => NonBreakingSpace); + t!(Markup[" "]: r"\" => Linebreak); + t!(Markup["a "]: r"a--" => Text("a"), EnDash); + t!(Markup["a1/"]: "- " => ListBullet, Space(0)); + t!(Markup[" "]: "." => EnumNumbering(None)); + t!(Markup[" "]: "1." => EnumNumbering(Some(1))); t!(Markup[" "]: "1.a" => Text("1."), Text("a")); t!(Markup[" /"]: "a1." => Text("a1.")); } @@ -734,7 +783,7 @@ mod tests { t!(Code: ";" => Semicolon); t!(Code: ":" => Colon); t!(Code: "+" => Plus); - t!(Code: "-" => Hyph); + t!(Code: "-" => Minus); t!(Code[" a1"]: "*" => Star); t!(Code[" a1"]: "/" => Slash); t!(Code: "=" => Eq); @@ -756,10 +805,10 @@ mod tests { t!(Code[" a/"]: "..." => Dots, Invalid(".")); // Test hyphen as symbol vs part of identifier. - t!(Code[" /"]: "-1" => Hyph, Int(1)); - t!(Code[" /"]: "-a" => Hyph, Ident("a")); - t!(Code[" /"]: "--1" => Hyph, Hyph, Int(1)); - t!(Code[" /"]: "--_a" => Hyph, Hyph, Ident("_a")); + t!(Code[" /"]: "-1" => Minus, Int(1)); + t!(Code[" /"]: "-a" => Minus, Ident("a")); + t!(Code[" /"]: "--1" => Minus, Minus, Int(1)); + t!(Code[" /"]: "--_a" => Minus, Minus, Ident("_a")); t!(Code[" /"]: "a-b" => Ident("a-b")); } @@ -776,13 +825,13 @@ mod tests { ("import", Import), ]; - for &(s, t) in &list { + for (s, t) in list.clone() { t!(Markup[" "]: format!("#{}", s) => t); t!(Markup[" "]: format!("#{0}#{0}", s) => t, t); - t!(Markup[" /"]: format!("# {}", s) => Token::Text("#"), Space(0), Text(s)); + t!(Markup[" /"]: format!("# {}", s) => Text("#"), Space(0), Text(s)); } - for &(s, t) in &list { + for (s, t) in list { t!(Code[" "]: s => t); t!(Markup[" /"]: s => Text(s)); } @@ -796,25 +845,23 @@ mod tests { #[test] fn test_tokenize_raw_blocks() { - let empty = Raw("", 1, true); - // Test basic raw block. - t!(Markup: "``" => empty); - t!(Markup: "`raw`" => Raw("raw", 1, true)); - t!(Markup[""]: "`]" => Raw("]", 1, false)); + t!(Markup: "``" => Raw("", None, 1, true, false)); + t!(Markup: "`raw`" => Raw("raw", None, 1, true, false)); + t!(Markup[""]: "`]" => Raw("]", None, 1, false, false)); // Test special symbols in raw block. - t!(Markup: "`[brackets]`" => Raw("[brackets]", 1, true)); - t!(Markup[""]: r"`\`` " => Raw(r"\", 1, true), Raw(" ", 1, false)); + t!(Markup: "`[brackets]`" => Raw("[brackets]", None, 1, true, false)); + t!(Markup[""]: r"`\`` " => Raw(r"\", None, 1, true, false), Raw(" ", None, 1, false, false)); // Test separated closing backticks. - t!(Markup: "```not `y`e`t```" => Raw("not `y`e`t", 3, true)); + t!(Markup: "```not `y`e`t```" => Raw("`y`e`t", Some("not"), 3, true, false)); // Test more backticks. - t!(Markup: "``nope``" => empty, Text("nope"), empty); - t!(Markup: "````🚀````" => Raw("🚀", 4, true)); - t!(Markup[""]: "`````👩🚀````noend" => Raw("👩🚀````noend", 5, false)); - t!(Markup[""]: "````raw``````" => Raw("raw", 4, true), empty); + t!(Markup: "``nope``" => Raw("", None, 1, true, false), Text("nope"), Raw("", None, 1, true, false)); + t!(Markup: "````🚀````" => Raw("", Some("🚀"), 4, true, false)); + t!(Markup[""]: "`````👩🚀````noend" => Raw("````noend", Some("👩🚀"), 5, false, false)); + t!(Markup[""]: "````raw``````" => Raw("", Some("raw"), 4, true, false), Raw("", None, 1, true, false)); } #[test] @@ -896,8 +943,8 @@ mod tests { let nums = ints.iter().map(|&(k, v)| (k, v as f64)).chain(floats); let suffixes = [ - ("%", Percent as fn(f64) -> Token<'static>), - ("fr", Fraction as fn(f64) -> Token<'static>), + ("%", Percentage as fn(f64) -> NodeKind), + ("fr", Fraction as fn(f64) -> NodeKind), ("mm", |x| Length(x, LengthUnit::Mm)), ("pt", |x| Length(x, LengthUnit::Pt)), ("cm", |x| Length(x, LengthUnit::Cm)), @@ -930,54 +977,54 @@ mod tests { t!(Code[""]: "\"hi" => Str("hi", false)); // Test escaped quote. - t!(Code: r#""a\"bc""# => Str(r#"a\"bc"#, true)); - t!(Code[""]: r#""\""# => Str(r#"\""#, false)); + t!(Code: r#""a\"bc""# => Str("a\"bc", true)); + t!(Code[""]: r#""\""# => Str("\"", false)); } #[test] fn test_tokenize_line_comments() { // Test line comment with no trailing newline. - t!(Both[""]: "//" => LineComment("")); + t!(Both[""]: "//" => LineComment); // Test line comment ends at newline. - t!(Both["a1/"]: "//bc\n" => LineComment("bc"), Space(1)); - t!(Both["a1/"]: "// bc \n" => LineComment(" bc "), Space(1)); - t!(Both["a1/"]: "//bc\r\n" => LineComment("bc"), Space(1)); + t!(Both["a1/"]: "//bc\n" => LineComment, Space(1)); + t!(Both["a1/"]: "// bc \n" => LineComment, Space(1)); + t!(Both["a1/"]: "//bc\r\n" => LineComment, Space(1)); // Test nested line comments. - t!(Both["a1/"]: "//a//b\n" => LineComment("a//b"), Space(1)); + t!(Both["a1/"]: "//a//b\n" => LineComment, Space(1)); } #[test] fn test_tokenize_block_comments() { // Test basic block comments. - t!(Both[""]: "/*" => BlockComment("")); - t!(Both: "/**/" => BlockComment("")); - t!(Both: "/*🏞*/" => BlockComment("🏞")); - t!(Both: "/*\n*/" => BlockComment("\n")); + t!(Both[""]: "/*" => BlockComment); + t!(Both: "/**/" => BlockComment); + t!(Both: "/*🏞*/" => BlockComment); + t!(Both: "/*\n*/" => BlockComment); // Test depth 1 and 2 nested block comments. - t!(Both: "/* /* */ */" => BlockComment(" /* */ ")); - t!(Both: "/*/*/**/*/*/" => BlockComment("/*/**/*/")); + t!(Both: "/* /* */ */" => BlockComment); + t!(Both: "/*/*/**/*/*/" => BlockComment); // Test two nested, one unclosed block comments. - t!(Both[""]: "/*/*/**/*/" => BlockComment("/*/**/*/")); + t!(Both[""]: "/*/*/**/*/" => BlockComment); // Test all combinations of up to two following slashes and stars. - t!(Both[""]: "/*" => BlockComment("")); - t!(Both[""]: "/*/" => BlockComment("/")); - t!(Both[""]: "/**" => BlockComment("*")); - t!(Both[""]: "/*//" => BlockComment("//")); - t!(Both[""]: "/*/*" => BlockComment("/*")); - t!(Both[""]: "/**/" => BlockComment("")); - t!(Both[""]: "/***" => BlockComment("**")); + t!(Both[""]: "/*" => BlockComment); + t!(Both[""]: "/*/" => BlockComment); + t!(Both[""]: "/**" => BlockComment); + t!(Both[""]: "/*//" => BlockComment); + t!(Both[""]: "/*/*" => BlockComment); + t!(Both[""]: "/**/" => BlockComment); + t!(Both[""]: "/***" => BlockComment); } #[test] fn test_tokenize_invalid() { // Test invalidly closed block comments. - t!(Both: "*/" => Token::Invalid("*/")); - t!(Both: "/**/*/" => BlockComment(""), Token::Invalid("*/")); + t!(Both: "*/" => Invalid("*/")); + t!(Both: "/**/*/" => BlockComment, Invalid("*/")); // Test invalid expressions. t!(Code: r"\" => Invalid(r"\")); @@ -990,6 +1037,6 @@ mod tests { // Test invalid number suffixes. t!(Code[" /"]: "1foo" => Invalid("1foo")); t!(Code: "1p%" => Invalid("1p"), Invalid("%")); - t!(Code: "1%%" => Percent(1.0), Invalid("%")); + t!(Code: "1%%" => Percentage(1.0), Invalid("%")); } } |