diff options
Diffstat (limited to 'src/parse')
| -rw-r--r-- | src/parse/mod.rs | 70 | ||||
| -rw-r--r-- | src/parse/parser.rs | 343 |
2 files changed, 200 insertions, 213 deletions
diff --git a/src/parse/mod.rs b/src/parse/mod.rs index f033e01f..9eabcfc7 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -34,18 +34,16 @@ fn tree(p: &mut Parser) -> SyntaxTree { tree_while(p, true, &mut |_| true) } -/// Parse a syntax tree that stays right of the column at the start of the next -/// non-whitespace token. -fn tree_indented(p: &mut Parser) -> SyntaxTree { +/// Parse a syntax tree that stays right of the given column. +fn tree_indented(p: &mut Parser, column: usize) -> SyntaxTree { p.eat_while(|t| match t { Token::Space(n) => n == 0, Token::LineComment(_) | Token::BlockComment(_) => true, _ => false, }); - let column = p.column(p.next_start()); tree_while(p, false, &mut |p| match p.peek() { - Some(Token::Space(n)) if n >= 1 => p.column(p.next_end()) >= column, + Some(Token::Space(n)) if n >= 1 => p.column(p.next_end()) > column, _ => true, }) } @@ -68,7 +66,7 @@ where let start = p.next_start(); let tree = tree_while(p, true, f); call.args.items.push(CallArg::Pos(Expr::Template(TemplateExpr { - span: p.span(start), + span: p.span_from(start), tree: Rc::new(tree), }))); } @@ -189,7 +187,8 @@ fn raw(p: &mut Parser, token: RawToken) -> SyntaxNode { /// Parse a heading. fn heading(p: &mut Parser) -> SyntaxNode { let start = p.next_start(); - p.assert(Token::Eq); + let column = p.column(start); + p.eat_assert(Token::Eq); // Count depth. let mut level: usize = 1; @@ -198,28 +197,29 @@ fn heading(p: &mut Parser) -> SyntaxNode { } if level > 6 { - return SyntaxNode::Text(p.eaten_from(start).into()); + return SyntaxNode::Text(p.get(start .. p.prev_end()).into()); } - let body = tree_indented(p); - - SyntaxNode::Heading(HeadingNode { span: p.span(start), level, body }) + let body = tree_indented(p, column); + SyntaxNode::Heading(HeadingNode { span: p.span_from(start), level, body }) } /// Parse a single list item. fn list_item(p: &mut Parser) -> SyntaxNode { let start = p.next_start(); - p.assert(Token::Hyph); - let body = tree_indented(p); - SyntaxNode::List(ListItem { span: p.span(start), body }) + let column = p.column(start); + p.eat_assert(Token::Hyph); + let body = tree_indented(p, column); + SyntaxNode::List(ListItem { span: p.span_from(start), body }) } /// Parse a single enum item. fn enum_item(p: &mut Parser, number: Option<usize>) -> SyntaxNode { let start = p.next_start(); - p.assert(Token::Numbering(number)); - let body = tree_indented(p); - SyntaxNode::Enum(EnumItem { span: p.span(start), number, body }) + let column = p.column(start); + p.eat_assert(Token::Numbering(number)); + let body = tree_indented(p, column); + SyntaxNode::Enum(EnumItem { span: p.span_from(start), number, body }) } /// Parse an expression. @@ -240,7 +240,7 @@ fn expr_with(p: &mut Parser, atomic: bool, min_prec: usize) -> Option<Expr> { Some(op) => { let prec = op.precedence(); let expr = Box::new(expr_with(p, atomic, prec)?); - Expr::Unary(UnaryExpr { span: p.span(start), op, expr }) + Expr::Unary(UnaryExpr { span: p.span_from(start), op, expr }) } None => primary(p, atomic)?, }; @@ -529,7 +529,7 @@ fn block(p: &mut Parser, scoping: bool) -> Expr { fn call(p: &mut Parser, callee: Expr) -> Option<Expr> { let mut wide = p.eat_if(Token::Excl); if wide && p.outer_mode() == TokenMode::Code { - let span = p.span(callee.span().start); + let span = p.span_from(callee.span().start); p.error(span, "wide calls are only allowed directly in templates"); wide = false; } @@ -552,7 +552,7 @@ fn call(p: &mut Parser, callee: Expr) -> Option<Expr> { } Some(Expr::Call(CallExpr { - span: p.span(callee.span().start), + span: p.span_from(callee.span().start), callee: Box::new(callee), wide, args, @@ -571,7 +571,7 @@ fn args(p: &mut Parser) -> CallArgs { fn with_expr(p: &mut Parser, callee: Expr) -> Option<Expr> { if p.peek() == Some(Token::LeftParen) { Some(Expr::With(WithExpr { - span: p.span(callee.span().start), + span: p.span_from(callee.span().start), callee: Box::new(callee), args: args(p), })) @@ -584,7 +584,7 @@ fn with_expr(p: &mut Parser, callee: Expr) -> Option<Expr> { /// Parse a let expression. fn let_expr(p: &mut Parser) -> Option<Expr> { let start = p.next_start(); - p.assert(Token::Let); + p.eat_assert(Token::Let); let mut let_expr = None; if let Some(binding) = ident(p) { @@ -622,7 +622,7 @@ fn let_expr(p: &mut Parser) -> Option<Expr> { } let_expr = Some(Expr::Let(LetExpr { - span: p.span(start), + span: p.span_from(start), binding, init: init.map(Box::new), })); @@ -634,7 +634,7 @@ fn let_expr(p: &mut Parser) -> Option<Expr> { /// Parse an if expresion. fn if_expr(p: &mut Parser) -> Option<Expr> { let start = p.next_start(); - p.assert(Token::If); + p.eat_assert(Token::If); let mut if_expr = None; if let Some(condition) = expr(p) { @@ -650,7 +650,7 @@ fn if_expr(p: &mut Parser) -> Option<Expr> { } if_expr = Some(Expr::If(IfExpr { - span: p.span(start), + span: p.span_from(start), condition: Box::new(condition), if_body: Box::new(if_body), else_body: else_body.map(Box::new), @@ -664,13 +664,13 @@ fn if_expr(p: &mut Parser) -> Option<Expr> { /// Parse a while expresion. fn while_expr(p: &mut Parser) -> Option<Expr> { let start = p.next_start(); - p.assert(Token::While); + p.eat_assert(Token::While); let mut while_expr = None; if let Some(condition) = expr(p) { if let Some(body) = body(p) { while_expr = Some(Expr::While(WhileExpr { - span: p.span(start), + span: p.span_from(start), condition: Box::new(condition), body: Box::new(body), })); @@ -683,15 +683,15 @@ fn while_expr(p: &mut Parser) -> Option<Expr> { /// Parse a for expression. fn for_expr(p: &mut Parser) -> Option<Expr> { let start = p.next_start(); - p.assert(Token::For); + p.eat_assert(Token::For); let mut for_expr = None; if let Some(pattern) = for_pattern(p) { - if p.expect(Token::In) { + if p.eat_expect(Token::In) { if let Some(iter) = expr(p) { if let Some(body) = body(p) { for_expr = Some(Expr::For(ForExpr { - span: p.span(start), + span: p.span_from(start), pattern, iter: Box::new(iter), body: Box::new(body), @@ -718,7 +718,7 @@ fn for_pattern(p: &mut Parser) -> Option<ForPattern> { /// Parse an import expression. fn import_expr(p: &mut Parser) -> Option<Expr> { let start = p.next_start(); - p.assert(Token::Import); + p.eat_assert(Token::Import); let imports = if p.eat_if(Token::Star) { // This is the wildcard scenario. @@ -735,10 +735,10 @@ fn import_expr(p: &mut Parser) -> Option<Expr> { }; let mut import_expr = None; - if p.expect(Token::From) { + if p.eat_expect(Token::From) { if let Some(path) = expr(p) { import_expr = Some(Expr::Import(ImportExpr { - span: p.span(start), + span: p.span_from(start), imports, path: Box::new(path), })); @@ -751,11 +751,11 @@ fn import_expr(p: &mut Parser) -> Option<Expr> { /// Parse an include expression. fn include_expr(p: &mut Parser) -> Option<Expr> { let start = p.next_start(); - p.assert(Token::Include); + p.eat_assert(Token::Include); expr(p).map(|path| { Expr::Include(IncludeExpr { - span: p.span(start), + span: p.span_from(start), path: Box::new(path), }) }) diff --git a/src/parse/parser.rs b/src/parse/parser.rs index 326fc280..4f2e59c6 100644 --- a/src/parse/parser.rs +++ b/src/parse/parser.rs @@ -1,5 +1,4 @@ use std::fmt::{self, Debug, Formatter}; -use std::ops::Range; use super::{TokenMode, Tokens}; use crate::diag::Error; @@ -22,9 +21,9 @@ pub struct Parser<'s> { /// (Same as `next` except if we are at the end of group, then `None`). peeked: Option<Token<'s>>, /// The end position of the last (non-whitespace if in code mode) token. - prev_end: usize, + prev_end: Pos, /// The start position of the peeked token. - next_start: usize, + next_start: Pos, } /// A logical group of tokens, e.g. `[...]`. @@ -32,7 +31,7 @@ pub struct Parser<'s> { struct GroupEntry { /// The start position of the group. Used by `Parser::end_group` to return /// The group's full span. - pub start: usize, + pub start: Pos, /// The kind of group this is. This decides which tokens will end the group. /// For example, a [`Group::Paren`] will be ended by /// [`Token::RightParen`]. @@ -70,8 +69,8 @@ impl<'s> Parser<'s> { groups: vec![], next, peeked: next, - prev_end: 0, - next_start: 0, + prev_end: Pos::ZERO, + next_start: Pos::ZERO, } } @@ -80,37 +79,144 @@ impl<'s> Parser<'s> { self.errors } - /// Add an error with location and message. - pub fn error(&mut self, span: impl Into<Span>, message: impl Into<String>) { - self.errors.push(Error::new(self.source.id(), span, message)); + /// Whether the end of the source string or group is reached. + pub fn eof(&self) -> bool { + self.peek().is_none() } - /// Eat the next token and add an error that it is not the expected `thing`. - pub fn expected(&mut self, what: &str) { - let before = self.next_start(); - if let Some(found) = self.eat() { - let after = self.prev_end(); - self.error( - before .. after, - format!("expected {}, found {}", what, found.name()), - ); + /// Consume the next token. + pub fn eat(&mut self) -> Option<Token<'s>> { + let token = self.peek()?; + self.bump(); + Some(token) + } + + /// Eat the next token and return its source range. + pub fn eat_span(&mut self) -> Span { + let start = self.next_start(); + self.eat(); + Span::new(start, self.prev_end()) + } + + /// Consume the next token if it is the given one. + pub fn eat_if(&mut self, t: Token) -> bool { + if self.peek() == Some(t) { + self.bump(); + true } else { - self.expected_at(self.next_start(), what); + false } } - /// Add an error that `what` was expected at the given position. - pub fn expected_at(&mut self, pos: impl Into<Pos>, what: &str) { - self.error(pos.into(), format!("expected {}", what)); + /// Consume the next token if the closure maps it a to `Some`-variant. + pub fn eat_map<T, F>(&mut self, f: F) -> Option<T> + where + F: FnOnce(Token<'s>) -> Option<T>, + { + let token = self.peek()?; + let mapped = f(token); + if mapped.is_some() { + self.bump(); + } + mapped } - /// Eat the next token and add an error that it is unexpected. - pub fn unexpected(&mut self) { - let before = self.next_start(); - if let Some(found) = self.eat() { - let after = self.prev_end(); - self.error(before .. after, format!("unexpected {}", found.name())); + /// Consume the next token if it is the given one and produce an error if + /// not. + pub fn eat_expect(&mut self, t: Token) -> bool { + let eaten = self.eat_if(t); + if !eaten { + self.expected_at(self.prev_end(), t.name()); } + eaten + } + + /// Consume the next token, debug-asserting that it is one of the given ones. + pub fn eat_assert(&mut self, t: Token) { + let next = self.eat(); + debug_assert_eq!(next, Some(t)); + } + + /// Consume tokens while the condition is true. + pub fn eat_while<F>(&mut self, mut f: F) + where + F: FnMut(Token<'s>) -> bool, + { + while self.peek().map_or(false, |t| f(t)) { + self.eat(); + } + } + + /// Peek at the next token without consuming it. + pub fn peek(&self) -> Option<Token<'s>> { + self.peeked + } + + /// Peek at the next token if it follows immediately after the last one + /// without any whitespace in between. + pub fn peek_direct(&self) -> Option<Token<'s>> { + if self.next_start() == self.prev_end() { + self.peeked + } else { + None + } + } + + /// Peek at the span of the next token. + /// + /// Has length zero if `peek()` returns `None`. + pub fn peek_span(&self) -> Span { + Span::new(self.next_start(), self.next_end()) + } + + /// Peek at the source of the next token. + pub fn peek_src(&self) -> &'s str { + self.get(self.peek_span()) + } + + /// Checks whether the next token fulfills a condition. + /// + /// Returns `false` if there is no next token. + pub fn check<F>(&self, f: F) -> bool + where + F: FnOnce(Token<'s>) -> bool, + { + self.peek().map_or(false, f) + } + + /// The byte position at which the last token ended. + /// + /// Refers to the end of the last _non-whitespace_ token in code mode. + pub fn prev_end(&self) -> Pos { + self.prev_end.into() + } + + /// The byte position at which the next token starts. + pub fn next_start(&self) -> Pos { + self.next_start.into() + } + + /// The byte position at which the next token will end. + /// + /// Is the same as [`next_start()`][Self::next_start] if `peek()` returns + /// `None`. + pub fn next_end(&self) -> Pos { + self.tokens.index().into() + } + + /// The span from `start` to [`self.prev_end()`](Self::prev_end). + pub fn span_from(&self, start: Pos) -> Span { + Span::new(start, self.prev_end()) + } + + /// Determine the column index for the given byte position. + pub fn column(&self, pos: Pos) -> usize { + self.source.pos_to_column(pos).unwrap() + } + + /// Slice out part of the source string. + pub fn get(&self, span: impl Into<Span>) -> &'s str { + self.tokens.scanner().get(span.into().to_range()) } /// Continue parsing in a group. @@ -131,9 +237,9 @@ impl<'s> Parser<'s> { self.repeek(); match kind { - Group::Paren => self.assert(Token::LeftParen), - Group::Bracket => self.assert(Token::LeftBracket), - Group::Brace => self.assert(Token::LeftBrace), + Group::Paren => self.eat_assert(Token::LeftParen), + Group::Bracket => self.eat_assert(Token::LeftBracket), + Group::Brace => self.eat_assert(Token::LeftBrace), Group::Stmt => {} Group::Expr => {} Group::Imports => {} @@ -171,7 +277,8 @@ impl<'s> Parser<'s> { // Rescan the peeked token if the mode changed. if rescan { - self.jump(self.prev_end()); + self.tokens.jump(self.prev_end().to_usize()); + self.bump(); } Span::new(group.start, self.prev_end()) @@ -188,163 +295,43 @@ impl<'s> Parser<'s> { self.groups.last().map_or(TokenMode::Markup, |group| group.outer_mode) } - /// Whether the end of the source string or group is reached. - pub fn eof(&self) -> bool { - self.peek().is_none() - } - - /// Peek at the next token without consuming it. - pub fn peek(&self) -> Option<Token<'s>> { - self.peeked - } - - /// Peek at the next token if it follows immediately after the last one - /// without any whitespace in between. - pub fn peek_direct(&self) -> Option<Token<'s>> { - if self.next_start() == self.prev_end() { - self.peeked - } else { - None - } - } - - /// Peek at the span of the next token. - /// - /// Has length zero if `peek()` returns `None`. - pub fn peek_span(&self) -> Span { - self.peek_range().into() - } - - /// Peek at the source of the next token. - pub fn peek_src(&self) -> &'s str { - self.tokens.scanner().get(self.peek_range()) - } - - /// Peek at the source range (start and end index) of the next token. - pub fn peek_range(&self) -> Range<usize> { - self.next_start() .. self.next_end() - } - - /// Checks whether the next token fulfills a condition. - /// - /// Returns `false` if there is no next token. - pub fn check<F>(&self, f: F) -> bool - where - F: FnOnce(Token<'s>) -> bool, - { - self.peek().map_or(false, f) - } - - /// Consume the next token. - pub fn eat(&mut self) -> Option<Token<'s>> { - let token = self.peek()?; - self.bump(); - Some(token) + /// Add an error with location and message. + pub fn error(&mut self, span: impl Into<Span>, message: impl Into<String>) { + self.errors.push(Error::new(self.source.id(), span, message)); } - /// Consume the next token if it is the given one. - pub fn eat_if(&mut self, t: Token) -> bool { - if self.peek() == Some(t) { - self.bump(); - true + /// Eat the next token and add an error that it is not the expected `thing`. + pub fn expected(&mut self, what: &str) { + let before = self.next_start(); + if let Some(found) = self.eat() { + let after = self.prev_end(); + self.error( + before .. after, + format!("expected {}, found {}", what, found.name()), + ); } else { - false - } - } - - /// Consume tokens while the condition is true. - pub fn eat_while<F>(&mut self, mut f: F) - where - F: FnMut(Token<'s>) -> bool, - { - while self.peek().map_or(false, |t| f(t)) { - self.eat(); - } - } - - /// Consume the next token if the closure maps it a to `Some`-variant. - pub fn eat_map<T, F>(&mut self, f: F) -> Option<T> - where - F: FnOnce(Token<'s>) -> Option<T>, - { - let token = self.peek()?; - let mapped = f(token); - if mapped.is_some() { - self.bump(); + self.expected_at(self.next_start(), what); } - mapped } - /// Eat the next token and return its source range. - pub fn eat_span(&mut self) -> Span { - let start = self.next_start(); - self.eat(); - Span::new(start, self.prev_end()) + /// Add an error that `what` was expected at the given position. + pub fn expected_at(&mut self, pos: Pos, what: &str) { + self.error(pos, format!("expected {}", what)); } - /// Consume the next token if it is the given one and produce an error if - /// not. - pub fn expect(&mut self, t: Token) -> bool { - let eaten = self.eat_if(t); - if !eaten { - self.expected_at(self.prev_end(), t.name()); + /// Eat the next token and add an error that it is unexpected. + pub fn unexpected(&mut self) { + let before = self.next_start(); + if let Some(found) = self.eat() { + let after = self.prev_end(); + self.error(before .. after, format!("unexpected {}", found.name())); } - eaten - } - - /// Consume the next token, debug-asserting that it is one of the given ones. - pub fn assert(&mut self, t: Token) { - let next = self.eat(); - debug_assert_eq!(next, Some(t)); - } - - /// The index at which the last token ended. - /// - /// Refers to the end of the last _non-whitespace_ token in code mode. - pub fn prev_end(&self) -> usize { - self.prev_end - } - - /// The index at which the next token starts. - pub fn next_start(&self) -> usize { - self.next_start - } - - /// The index at which the next token will end. - /// - /// Is the same as [`next_start()`][Self::next_start] if `peek()` returns - /// `None`. - pub fn next_end(&self) -> usize { - self.tokens.index() - } - - /// Determine the column for the given index in the source. - pub fn column(&self, index: usize) -> usize { - self.source.pos_to_column(index.into()).unwrap() - } - - /// The span from `start` to [`self.prev_end()`](Self::prev_end). - pub fn span(&self, start: impl Into<Pos>) -> Span { - Span::new(start, self.prev_end()) - } - - /// Return the source string from `start` to the end of the previous token. - pub fn eaten_from(&self, start: usize) -> &'s str { - self.tokens.scanner().get(start .. self.prev_end()) - } - - /// Jump to an index in the string. - /// - /// You need to know the correct column. - fn jump(&mut self, index: usize) { - self.tokens.jump(index); - self.bump(); } /// Move to the next token. fn bump(&mut self) { - self.prev_end = self.tokens.index(); - self.next_start = self.tokens.index(); + self.prev_end = self.tokens.index().into(); + self.next_start = self.tokens.index().into(); self.next = self.tokens.next(); if self.tokens.mode() == TokenMode::Code { @@ -355,7 +342,7 @@ impl<'s> Parser<'s> { Some(Token::BlockComment(_)) => true, _ => false, } { - self.next_start = self.tokens.index(); + self.next_start = self.tokens.index().into(); self.next = self.tokens.next(); } } @@ -399,7 +386,7 @@ impl<'s> Parser<'s> { impl Debug for Parser<'_> { fn fmt(&self, f: &mut Formatter) -> fmt::Result { let mut s = self.tokens.scanner(); - s.jump(self.next_start()); + s.jump(self.next_start().to_usize()); write!(f, "Parser({}|{})", s.eaten(), s.rest()) } } |