From 904bc392abdcd7c48fa4541594f6218168afb61f Mon Sep 17 00:00:00 2001 From: Laurenz Date: Fri, 2 Oct 2020 19:17:47 +0200 Subject: =?UTF-8?q?Remove=20spans=20from=20token=20iterator=20=F0=9F=A7=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/parse/parser.rs | 110 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 71 insertions(+), 39 deletions(-) (limited to 'src/parse/parser.rs') diff --git a/src/parse/parser.rs b/src/parse/parser.rs index 9d5eb0a9..d34730c8 100644 --- a/src/parse/parser.rs +++ b/src/parse/parser.rs @@ -2,15 +2,16 @@ use std::fmt::{self, Debug, Formatter}; use super::{Scanner, TokenMode, Tokens}; use crate::diagnostic::Diagnostic; -use crate::syntax::{Decoration, Pos, Span, Spanned, Token}; +use crate::syntax::{Decoration, Pos, Span, SpanWith, Spanned, Token}; use crate::Feedback; /// A convenient token-based parser. pub struct Parser<'s> { tokens: Tokens<'s>, - peeked: Option>>, + peeked: Option>, modes: Vec, - groups: Vec<(Pos, Group)>, + groups: Vec, + pos: Pos, f: Feedback, } @@ -22,6 +23,7 @@ impl<'s> Parser<'s> { peeked: None, modes: vec![], groups: vec![], + pos: Pos::ZERO, f: Feedback::new(), } } @@ -39,12 +41,14 @@ impl<'s> Parser<'s> { /// Eat the next token and add a diagnostic that it was not the expected /// `thing`. pub fn diag_expected(&mut self, thing: &str) { + let before = self.pos(); if let Some(found) = self.eat() { + let after = self.pos(); self.diag(error!( - found.span, + before .. after, "expected {}, found {}", thing, - found.v.name(), + found.name(), )); } else { self.diag_expected_at(thing, self.pos()); @@ -89,25 +93,24 @@ impl<'s> Parser<'s> { /// # Panics /// This panics if the next token does not start the given group. pub fn start_group(&mut self, group: Group) { - let start = self.pos(); match group { Group::Paren => self.eat_assert(Token::LeftParen), Group::Bracket => self.eat_assert(Token::LeftBracket), Group::Brace => self.eat_assert(Token::LeftBrace), Group::Subheader => {} } - self.groups.push((start, group)); + self.groups.push(group); } /// Ends the parsing of a group and returns the span of the whole group. /// /// # Panics /// This panics if no group was started. - pub fn end_group(&mut self) -> Span { + pub fn end_group(&mut self) { // Check that we are indeed at the end of the group. debug_assert_eq!(self.peek(), None, "unfinished group"); - let (start, group) = self.groups.pop().expect("unstarted group"); + let group = self.groups.pop().expect("unstarted group"); let end = match group { Group::Paren => Some(Token::RightParen), Group::Bracket => Some(Token::RightBracket), @@ -119,14 +122,12 @@ impl<'s> Parser<'s> { // This `peek()` can't be used directly because it hides the end of // group token. To circumvent this, we drop down to `self.peeked`. self.peek(); - if self.peeked.map(|s| s.v) == Some(token) { - self.peeked = None; + if self.peeked == Some(token) { + self.bump(); } else { self.diag(error!(self.pos(), "expected {}", token.name())); } } - - Span::new(start, self.pos()) } /// Skip whitespace tokens. @@ -136,34 +137,43 @@ impl<'s> Parser<'s> { }); } + /// Execute `f` and return the result alongside the span of everything `f` + /// ate. + pub fn span(&mut self, f: impl FnOnce(&mut Self) -> T) -> Spanned { + let start = self.pos; + f(self).span_with(start .. self.pos) + } + /// Consume the next token. - pub fn eat(&mut self) -> Option>> { + pub fn eat(&mut self) -> Option> { self.peek()?; - self.peeked.take() + self.bump() } /// Consume the next token if it is the given one. - pub fn eat_if(&mut self, t: Token) -> Option>> { - if self.peek()? == t { self.peeked.take() } else { None } + pub fn eat_if(&mut self, t: Token) -> bool { + if self.peek() == Some(t) { + self.bump(); + true + } else { + false + } } /// Consume the next token if the closure maps it a to `Some`-variant. - pub fn eat_map( - &mut self, - mut f: impl FnMut(Token<'s>) -> Option, - ) -> Option> { + pub fn eat_map(&mut self, f: impl FnOnce(Token<'s>) -> Option) -> Option { let token = self.peek()?; - if let Some(t) = f(token) { - self.peeked.take().map(|spanned| spanned.map(|_| t)) - } else { - None + let out = f(token); + if out.is_some() { + self.bump(); } + out } /// Consume the next token, debug-asserting that it is the given one. pub fn eat_assert(&mut self, t: Token) { let next = self.eat(); - debug_assert_eq!(next.map(|s| s.v), Some(t)); + debug_assert_eq!(next, Some(t)); } /// Consume tokens while the condition is true. @@ -182,7 +192,7 @@ impl<'s> Parser<'s> { if f(t) { break; } - self.peeked = None; + self.bump(); count += 1; } count @@ -191,11 +201,11 @@ impl<'s> Parser<'s> { /// Peek at the next token without consuming it. pub fn peek(&mut self) -> Option> { let token = match self.peeked { - Some(token) => token.v, + Some(token) => token, None => { let token = self.tokens.next()?; self.peeked = Some(token); - token.v + token } }; @@ -207,7 +217,7 @@ impl<'s> Parser<'s> { _ => return Some(token), }; - if self.groups.iter().rev().any(|&(_, g)| g == group) { + if self.groups.contains(&group) { None } else { Some(token) @@ -217,7 +227,7 @@ impl<'s> Parser<'s> { /// Checks whether the next token fulfills a condition. /// /// Returns `false` if there is no next token. - pub fn check(&mut self, f: impl FnMut(Token<'s>) -> bool) -> bool { + pub fn check(&mut self, f: impl FnOnce(Token<'s>) -> bool) -> bool { self.peek().map(f).unwrap_or(false) } @@ -229,30 +239,52 @@ impl<'s> Parser<'s> { /// The position in the string at which the last token ends and next token /// will start. pub fn pos(&self) -> Pos { - self.peeked.map(|s| s.span.start).unwrap_or_else(|| self.tokens.pos()) + self.pos } /// Jump to a position in the source string. pub fn jump(&mut self, pos: Pos) { self.tokens.jump(pos); - self.peeked = None; + self.bump(); } - /// Returns the part of the source string that is spanned by the given span. - pub fn get(&self, span: Span) -> &'s str { - self.scanner().get(span.start.to_usize() .. span.end.to_usize()) + /// Slice a part out of the source string. + pub fn get(&self, span: impl Into) -> &'s str { + self.tokens.scanner().get(span.into().to_range()) + } + + /// The full source string up to the current index. + pub fn eaten(&self) -> &'s str { + self.tokens.scanner().get(.. self.pos.to_usize()) + } + + /// The source string from `start` to the current index. + pub fn eaten_from(&self, start: Pos) -> &'s str { + self.tokens.scanner().get(start.to_usize() .. self.pos.to_usize()) + } + + /// The remaining source string after the current index. + pub fn rest(&self) -> &'s str { + self.tokens.scanner().get(self.pos.to_usize() ..) } /// The underlying scanner. pub fn scanner(&self) -> Scanner<'s> { - self.tokens.scanner() + let mut scanner = self.tokens.scanner().clone(); + scanner.jump(self.pos.to_usize()); + scanner + } + + /// Set the position to the tokenizer's position and take the peeked token. + fn bump(&mut self) -> Option> { + self.pos = self.tokens.pos(); + self.peeked.take() } } impl Debug for Parser<'_> { fn fmt(&self, f: &mut Formatter) -> fmt::Result { - let s = self.scanner(); - write!(f, "Parser({}|{})", s.eaten(), s.rest()) + write!(f, "Parser({}|{})", self.eaten(), self.rest()) } } -- cgit v1.2.3