diff options
Diffstat (limited to 'src/parse/parser.rs')
| -rw-r--r-- | src/parse/parser.rs | 124 |
1 files changed, 64 insertions, 60 deletions
diff --git a/src/parse/parser.rs b/src/parse/parser.rs index 6269ad73..27346587 100644 --- a/src/parse/parser.rs +++ b/src/parse/parser.rs @@ -1,6 +1,7 @@ use std::fmt::{self, Debug, Formatter}; +use std::ops::Range; -use super::{Scanner, TokenMode, Tokens}; +use super::{search_column, TokenMode, Tokens}; use crate::diag::{Diag, DiagSet}; use crate::syntax::{Pos, Span, Token}; @@ -17,10 +18,10 @@ pub struct Parser<'s> { /// The peeked token. /// (Same as `next` except if we are at the end of group, then `None`). peeked: Option<Token<'s>>, - /// The start position of the peeked token. - next_start: Pos, /// The end position of the last (non-whitespace if in code mode) token. - last_end: Pos, + prev_end: usize, + /// The start position of the peeked token. + next_start: usize, } /// A logical group of tokens, e.g. `[...]`. @@ -28,7 +29,7 @@ pub struct Parser<'s> { struct GroupEntry { /// The start position of the group. Used by `Parser::end_group` to return /// The group's full span. - pub start: Pos, + pub start: usize, /// The kind of group this is. This decides which tokens will end the group. /// For example, a [`Group::Paren`] will be ended by /// [`Token::RightParen`]. @@ -59,12 +60,12 @@ impl<'s> Parser<'s> { let next = tokens.next(); Self { diags: DiagSet::new(), - next, tokens, - last_end: Pos::ZERO, - peeked: next, - next_start: Pos::ZERO, groups: vec![], + next, + peeked: next, + prev_end: 0, + next_start: 0, } } @@ -76,9 +77,9 @@ impl<'s> Parser<'s> { /// Eat the next token and add a diagnostic that it is not the expected /// `thing`. pub fn expected(&mut self, what: &str) { - let before = self.next_start; + let before = self.next_start(); if let Some(found) = self.eat() { - let after = self.last_end; + let after = self.prev_end(); self.diag(error!( before .. after, "expected {}, found {}", @@ -86,20 +87,20 @@ impl<'s> Parser<'s> { found.name(), )); } else { - self.expected_at(what, self.next_start); + self.expected_at(what, self.next_start()); } } /// Add a diagnostic that `what` was expected at the given position. - pub fn expected_at(&mut self, what: &str, pos: Pos) { - self.diag(error!(pos, "expected {}", what)); + pub fn expected_at(&mut self, what: &str, pos: impl Into<Pos>) { + self.diag(error!(pos.into(), "expected {}", what)); } /// Eat the next token and add a diagnostic that it is unexpected. pub fn unexpected(&mut self) { - let before = self.next_start; + let before = self.next_start(); if let Some(found) = self.eat() { - let after = self.last_end; + let after = self.prev_end(); self.diag(error!(before .. after, "unexpected {}", found.name())); } } @@ -110,11 +111,10 @@ impl<'s> Parser<'s> { /// `eat()` and `peek()` return `None`. Parsing can only continue with /// a matching call to `end_group`. /// - /// # Panics /// This panics if the next token does not start the given group. pub fn start_group(&mut self, kind: Group, mode: TokenMode) { self.groups.push(GroupEntry { - start: self.next_start, + start: self.next_start(), kind, outer_mode: self.tokens.mode(), }); @@ -133,7 +133,6 @@ impl<'s> Parser<'s> { /// End the parsing of a group. /// - /// # Panics /// This panics if no group was started. pub fn end_group(&mut self) -> Span { let prev_mode = self.tokens.mode(); @@ -156,17 +155,16 @@ impl<'s> Parser<'s> { self.bump(); rescan = false; } else if required { - self.diag(error!(self.next_start, "expected {}", end.name())); + self.diag(error!(self.next_start(), "expected {}", end.name())); } } // Rescan the peeked token if the mode changed. if rescan { - self.tokens.jump(self.last_end); - self.bump(); + self.jump(self.prev_end()); } - Span::new(group.start, self.last_end) + Span::new(group.start, self.prev_end()) } /// The tokenization mode outside of the current group. @@ -193,7 +191,7 @@ impl<'s> Parser<'s> { /// Peek at the next token if it follows immediately after the last one /// without any whitespace in between. pub fn peek_direct(&self) -> Option<Token<'s>> { - if self.next_start == self.last_end { + if self.next_start() == self.prev_end() { self.peeked } else { None @@ -204,15 +202,17 @@ impl<'s> Parser<'s> { /// /// Has length zero if `peek()` returns `None`. pub fn peek_span(&self) -> Span { - Span::new( - self.next_start, - if self.eof() { self.next_start } else { self.tokens.pos() }, - ) + self.peek_range().into() } /// Peek at the source of the next token. pub fn peek_src(&self) -> &'s str { - self.get(self.peek_span()) + self.tokens.scanner().get(self.peek_range()) + } + + /// Peek at the source range (start and end index) of the next token. + pub fn peek_range(&self) -> Range<usize> { + self.next_start() .. self.next_end() } /// Checks whether the next token fulfills a condition. @@ -255,11 +255,11 @@ impl<'s> Parser<'s> { mapped } - /// Eat the next token and return its span. + /// Eat the next token and return its source range. pub fn eat_span(&mut self) -> Span { - let start = self.next_start; + let start = self.next_start(); self.eat(); - Span::new(start, self.last_end) + Span::new(start, self.prev_end()) } /// Consume the next token if it is the given one and produce a diagnostic @@ -267,7 +267,7 @@ impl<'s> Parser<'s> { pub fn expect(&mut self, t: Token) -> bool { let eaten = self.eat_if(t); if !eaten { - self.expected_at(t.name(), self.last_end); + self.expected_at(t.name(), self.prev_end()); } eaten } @@ -290,45 +290,48 @@ impl<'s> Parser<'s> { } } - /// The position at which the next token starts. - pub fn start(&self) -> Pos { - self.next_start - } - - /// The position at which the last token ended. + /// The index at which the last token ended. /// /// Refers to the end of the last _non-whitespace_ token in code mode. - pub fn end(&self) -> Pos { - self.last_end + pub fn prev_end(&self) -> usize { + self.prev_end } - /// The span from `start` to the end of the last token. - pub fn span(&self, start: Pos) -> Span { - Span::new(start, self.last_end) + /// The index at which the next token starts. + pub fn next_start(&self) -> usize { + self.next_start } - /// Jump to a position in the source string. - pub fn jump(&mut self, pos: Pos) { - self.tokens.jump(pos); - self.bump(); + /// The index at which the next token will end. + /// + /// Is the same as [`next_start()`][Self::next_start] if `peek()` returns + /// `None`. + pub fn next_end(&self) -> usize { + self.tokens.index() } - /// Slice a part out of the source string. - pub fn get(&self, span: impl Into<Span>) -> &'s str { - self.tokens.scanner().get(span.into().to_range()) + /// Determine the column for the given index in the source. + pub fn column(&self, index: usize) -> usize { + search_column(self.tokens.scanner().get(.. index)) } - /// The underlying scanner. - pub fn scanner(&self) -> Scanner<'s> { - let mut scanner = self.tokens.scanner().clone(); - scanner.jump(self.next_start.to_usize()); - scanner + /// The span from `start` to [`self.prev_end()`](Self::prev_end). + pub fn span(&self, start: impl Into<Pos>) -> Span { + Span::new(start, self.prev_end()) + } + + /// Jump to an index in the string. + /// + /// You need to know the correct column. + fn jump(&mut self, index: usize) { + self.tokens.jump(index); + self.bump(); } /// Move to the next token. fn bump(&mut self) { - self.last_end = self.tokens.pos(); - self.next_start = self.tokens.pos(); + self.prev_end = self.tokens.index(); + self.next_start = self.tokens.index(); self.next = self.tokens.next(); if self.tokens.mode() == TokenMode::Code { @@ -339,7 +342,7 @@ impl<'s> Parser<'s> { Some(Token::BlockComment(_)) => true, _ => false, } { - self.next_start = self.tokens.pos(); + self.next_start = self.tokens.index(); self.next = self.tokens.next(); } } @@ -381,7 +384,8 @@ impl<'s> Parser<'s> { impl Debug for Parser<'_> { fn fmt(&self, f: &mut Formatter) -> fmt::Result { - let s = self.scanner(); + let mut s = self.tokens.scanner(); + s.jump(self.next_start()); write!(f, "Parser({}|{})", s.eaten(), s.rest()) } } |