From aac3afcba8ee9b3692f784c78626aa0596aaf612 Mon Sep 17 00:00:00 2001 From: Martin Haug Date: Mon, 21 Feb 2022 13:48:21 +0100 Subject: Remove `Parbreak` as a `NodeKind` --- src/parse/parser.rs | 7 ------- 1 file changed, 7 deletions(-) (limited to 'src/parse/parser.rs') diff --git a/src/parse/parser.rs b/src/parse/parser.rs index e495dbd0..545f6fd4 100644 --- a/src/parse/parser.rs +++ b/src/parse/parser.rs @@ -176,13 +176,6 @@ impl<'s> Parser<'s> { } } - /// Eat the current token, but change its type. - pub fn convert(&mut self, kind: NodeKind) { - let marker = self.marker(); - self.eat(); - marker.convert(self, kind); - } - /// Whether the current token is of the given type. pub fn at(&self, kind: &NodeKind) -> bool { self.peek() == Some(kind) -- cgit v1.2.3 From 20ac96f27a2e06b985abc1c95049c32c2b88ef5d Mon Sep 17 00:00:00 2001 From: Martin Haug Date: Mon, 21 Feb 2022 22:49:50 +0100 Subject: New incremental parsing paradigm Also move column offset into scanner. This fixes #62 --- src/parse/parser.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'src/parse/parser.rs') diff --git a/src/parse/parser.rs b/src/parse/parser.rs index 545f6fd4..d9cc0e31 100644 --- a/src/parse/parser.rs +++ b/src/parse/parser.rs @@ -7,8 +7,6 @@ use crate::syntax::{ErrorPos, Green, GreenData, GreenNode, NodeKind}; /// A convenient token-based parser. pub struct Parser<'s> { - /// Offsets the indentation on the first line of the source. - column_offset: usize, /// An iterator over the source tokens. tokens: Tokens<'s>, /// Whether we are at the end of the file or of a group. @@ -22,7 +20,7 @@ pub struct Parser<'s> { /// The stack of open groups. groups: Vec, /// The children of the currently built node. - children: Vec, + pub children: Vec, /// Whether the last group was not correctly terminated. unterminated_group: bool, /// Whether a group terminator was found, that did not close a group. @@ -32,10 +30,13 @@ pub struct Parser<'s> { impl<'s> Parser<'s> { /// Create a new parser for the source string. pub fn new(src: &'s str, mode: TokenMode) -> Self { - let mut tokens = Tokens::new(src, mode); + Self::with_offset(src, mode, 0) + } + + fn with_offset(src: &'s str, mode: TokenMode, offset: usize) -> Self { + let mut tokens = Tokens::new(src, mode, offset); let current = tokens.next(); Self { - column_offset: 0, tokens, eof: current.is_none(), current, @@ -52,9 +53,7 @@ impl<'s> Parser<'s> { /// that does not need to be parsed but taken into account for column /// calculation. pub fn with_prefix(prefix: &str, src: &'s str, mode: TokenMode) -> Self { - let mut p = Self::new(src, mode); - p.column_offset = Scanner::new(prefix).column(prefix.len()); - p + Self::with_offset(src, mode, Scanner::new(prefix).column(prefix.len())) } /// End the parsing process and return the last child. @@ -226,7 +225,7 @@ impl<'s> Parser<'s> { /// Determine the column index for the given byte index. pub fn column(&self, index: usize) -> usize { - self.tokens.scanner().column_offset(index, self.column_offset) + self.tokens.scanner().column(index) } /// Continue parsing in a group. -- cgit v1.2.3 From 4c8634c600ad0bba03ccdf884b32f234ecbff30c Mon Sep 17 00:00:00 2001 From: Martin Haug Date: Wed, 23 Feb 2022 13:57:15 +0100 Subject: Early stop for falling indents. Fix code edits and at_start handling. Also fix dedenting for multi-byte chars in raw blocks. --- src/parse/parser.rs | 7 ------- 1 file changed, 7 deletions(-) (limited to 'src/parse/parser.rs') diff --git a/src/parse/parser.rs b/src/parse/parser.rs index d9cc0e31..8588e586 100644 --- a/src/parse/parser.rs +++ b/src/parse/parser.rs @@ -64,13 +64,6 @@ impl<'s> Parser<'s> { /// End the parsing process and return multiple children and whether the /// last token was terminated. pub fn consume(self) -> Option<(Vec, bool)> { - (self.eof() && self.terminated()) - .then(|| (self.children, self.tokens.terminated())) - } - - /// End the parsing process and return multiple children and whether the - /// last token was terminated, even if there remains stuff in the string. - pub fn consume_open_ended(self) -> Option<(Vec, bool)> { self.terminated().then(|| (self.children, self.tokens.terminated())) } -- cgit v1.2.3 From 9fda623b02b2a0a9e9cdf806d9945d0759c8bf01 Mon Sep 17 00:00:00 2001 From: Martin Haug Date: Wed, 23 Feb 2022 20:06:48 +0100 Subject: Code Review: That's just like your struct, man. --- src/parse/parser.rs | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) (limited to 'src/parse/parser.rs') diff --git a/src/parse/parser.rs b/src/parse/parser.rs index 8588e586..123871a5 100644 --- a/src/parse/parser.rs +++ b/src/parse/parser.rs @@ -2,7 +2,7 @@ use core::slice::SliceIndex; use std::fmt::{self, Display, Formatter}; use std::mem; -use super::{Scanner, TokenMode, Tokens}; +use super::{TokenMode, Tokens}; use crate::syntax::{ErrorPos, Green, GreenData, GreenNode, NodeKind}; /// A convenient token-based parser. @@ -30,11 +30,14 @@ pub struct Parser<'s> { impl<'s> Parser<'s> { /// Create a new parser for the source string. pub fn new(src: &'s str, mode: TokenMode) -> Self { - Self::with_offset(src, mode, 0) + Self::with_prefix("", src, mode) } - fn with_offset(src: &'s str, mode: TokenMode, offset: usize) -> Self { - let mut tokens = Tokens::new(src, mode, offset); + /// Create a new parser for the source string that is prefixed by some text + /// that does not need to be parsed but taken into account for column + /// calculation. + pub fn with_prefix(prefix: &str, src: &'s str, mode: TokenMode) -> Self { + let mut tokens = Tokens::with_prefix(prefix, src, mode); let current = tokens.next(); Self { tokens, @@ -49,13 +52,6 @@ impl<'s> Parser<'s> { } } - /// Create a new parser for the source string that is prefixed by some text - /// that does not need to be parsed but taken into account for column - /// calculation. - pub fn with_prefix(prefix: &str, src: &'s str, mode: TokenMode) -> Self { - Self::with_offset(src, mode, Scanner::new(prefix).column(prefix.len())) - } - /// End the parsing process and return the last child. pub fn finish(self) -> Vec { self.children @@ -218,7 +214,7 @@ impl<'s> Parser<'s> { /// Determine the column index for the given byte index. pub fn column(&self, index: usize) -> usize { - self.tokens.scanner().column(index) + self.tokens.column(index) } /// Continue parsing in a group. -- cgit v1.2.3