Decoupled function parser 🔗 [WIP]

1 files changed, 168 insertions, 206 deletions

diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs
index 24bef7ce..ed343050 100644
--- a/src/syntax/parsing.rs
+++ b/src/syntax/parsing.rs
@@ -3,10 +3,6 @@ use super::*;
 use Token::*;
 
 
-pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed<SyntaxModel> {
-    Parser::new(start, src, ctx).parse()
-}
-
 /// The context for parsing.
 #[derive(Debug, Copy, Clone)]
 pub struct ParseContext<'a> {
@@ -14,185 +10,169 @@ pub struct ParseContext<'a> {
     pub scope: &'a Scope,
 }
 
-struct Parser<'s> {
-    src: &'s str,
-    ctx: ParseContext<'s>,
-    tokens: Tokens<'s>,
-    peeked: Option<Option<Spanned<Token<'s>>>>,
-    position: Position,
-    last_position: Position,
-    errors: SpanVec<Error>,
-    decorations: SpanVec<Decoration>,
+pub struct Parsed<T> {
+    pub output: T,
+    pub errors: SpanVec<Error>,
+    pub decorations: SpanVec<Decoration>,
 }
 
-impl<'s> Parser<'s> {
-    fn new(start: Position, src: &'s str, ctx: ParseContext<'s>) -> Parser<'s> {
-        Parser {
-            src,
-            ctx,
-            tokens: tokenize(start, src),
-            peeked: None,
-            position: Position::ZERO,
-            last_position: Position::ZERO,
-            errors: vec![],
-            decorations: vec![],
-        }
-    }
-
-    /// The main parsing entrypoint.
-    fn parse(mut self) -> Parsed<SyntaxModel> {
-        let mut model = SyntaxModel::new();
-
-        while let Some(token) = self.eat() {
-            let mut span = token.span;
-            let node = match token.v {
-                LineComment(_) | BlockComment(_) => None,
-                Whitespace(newlines) => Some(if newlines >= 2 {
-                    Node::Newline
-                } else {
-                    Node::Space
-                }),
-
-                LeftBracket => self.parse_func().map(|spanned| {
-                    span = spanned.span;
-                    spanned.v
-                }),
-
-                Star       => Some(Node::ToggleBolder),
-                Underscore => Some(Node::ToggleItalic),
-                Backtick   => Some(Node::ToggleMonospace),
-                Text(text) => Some(Node::Text(text.to_owned())),
-
-                _ => {
-                    self.unexpected(token);
-                    None
-                }
-            };
-
-            if let Some(v) = node {
-                model.add(Spanned { v, span });
-            }
-        }
-
+impl<T> Parsed<T> {
+    pub fn map<F, U>(self, f: F) -> Parsed<U> where F: FnOnce(T) -> U {
         Parsed {
-            output: model,
+            output: f(self.output),
             errors: self.errors,
             decorations: self.decorations,
         }
     }
+}
 
-    /// Parses a function including header and body with the cursor starting
-    /// right behind the first opening bracket.
-    fn parse_func(&mut self) -> Option<Spanned<Node>> {
-        let start = self.last_pos();
-
-        let header = self.parse_func_header();
-        self.eat_until(|t| t == RightBracket, false);
+pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed<SyntaxModel> {
+    let mut model = SyntaxModel::new();
+    let mut errors = Vec::new();
+    let mut decorations = Vec::new();
 
-        if self.eat().map(Spanned::value) != Some(RightBracket) {
-            self.expected_at("closing bracket", self.pos());
-        }
+    let mut tokens = Tokens::new(start, src, TokenizationMode::Body);
 
-        let body = if self.peekv() == Some(LeftBracket) {
-            self.eat();
+    while let Some(token) = tokens.next() {
+        let span = token.span;
 
-            let start_index = self.tokens.index();
-            let start_position = self.tokens.pos();
-
-            let found = self.tokens.move_to_closing_bracket();
+        let node = match token.v {
+            Space(newlines) => if newlines >= 2 {
+                Node::Newline
+            } else {
+                Node::Space
+            },
 
-            let end_index = self.tokens.index();
-            let end_position = self.tokens.pos();
+            Function { header, body, terminated } => {
+                let parsed: Parsed<Node> = FuncParser::new(header, body, ctx).parse();
 
-            let body = &self.src[start_index .. end_index];
+                errors.extend(offset_spans(parsed.errors, span.start));
+                decorations.extend(offset_spans(parsed.decorations, span.start));
 
-            self.position = end_position;
+                if !terminated {
+                    errors.push(err!(Span::at(span.end); "expected closing bracket"));
+                }
 
-            if found {
-                let next = self.eat().map(Spanned::value);
-                debug_assert_eq!(next, Some(RightBracket));
-            } else {
-                self.expected_at("closing bracket", self.pos());
+                parsed.output
             }
 
-            Some(Spanned::new(body, Span::new(start_position, end_position)))
-        } else {
-            None
-        };
+            Star       => Node::ToggleBolder,
+            Underscore => Node::ToggleItalic,
+            Backtick   => Node::ToggleMonospace,
+            Text(text) => Node::Text(text.to_owned()),
+
+            LineComment(_) | BlockComment(_) => continue,
 
-        let header = header?;
-        let (parser, decoration) = match self.ctx.scope.get_parser(header.name.v.as_str()) {
-            Ok(parser) => (parser, Decoration::ValidFuncName),
-            Err(parser) => {
-                let error = Error::new(format!("unknown function: `{}`", header.name.v));
-                self.errors.push(Spanned::new(error, header.name.span));
-                (parser, Decoration::InvalidFuncName)
+            other => {
+                errors.push(err!(span; "unexpected {}", name(other)));
+                continue;
             }
         };
 
-        self.decorations.push(Spanned::new(decoration, header.name.span));
+        model.add(Spanned { v: node, span: token.span });
+    }
 
-        let parsed = parser(header, body, self.ctx);
-        self.errors.extend(offset_spans(parsed.errors, start));
-        self.decorations.extend(offset_spans(parsed.decorations, start));
+    Parsed { output: model, errors, decorations }
+}
 
-        let node = Node::Model(parsed.output);
+struct FuncParser<'s> {
+    ctx: ParseContext<'s>,
+    errors: SpanVec<Error>,
+    decorations: SpanVec<Decoration>,
+    tokens: Tokens<'s>,
+    peeked: Option<Option<Spanned<Token<'s>>>>,
+    body: Option<(Position, &'s str)>,
+}
 
-        let end = self.pos();
-        let span = Span { start, end };
+impl<'s> FuncParser<'s> {
+    fn new(
+        header: &'s str,
+        body: Option<(Position, &'s str)>,
+        ctx: ParseContext<'s>
+    ) -> FuncParser<'s> {
+        FuncParser {
+            ctx,
+            errors: vec![],
+            decorations: vec![],
+            tokens: Tokens::new(Position::new(0, 1), header, TokenizationMode::Header),
+            peeked: None,
+            body,
+        }
+    }
+
+    fn parse(mut self) -> Parsed<Node> {
+        let parsed = if let Some(header) = self.parse_func_header() {
+            let name = header.name.v.as_str();
+            let (parser, deco) = match self.ctx.scope.get_parser(name) {
+                Ok(parser) => (parser, Decoration::ValidFuncName),
+                Err(parser) => {
+                    self.errors.push(err!(header.name.span; "unknown function"));
+                    (parser, Decoration::InvalidFuncName)
+                }
+            };
 
-        Some(Spanned { v: node, span })
+            self.decorations.push(Spanned::new(deco, header.name.span));
+
+            parser(header, self.body, self.ctx)
+        } else {
+            let default = FuncHeader {
+                name: Spanned::new(Ident("".to_string()), Span::ZERO),
+                args: FuncArgs::new(),
+            };
+
+            // Use the fallback function such that the body is still rendered
+            // even if the header is completely unparsable.
+            self.ctx.scope.get_fallback_parser()(default, self.body, self.ctx)
+        };
+
+        self.errors.extend(parsed.errors);
+        self.decorations.extend(parsed.decorations);
+
+        Parsed {
+            output: Node::Model(parsed.output),
+            errors: self.errors,
+            decorations: self.decorations,
+        }
     }
 
-    /// Parses a function header including the closing bracket.
     fn parse_func_header(&mut self) -> Option<FuncHeader> {
+        let start = self.pos();
         self.skip_whitespace();
-        let name = self.parse_func_name()?;
 
-        self.skip_whitespace();
-        let args = match self.peek() {
-            Some(Spanned { v: Colon, .. }) => {
-                self.eat();
-                self.parse_func_args()
+        let name = match self.eat() {
+            Some(Spanned { v: ExprIdent(ident), span }) => {
+                Spanned { v: Ident(ident.to_string()), span }
             }
-            Some(Spanned { v: RightBracket, .. }) => FuncArgs::new(),
             other => {
-                self.expected_at("colon or closing bracket", name.span.end);
-                FuncArgs::new()
+                self.expected_found_or_at("identifier", other, start);
+                return None;
             }
         };
 
-        Some(FuncHeader { name, args })
-    }
-
-    /// Parses the function name if is the next token. Otherwise, it adds an
-    /// error and returns `None`.
-    fn parse_func_name(&mut self) -> Option<Spanned<Ident>> {
-        match self.peek() {
-            Some(Spanned { v: ExprIdent(ident), span }) => {
-                self.eat();
-                return Some(Spanned { v: Ident(ident.to_string()), span });
+        self.skip_whitespace();
+        let args = match self.eat().map(Spanned::value) {
+            Some(Colon) => self.parse_func_args(),
+            Some(_) => {
+                self.expected_at("colon", name.span.end);
+                FuncArgs::new()
             }
-            other => self.expected_found_or_at("identifier", other, self.pos()),
-        }
+            None => FuncArgs::new(),
+        };
 
-        None
+        Some(FuncHeader { name, args })
     }
 
-    /// Parses the function arguments and stops right before the final closing
-    /// bracket.
     fn parse_func_args(&mut self) -> FuncArgs {
         let mut args = FuncArgs::new();
 
-        loop {
-            self.skip_whitespace();
-            match self.peekv() {
-                Some(RightBracket) | None => break,
-                _ => match self.parse_arg() {
-                    Some(arg) => args.add(arg),
-                    None => {}
-                }
+        self.skip_whitespace();
+        while self.peek().is_some() {
+            match self.parse_arg() {
+                Some(arg) => args.add(arg),
+                None => {}
             }
+
+            self.skip_whitespace();
         }
 
         args
@@ -221,7 +201,7 @@ impl<'s> Parser<'s> {
                     })
                 })
             } else {
-                Some(Arg::Pos(Spanned::new(Expression::Ident(ident), span)))
+                Some(Arg::Pos(Spanned::new(Expr::Ident(ident), span)))
             }
         } else {
             self.parse_expr().map(|expr| Arg::Pos(expr))
@@ -230,7 +210,6 @@ impl<'s> Parser<'s> {
         if let Some(arg) = &arg {
             self.skip_whitespace();
             match self.peekv() {
-                Some(RightBracket) => {}
                 Some(Comma) => { self.eat(); }
                 Some(_) => self.expected_at("comma", arg.span().end),
                 _ => {}
@@ -244,19 +223,27 @@ impl<'s> Parser<'s> {
     }
 
     /// Parse a atomic or compound (tuple / object) expression.
-    fn parse_expr(&mut self) -> Option<Spanned<Expression>> {
+    fn parse_expr(&mut self) -> Option<Spanned<Expr>> {
         let first = self.peek()?;
-        let mut expr = |v| {
-            self.eat();
-            Spanned { v, span: first.span }
-        };
+        let spanned = |v| Spanned { v, span: first.span };
 
         Some(match first.v {
-            ExprIdent(i) => expr(Expression::Ident(Ident(i.to_string()))),
-            ExprStr(s) => expr(Expression::Str(s.to_string())),
-            ExprNumber(n) => expr(Expression::Number(n)),
-            ExprSize(s) => expr(Expression::Size(s)),
-            ExprBool(b) => expr(Expression::Bool(b)),
+            ExprIdent(i) => {
+                self.eat();
+                spanned(Expr::Ident(Ident(i.to_string())))
+            }
+            ExprStr { string, terminated } => {
+                if !terminated {
+                    self.expected_at("quote", first.span.end);
+                }
+
+                self.eat();
+                spanned(Expr::Str(string.to_string()))
+            }
+            ExprNumber(n) => { self.eat(); spanned(Expr::Number(n)) }
+            ExprSize(s) => { self.eat(); spanned(Expr::Size(s)) }
+            ExprBool(b) => { self.eat(); spanned(Expr::Bool(b)) }
+
             LeftParen => self.parse_tuple(),
             LeftBrace => self.parse_object(),
             _ => return None,
@@ -264,56 +251,48 @@ impl<'s> Parser<'s> {
     }
 
     /// Parse a tuple expression.
-    fn parse_tuple(&mut self) -> Spanned<Expression> {
+    fn parse_tuple(&mut self) -> Spanned<Expr> {
         let start = self.pos();
 
         // TODO: Do the thing.
-        self.eat_until(|t| matches!(t, RightParen | RightBracket), false);
-        if self.peekv() == Some(RightParen) {
-            self.eat();
-        }
+        self.eat_until(|t| t == RightParen, true);
 
         let end = self.pos();
         let span = Span { start, end };
 
-        Spanned { v: Expression::Tuple(Tuple::new()), span }
+        Spanned { v: Expr::Tuple(Tuple::new()), span }
     }
 
     /// Parse an object expression.
-    fn parse_object(&mut self) -> Spanned<Expression> {
+    fn parse_object(&mut self) -> Spanned<Expr> {
         let start = self.pos();
 
         // TODO: Do the thing.
-        self.eat_until(|t| matches!(t, RightBrace | RightBracket), false);
-        if self.peekv() == Some(RightBrace) {
-            self.eat();
-        }
+        self.eat_until(|t| t == RightBrace, true);
 
         let end = self.pos();
         let span = Span { start, end };
 
-        Spanned { v: Expression::Object(Object::new()), span }
+        Spanned { v: Expr::Object(Object::new()), span }
     }
 
     /// Skip all whitespace/comment tokens.
     fn skip_whitespace(&mut self) {
         self.eat_until(|t|
-            !matches!(t, Whitespace(_) | LineComment(_) | BlockComment(_)), false)
-    }
-
-    /// Add an error about an `thing` which was expected but not found at the
-    /// given position.
-    fn expected_at(&mut self, thing: &str, pos: Position) {
-        let error = Error::new(format!("expected {}", thing));
-        self.errors.push(Spanned::new(error, Span::at(pos)));
+            !matches!(t, Space(_) | LineComment(_) | BlockComment(_)), false)
     }
 
     /// Add an error about an expected `thing` which was not found, showing
     /// what was found instead.
     fn expected_found(&mut self, thing: &str, found: Spanned<Token>) {
-        let message = format!("expected {}, found {}", thing, name(found.v));
-        let error = Error::new(message);
-        self.errors.push(Spanned::new(error, found.span));
+        self.errors.push(err!(found.span;
+            "expected {}, found {}", thing, name(found.v)));
+    }
+
+    /// Add an error about an `thing` which was expected but not found at the
+    /// given position.
+    fn expected_at(&mut self, thing: &str, pos: Position) {
+        self.errors.push(err!(Span::at(pos); "expected {}", thing));
     }
 
     /// Add a found-error if `found` is some and a positional error, otherwise.
@@ -329,12 +308,6 @@ impl<'s> Parser<'s> {
         }
     }
 
-    /// Add an error about an unexpected token `found`.
-    fn unexpected(&mut self, found: Spanned<Token>) {
-        let error = Error::new(format!("unexpected {}", name(found.v)));
-        self.errors.push(Spanned::new(error, found.span));
-    }
-
     /// Consume tokens until the function returns true and only consume the last
     /// token if instructed to.
     fn eat_until<F>(&mut self, mut f: F, eat_match: bool)
@@ -351,19 +324,10 @@ impl<'s> Parser<'s> {
         }
     }
 
-    /// Consume and return the next token, update positions and colorize the
-    /// token. All colorable tokens are per default colorized here, to override
-    /// a colorization use `Colorization::replace_last`.
+    /// Consume and return the next token.
     fn eat(&mut self) -> Option<Spanned<Token<'s>>> {
-        let token = self.peeked.take()
-            .unwrap_or_else(|| self.tokens.next());
-
-        if let Some(token) = token {
-            self.last_position = self.position;
-            self.position = token.span.end;
-        }
-
-        token
+        self.peeked.take()
+            .unwrap_or_else(|| self.tokens.next())
     }
 
     /// Peek at the next token without consuming it.
@@ -379,23 +343,18 @@ impl<'s> Parser<'s> {
     /// The position at the end of the last eat token / start of the peekable
     /// token.
     fn pos(&self) -> Position {
-        self.position
-    }
-
-    /// The position at the start of the last eaten token.
-    fn last_pos(&self) -> Position {
-        self.last_position
+        self.peeked.flatten()
+            .map(|s| s.span.start)
+            .unwrap_or_else(|| self.tokens.pos())
     }
 }
 
-/// The name of a token in an `expected <...>` error.
+/// The name of a token in an `(un)expected <...>` error.
 fn name(token: Token) -> &'static str {
     match token {
-        Whitespace(_) => "whitespace",
+        Space(_)      => "space",
         LineComment(_) | BlockComment(_) => "comment",
-        StarSlash     => "end of block comment",
-        LeftBracket   => "opening bracket",
-        RightBracket  => "closing bracket",
+        Function { .. } => "function",
         LeftParen     => "opening paren",
         RightParen    => "closing paren",
         LeftBrace     => "opening brace",
@@ -404,13 +363,16 @@ fn name(token: Token) -> &'static str {
         Comma         => "comma",
         Equals        => "equals sign",
         ExprIdent(_)  => "identifier",
-        ExprStr(_)    => "string",
+        ExprStr { .. }    => "string",
         ExprNumber(_) => "number",
         ExprSize(_)   => "size",
-        ExprBool(_)   => "bool",
+        ExprBool(_)   => "boolean",
         Star          => "star",
         Underscore    => "underscore",
         Backtick      => "backtick",
         Text(_)       => "invalid identifier",
+        Invalid("]")  => "closing bracket",
+        Invalid("*/") => "end of block comment",
+        Invalid(_)    => "invalid token",
     }
 }