diff options
Diffstat (limited to 'src/syntax')
| -rw-r--r-- | src/syntax/mod.rs | 50 | ||||
| -rw-r--r-- | src/syntax/parsing.rs | 366 |
2 files changed, 285 insertions, 131 deletions
diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs index bcec05af..e9725f04 100644 --- a/src/syntax/mod.rs +++ b/src/syntax/mod.rs @@ -154,6 +154,22 @@ pub struct FuncArgs { pub keyword: Object, } +#[derive(Debug, Clone, PartialEq)] +pub enum Arg { + Pos(Spanned<Expression>), + Key(Pair), +} + +impl Arg { + /// The span or the value or combined span of key and value. + pub fn span(&self) -> Span { + match self { + Arg::Pos(spanned) => spanned.span, + Arg::Key(Pair { key, value }) => Span::merge(key.span, value.span), + } + } +} + impl FuncArgs { pub fn new() -> FuncArgs { FuncArgs { @@ -246,32 +262,40 @@ pub struct Colorization { pub tokens: Vec<Spanned<ColorToken>>, } +impl Colorization { + pub fn new() -> Colorization { + Colorization { tokens: vec![] } + } + + pub fn add(&mut self, token: ColorToken, span: Span) { + self.tokens.push(Spanned { v: token, span }); + } + + pub fn replace_last(&mut self, token: ColorToken) { + self.tokens.last_mut().expect("replace_last: no token").v = token; + } +} + /// Entities which can be colored by syntax highlighting. #[derive(Debug, Copy, Clone, Eq, PartialEq)] pub enum ColorToken { Comment, - Bracket, FuncName, Colon, - Key, Equals, Comma, - Paren, Brace, - ExprIdent, ExprStr, ExprNumber, ExprSize, ExprBool, - Bold, Italic, Monospace, - Invalid, } @@ -279,3 +303,17 @@ pub enum ColorToken { pub struct ErrorMap { pub errors: Vec<Spanned<String>>, } + +impl ErrorMap { + pub fn new() -> ErrorMap { + ErrorMap { errors: vec![] } + } + + pub fn add(&mut self, message: impl Into<String>, span: Span) { + self.errors.push(Spanned { v: message.into(), span }); + } + + pub fn add_at(&mut self, message: impl Into<String>, pos: Position) { + self.errors.push(Spanned { v: message.into(), span: Span::at(pos) }) + } +} diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs index f0a68641..71d6b251 100644 --- a/src/syntax/parsing.rs +++ b/src/syntax/parsing.rs @@ -20,7 +20,6 @@ struct Parser<'s> { ctx: ParseContext<'s>, colorization: Colorization, error_map: ErrorMap, - tokens: Tokens<'s>, peeked: Option<Option<Spanned<Token<'s>>>>, position: Position, @@ -32,9 +31,8 @@ impl<'s> Parser<'s> { Parser { src, ctx, - error_map: ErrorMap { errors: vec![] }, - colorization: Colorization { tokens: vec![] }, - + error_map: ErrorMap::new(), + colorization: Colorization::new(), tokens: Tokens::new(src), peeked: None, position: Position::ZERO, @@ -42,6 +40,7 @@ impl<'s> Parser<'s> { } } + /// The main parsing entrypoint. fn parse(mut self) -> (SyntaxTree, Colorization, ErrorMap) { let mut tree = SyntaxTree::new(); @@ -79,10 +78,18 @@ impl<'s> Parser<'s> { (tree, self.colorization, self.error_map) } + /// Parses a function including header and body with the cursor starting + /// right behind the first opening bracket. fn parse_func(&mut self) -> Option<Spanned<Node>> { let start = self.last_pos(); let header = self.parse_func_header(); + self.eat_until(|t| t == RightBracket, false); + + if self.eat().map(Spanned::value) != Some(RightBracket) { + self.expected_at("closing bracket", self.pos()); + } + let call = self.parse_func_call(header)?; let end = self.pos(); @@ -91,21 +98,17 @@ impl<'s> Parser<'s> { Some(Spanned { v: Node::Func(call), span }) } + /// Parses a function header including the closing bracket. fn parse_func_header(&mut self) -> Option<FuncHeader> { self.skip_whitespace(); - - let name = self.parse_func_name().or_else(|| { - self.eat_until(|t| t == RightBracket, true); - None - })?; + let name = self.parse_func_name()?; self.skip_whitespace(); let args = match self.eat() { Some(Spanned { v: Colon, .. }) => self.parse_func_args(), Some(Spanned { v: RightBracket, .. }) => FuncArgs::new(), other => { - self.expected("colon or closing bracket", other); - self.eat_until(|t| t == RightBracket, true); + self.expected_at("colon or closing bracket", name.span.end); FuncArgs::new() } }; @@ -113,8 +116,140 @@ impl<'s> Parser<'s> { Some(FuncHeader { name, args }) } + /// Parses the function name if is the next token. Otherwise, it adds an + /// error and returns `None`. + fn parse_func_name(&mut self) -> Option<Spanned<Ident>> { + match self.eat() { + Some(Spanned { v: ExprIdent(ident), span }) => { + self.colorization.replace_last(ColorToken::FuncName); + return Some(Spanned { v: Ident(ident.to_string()), span }); + } + other => self.expected_found_or_at("identifier", other, self.pos()), + } + + None + } + + /// Parses the function arguments and stops right before the final closing + /// bracket. + fn parse_func_args(&mut self) -> FuncArgs { + let mut args = FuncArgs::new(); + + loop { + self.skip_whitespace(); + match self.peekv() { + Some(RightBracket) | None => break, + _ => match self.parse_arg() { + Some(Arg::Pos(item)) => args.add_pos(item), + Some(Arg::Key(pair)) => args.add_key_pair(pair), + None => {} + } + } + } + + args + } + + /// Parse a positional or keyword argument. + fn parse_arg(&mut self) -> Option<Arg> { + let first = self.peek()?; + let span = first.span; + + let arg = if let ExprIdent(ident) = first.v { + self.eat(); + self.skip_whitespace(); + + let ident = Ident(ident.to_string()); + if let Some(Equals) = self.peekv() { + self.colorization.replace_last(ColorToken::Key); + + self.eat(); + self.skip_whitespace(); + + self.parse_expr().map(|value| { + Arg::Key(Pair { + key: Spanned { v: ident, span }, + value, + }) + }) + } else { + Some(Arg::Pos(Spanned::new(Expression::Ident(ident), span))) + } + } else { + self.parse_expr().map(|expr| Arg::Pos(expr)) + }; + + if let Some(arg) = &arg { + self.skip_whitespace(); + match self.peekv() { + Some(RightBracket) => {} + Some(Comma) => { self.eat(); } + Some(_) => self.expected_at("comma", arg.span().end), + _ => {} + } + } else { + let found = self.eat(); + self.expected_found_or_at("value", found, self.pos()); + } + + arg + } + + /// Parse a atomic or compound (tuple / object) expression. + fn parse_expr(&mut self) -> Option<Spanned<Expression>> { + let first = self.peek()?; + let mut expr = |v| { + self.eat(); + Spanned { v, span: first.span } + }; + + Some(match first.v { + ExprIdent(i) => expr(Expression::Ident(Ident(i.to_string()))), + ExprStr(s) => expr(Expression::Str(s.to_string())), + ExprNumber(n) => expr(Expression::Number(n)), + ExprSize(s) => expr(Expression::Size(s)), + ExprBool(b) => expr(Expression::Bool(b)), + LeftParen => self.parse_tuple(), + LeftBrace => self.parse_object(), + _ => return None, + }) + } + + /// Parse a tuple expression. + fn parse_tuple(&mut self) -> Spanned<Expression> { + let start = self.pos(); + + // TODO: Do the thing. + self.eat_until(|t| matches!(t, RightParen | RightBracket), false); + if self.peekv() == Some(RightParen) { + self.eat(); + } + + let end = self.pos(); + let span = Span { start, end }; + + Spanned { v: Expression::Tuple(Tuple::new()), span } + } + + /// Parse an object expression. + fn parse_object(&mut self) -> Spanned<Expression> { + let start = self.pos(); + + // TODO: Do the thing. + self.eat_until(|t| matches!(t, RightBrace | RightBracket), false); + if self.peekv() == Some(RightBrace) { + self.eat(); + } + + let end = self.pos(); + let span = Span { start, end }; + + Spanned { v: Expression::Object(Object::new()), span } + } + + /// Parse the body of a function invocation. fn parse_func_call(&mut self, header: Option<FuncHeader>) -> Option<FuncCall> { - let body = if self.peek() == Some(LeftBracket) { + let body = if self.peekv() == Some(LeftBracket) { self.eat(); let start = self.tokens.index(); @@ -127,9 +262,10 @@ impl<'s> Parser<'s> { let body = &self.src[start .. end]; if found { - assert_eq!(self.eat().map(Spanned::value), Some(RightBracket)); + let next = self.eat().map(Spanned::value); + debug_assert_eq!(next, Some(RightBracket)); } else { - self.error_here("expected closing bracket"); + self.expected_at("closing bracket", self.pos()); } Some(body) @@ -139,111 +275,57 @@ impl<'s> Parser<'s> { let header = header?; let parser = self.ctx.scope.get_parser(header.name.v.as_str()).or_else(|| { - self.error( - format!("unknown function: `{}`", header.name.v), - header.name.span - ); + let message = format!("unknown function: `{}`", header.name.v); + self.error_map.add(message, header.name.span); None })?; Some(FuncCall(parser(header, body, self.ctx).unwrap())) } - fn parse_func_name(&mut self) -> Option<Spanned<Ident>> { - match self.eat() { - Some(Spanned { v: ExprIdent(ident), span }) => { - self.color(Spanned { v: ColorToken::FuncName, span }, true); - Some(Spanned { v: Ident(ident.to_string()), span }) - } - other => { - self.expected("identifier", other); - None - } - } - } - - fn parse_func_args(&mut self) -> FuncArgs { - // todo!() - self.eat_until(|t| t == RightBracket, true); - FuncArgs::new() - } - - fn parse_tuple(&mut self) -> Spanned<Expression> { - todo!("parse_tuple") - } - - fn parse_object(&mut self) -> Spanned<Expression> { - todo!("parse_object") - } - + /// Skip all whitespace/comment tokens. fn skip_whitespace(&mut self) { - self.eat_until(|t| match t { - Whitespace(_) | LineComment(_) | BlockComment(_) => false, - _ => true, - }, false) - } - - fn expected(&mut self, thing: &str, found: Option<Spanned<Token>>) { - if let Some(Spanned { v: found, span }) = found { - self.error( - format!("expected {}, found {}", thing, name(found)), - span - ); - } else { - self.error_here(format!("expected {}", thing)); - } - } - - fn unexpected(&mut self, found: Spanned<Token>) { - self.error_map.errors.push(found.map(|t| format!("unexpected {}", name(t)))); + self.eat_until(|t| + !matches!(t, Whitespace(_) | LineComment(_) | BlockComment(_)), false) } - fn error(&mut self, message: impl Into<String>, span: Span) { - self.error_map.errors.push(Spanned { v: message.into(), span }); + /// Add an error about an `thing` which was expected but not found at the + /// given position. + fn expected_at(&mut self, thing: &str, pos: Position) { + self.error_map.add_at(format!("expected {}", thing), pos); } - fn error_here(&mut self, message: impl Into<String>) { - self.error(message, Span::at(self.pos())); + /// Add an error about an expected `thing` which was not found, showing + /// what was found instead. + fn expected_found(&mut self, thing: &str, found: Spanned<Token>) { + let message = format!("expected {}, found {}", thing, name(found.v)); + self.error_map.add(message, found.span); } - fn color(&mut self, token: Spanned<ColorToken>, replace_last: bool) { - if replace_last { - if let Some(last) = self.colorization.tokens.last_mut() { - *last = token; - return; - } + /// Add a found-error if `found` is some and a positional error, otherwise. + fn expected_found_or_at( + &mut self, + thing: &str, + found: Option<Spanned<Token>>, + pos: Position + ) { + match found { + Some(found) => self.expected_found(thing, found), + None => self.expected_at(thing, pos), } - - self.colorization.tokens.push(token); } - fn color_token(&mut self, token: Spanned<Token<'s>>) { - let colored = match token.v { - LineComment(_) | BlockComment(_) => Some(ColorToken::Comment), - StarSlash => Some(ColorToken::Invalid), - LeftBracket | RightBracket => Some(ColorToken::Bracket), - LeftParen | RightParen => Some(ColorToken::Paren), - LeftBrace | RightBrace => Some(ColorToken::Brace), - Colon => Some(ColorToken::Colon), - Comma => Some(ColorToken::Comma), - Equals => Some(ColorToken::Equals), - ExprIdent(_) => Some(ColorToken::ExprIdent), - ExprStr(_) => Some(ColorToken::ExprStr), - ExprNumber(_) => Some(ColorToken::ExprNumber), - ExprSize(_) => Some(ColorToken::ExprSize), - ExprBool(_) => Some(ColorToken::ExprBool), - _ => None, - }; - - if let Some(color) = colored { - self.colorization.tokens.push(Spanned { v: color, span: token.span }); - } + /// Add an error about an unexpected token `found`. + fn unexpected(&mut self, found: Spanned<Token>) { + self.error_map.add(format!("unexpected {}", name(found.v)), found.span); } + /// Consume tokens until the function returns true and only consume the last + /// token if instructed to. fn eat_until<F>(&mut self, mut f: F, eat_match: bool) where F: FnMut(Token<'s>) -> bool { while let Some(token) = self.peek() { - if f(token) { + if f(token.v) { if eat_match { self.eat(); } @@ -254,56 +336,90 @@ impl<'s> Parser<'s> { } } + /// Consume and return the next token, update positions and colorize the + /// token. All colorable tokens are per default colorized here, to override + /// a colorization use `Colorization::replace_last`. fn eat(&mut self) -> Option<Spanned<Token<'s>>> { - let token = self.peeked.take().unwrap_or_else(|| self.tokens.next()); + let token = self.peeked.take() + .unwrap_or_else(|| self.tokens.next()); - self.last_position = self.position; - if let Some(spanned) = token { - self.color_token(spanned); - self.position = spanned.span.end; + if let Some(token) = token { + if let Some(color) = color(token.v) { + self.colorization.add(color, token.span); + } + + self.last_position = self.position; + self.position = token.span.end; } token } - fn peek(&mut self) -> Option<Token<'s>> { + /// Peek at the next token without consuming it. + fn peek(&mut self) -> Option<Spanned<Token<'s>>> { let iter = &mut self.tokens; - self.peeked - .get_or_insert_with(|| iter.next()) - .map(Spanned::value) + *self.peeked.get_or_insert_with(|| iter.next()) + } + + fn peekv(&mut self) -> Option<Token<'s>> { + self.peek().map(Spanned::value) } + /// The position at the end of the last eat token / start of the peekable + /// token. fn pos(&self) -> Position { self.position } + /// The position at the start of the last eaten token. fn last_pos(&self) -> Position { self.last_position } } +/// The name of a token in an `expected <...>` error. fn name(token: Token) -> &'static str { match token { Whitespace(_) => "whitespace", LineComment(_) | BlockComment(_) => "comment", - StarSlash => "end of block comment", - LeftBracket => "opening bracket", - RightBracket => "closing bracket", - LeftParen => "opening paren", - RightParen => "closing paren", - LeftBrace => "opening brace", - RightBrace => "closing brace", - Colon => "colon", - Comma => "comma", - Equals => "equals sign", - ExprIdent(_) => "identifier", - ExprStr(_) => "string", + StarSlash => "end of block comment", + LeftBracket => "opening bracket", + RightBracket => "closing bracket", + LeftParen => "opening paren", + RightParen => "closing paren", + LeftBrace => "opening brace", + RightBrace => "closing brace", + Colon => "colon", + Comma => "comma", + Equals => "equals sign", + ExprIdent(_) => "identifier", + ExprStr(_) => "string", ExprNumber(_) => "number", - ExprSize(_) => "size", - ExprBool(_) => "bool", - Star => "star", - Underscore => "underscore", - Backtick => "backtick", - Text(_) => "text", + ExprSize(_) => "size", + ExprBool(_) => "bool", + Star => "star", + Underscore => "underscore", + Backtick => "backtick", + Text(_) => "invalid identifier", } } + +/// The color token corresponding to a token. +fn color(token: Token) -> Option<ColorToken> { + Some(match token { + LineComment(_) | BlockComment(_) => ColorToken::Comment, + LeftBracket | RightBracket => ColorToken::Bracket, + LeftParen | RightParen => ColorToken::Paren, + LeftBrace | RightBrace => ColorToken::Brace, + Colon => ColorToken::Colon, + Comma => ColorToken::Comma, + Equals => ColorToken::Equals, + ExprIdent(_) => ColorToken::ExprIdent, + ExprStr(_) => ColorToken::ExprStr, + ExprNumber(_) => ColorToken::ExprNumber, + ExprSize(_) => ColorToken::ExprSize, + ExprBool(_) => ColorToken::ExprBool, + StarSlash => ColorToken::Invalid, + _ => return None, + }) +} |