summaryrefslogtreecommitdiff
path: root/src/parse/parser.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/parse/parser.rs')
-rw-r--r--src/parse/parser.rs658
1 files changed, 658 insertions, 0 deletions
diff --git a/src/parse/parser.rs b/src/parse/parser.rs
new file mode 100644
index 00000000..2ff30397
--- /dev/null
+++ b/src/parse/parser.rs
@@ -0,0 +1,658 @@
+use std::str::FromStr;
+
+use super::escaping::*;
+use super::*;
+use crate::color::RgbaColor;
+use crate::compute::table::SpannedEntry;
+use crate::syntax::*;
+use crate::{Feedback, Pass};
+
+/// Parse a string of source code.
+pub fn parse(src: &str) -> Pass<SyntaxTree> {
+ Parser::new(src).parse()
+}
+
+struct Parser<'s> {
+ tokens: Tokens<'s>,
+ peeked: Option<Option<Spanned<Token<'s>>>>,
+ delimiters: Vec<(Pos, Token<'static>)>,
+ at_block_or_line_start: bool,
+ feedback: Feedback,
+}
+
+impl<'s> Parser<'s> {
+ fn new(src: &'s str) -> Self {
+ Self {
+ tokens: Tokens::new(src, TokenMode::Body),
+ peeked: None,
+ delimiters: vec![],
+ at_block_or_line_start: true,
+ feedback: Feedback::new(),
+ }
+ }
+
+ fn parse(mut self) -> Pass<SyntaxTree> {
+ let tree = self.parse_body_contents();
+ Pass::new(tree, self.feedback)
+ }
+}
+
+// Typesetting content.
+impl Parser<'_> {
+ fn parse_body_contents(&mut self) -> SyntaxTree {
+ let mut tree = SyntaxTree::new();
+
+ self.at_block_or_line_start = true;
+ while !self.eof() {
+ if let Some(node) = self.parse_node() {
+ tree.push(node);
+ }
+ }
+
+ tree
+ }
+
+ fn parse_node(&mut self) -> Option<Spanned<SyntaxNode>> {
+ let token = self.peek()?;
+ let end = Span::at(token.span.end);
+
+ // Set block or line start to false because most nodes have that effect, but
+ // remember the old value to actually check it for hashtags and because comments
+ // and spaces want to retain it.
+ let was_at_block_or_line_start = self.at_block_or_line_start;
+ self.at_block_or_line_start = false;
+
+ Some(match token.v {
+ // Starting from two newlines counts as a paragraph break, a single
+ // newline does not.
+ Token::Space(n) => {
+ if n == 0 {
+ self.at_block_or_line_start = was_at_block_or_line_start;
+ } else if n >= 1 {
+ self.at_block_or_line_start = true;
+ }
+
+ self.with_span(if n >= 2 {
+ SyntaxNode::Parbreak
+ } else {
+ SyntaxNode::Spacing
+ })
+ }
+
+ Token::LineComment(_) | Token::BlockComment(_) => {
+ self.at_block_or_line_start = was_at_block_or_line_start;
+ self.eat();
+ return None;
+ }
+
+ Token::LeftBracket => {
+ let call = self.parse_bracket_call(false);
+ self.at_block_or_line_start = false;
+ call.map(SyntaxNode::Call)
+ }
+
+ Token::Star => self.with_span(SyntaxNode::ToggleBolder),
+ Token::Underscore => self.with_span(SyntaxNode::ToggleItalic),
+ Token::Backslash => self.with_span(SyntaxNode::Linebreak),
+
+ Token::Hashtag if was_at_block_or_line_start => {
+ self.parse_heading().map(SyntaxNode::Heading)
+ }
+
+ Token::Raw { raw, terminated } => {
+ if !terminated {
+ error!(@self.feedback, end, "expected backtick");
+ }
+ self.with_span(SyntaxNode::Raw(unescape_raw(raw)))
+ }
+
+ Token::Code { lang, raw, terminated } => {
+ if !terminated {
+ error!(@self.feedback, end, "expected backticks");
+ }
+
+ let lang = lang.and_then(|lang| {
+ if let Some(ident) = Ident::new(lang.v) {
+ Some(Spanned::new(ident, lang.span))
+ } else {
+ error!(@self.feedback, lang.span, "invalid identifier");
+ None
+ }
+ });
+
+ let mut lines = unescape_code(raw);
+ let block = lines.len() > 1;
+
+ if lines.last().map(|s| s.is_empty()).unwrap_or(false) {
+ lines.pop();
+ }
+
+ self.with_span(SyntaxNode::Code(Code { lang, lines, block }))
+ }
+
+ Token::Text(text) => self.with_span(SyntaxNode::Text(text.to_string())),
+ Token::Hashtag => self.with_span(SyntaxNode::Text("#".to_string())),
+
+ Token::UnicodeEscape { sequence, terminated } => {
+ if !terminated {
+ error!(@self.feedback, end, "expected closing brace");
+ }
+
+ if let Some(c) = hex_to_char(sequence) {
+ self.with_span(SyntaxNode::Text(c.to_string()))
+ } else {
+ error!(@self.feedback, token.span, "invalid unicode escape sequence");
+ self.eat();
+ return None;
+ }
+ }
+
+ unexpected => {
+ error!(@self.feedback, token.span, "unexpected {}", unexpected.name());
+ self.eat();
+ return None;
+ }
+ })
+ }
+
+ fn parse_heading(&mut self) -> Spanned<Heading> {
+ let start = self.pos();
+ self.assert(Token::Hashtag);
+
+ let mut level = 0;
+ while self.peekv() == Some(Token::Hashtag) {
+ level += 1;
+ self.eat();
+ }
+
+ let span = Span::new(start, self.pos());
+ let level = Spanned::new(level, span);
+
+ if level.v > 5 {
+ warning!(
+ @self.feedback, level.span,
+ "section depth larger than 6 has no effect",
+ );
+ }
+
+ self.skip_ws();
+
+ let mut tree = SyntaxTree::new();
+ while !self.eof() && !matches!(self.peekv(), Some(Token::Space(n)) if n >= 1) {
+ if let Some(node) = self.parse_node() {
+ tree.push(node);
+ }
+ }
+
+ let span = Span::new(start, self.pos());
+ Spanned::new(Heading { level, tree }, span)
+ }
+}
+
+// Function calls.
+impl Parser<'_> {
+ fn parse_bracket_call(&mut self, chained: bool) -> Spanned<CallExpr> {
+ let before_bracket = self.pos();
+ if !chained {
+ self.start_group(Group::Bracket);
+ self.tokens.push_mode(TokenMode::Header);
+ }
+
+ let before_name = self.pos();
+ self.start_group(Group::Subheader);
+ self.skip_ws();
+ let name = self.parse_ident().unwrap_or_else(|| {
+ self.expected_found_or_at("function name", before_name);
+ Spanned::new(Ident(String::new()), Span::at(before_name))
+ });
+
+ self.skip_ws();
+
+ let mut args = match self.eatv() {
+ Some(Token::Colon) => self.parse_table_contents().0,
+ Some(_) => {
+ self.expected_at("colon", name.span.end);
+ while self.eat().is_some() {}
+ TableExpr::new()
+ }
+ None => TableExpr::new(),
+ };
+
+ self.end_group();
+ self.skip_ws();
+ let (has_chained_child, end) = if self.peek().is_some() {
+ let item = self.parse_bracket_call(true);
+ let span = item.span;
+ let t = vec![item.map(SyntaxNode::Call)];
+ args.push(SpannedEntry::val(Spanned::new(Expr::Tree(t), span)));
+ (true, span.end)
+ } else {
+ self.tokens.pop_mode();
+ (false, self.end_group().end)
+ };
+
+ let start = if chained { before_name } else { before_bracket };
+ let mut span = Span::new(start, end);
+
+ if self.check(Token::LeftBracket) && !has_chained_child {
+ self.start_group(Group::Bracket);
+ self.tokens.push_mode(TokenMode::Body);
+
+ let body = self.parse_body_contents();
+
+ self.tokens.pop_mode();
+ let body_span = self.end_group();
+
+ let expr = Expr::Tree(body);
+ args.push(SpannedEntry::val(Spanned::new(expr, body_span)));
+ span.expand(body_span);
+ }
+
+ Spanned::new(CallExpr { name, args }, span)
+ }
+
+ fn parse_paren_call(&mut self, name: Spanned<Ident>) -> Spanned<CallExpr> {
+ self.start_group(Group::Paren);
+ let args = self.parse_table_contents().0;
+ let args_span = self.end_group();
+ let span = Span::merge(name.span, args_span);
+ Spanned::new(CallExpr { name, args }, span)
+ }
+}
+
+// Tables.
+impl Parser<'_> {
+ fn parse_table_contents(&mut self) -> (TableExpr, bool) {
+ let mut table = TableExpr::new();
+ let mut comma_and_keyless = true;
+
+ while {
+ self.skip_ws();
+ !self.eof()
+ } {
+ let (key, val) = if let Some(ident) = self.parse_ident() {
+ self.skip_ws();
+
+ match self.peekv() {
+ Some(Token::Equals) => {
+ self.eat();
+ self.skip_ws();
+ if let Some(value) = self.parse_expr() {
+ (Some(ident), value)
+ } else {
+ self.expected("value");
+ continue;
+ }
+ }
+
+ Some(Token::LeftParen) => {
+ let call = self.parse_paren_call(ident);
+ (None, call.map(Expr::Call))
+ }
+
+ _ => (None, ident.map(Expr::Ident)),
+ }
+ } else if let Some(value) = self.parse_expr() {
+ (None, value)
+ } else {
+ self.expected("value");
+ continue;
+ };
+
+ let behind = val.span.end;
+ if let Some(key) = key {
+ comma_and_keyless = false;
+ table.insert(key.v.0, SpannedEntry::new(key.span, val));
+ self.feedback
+ .decorations
+ .push(Spanned::new(Decoration::TableKey, key.span));
+ } else {
+ table.push(SpannedEntry::val(val));
+ }
+
+ if {
+ self.skip_ws();
+ self.eof()
+ } {
+ break;
+ }
+
+ self.expect_at(Token::Comma, behind);
+ comma_and_keyless = false;
+ }
+
+ let coercable = comma_and_keyless && !table.is_empty();
+ (table, coercable)
+ }
+}
+
+type Binop = fn(Box<Spanned<Expr>>, Box<Spanned<Expr>>) -> Expr;
+
+// Expressions and values.
+impl Parser<'_> {
+ fn parse_expr(&mut self) -> Option<Spanned<Expr>> {
+ self.parse_binops("summand", Self::parse_term, |token| match token {
+ Token::Plus => Some(Expr::Add),
+ Token::Hyphen => Some(Expr::Sub),
+ _ => None,
+ })
+ }
+
+ fn parse_term(&mut self) -> Option<Spanned<Expr>> {
+ self.parse_binops("factor", Self::parse_factor, |token| match token {
+ Token::Star => Some(Expr::Mul),
+ Token::Slash => Some(Expr::Div),
+ _ => None,
+ })
+ }
+
+ /// Parse expression of the form `<operand> (<op> <operand>)*`.
+ fn parse_binops(
+ &mut self,
+ operand_name: &str,
+ mut parse_operand: impl FnMut(&mut Self) -> Option<Spanned<Expr>>,
+ mut parse_op: impl FnMut(Token) -> Option<Binop>,
+ ) -> Option<Spanned<Expr>> {
+ let mut left = parse_operand(self)?;
+
+ self.skip_ws();
+ while let Some(token) = self.peek() {
+ if let Some(op) = parse_op(token.v) {
+ self.eat();
+ self.skip_ws();
+
+ if let Some(right) = parse_operand(self) {
+ let span = Span::merge(left.span, right.span);
+ let v = op(Box::new(left), Box::new(right));
+ left = Spanned::new(v, span);
+ self.skip_ws();
+ continue;
+ }
+
+ error!(
+ @self.feedback, Span::merge(left.span, token.span),
+ "missing right {}", operand_name,
+ );
+ }
+ break;
+ }
+
+ Some(left)
+ }
+
+ fn parse_factor(&mut self) -> Option<Spanned<Expr>> {
+ if let Some(hyph) = self.check_eat(Token::Hyphen) {
+ self.skip_ws();
+ if let Some(factor) = self.parse_factor() {
+ let span = Span::merge(hyph.span, factor.span);
+ Some(Spanned::new(Expr::Neg(Box::new(factor)), span))
+ } else {
+ error!(@self.feedback, hyph.span, "dangling minus");
+ None
+ }
+ } else {
+ self.parse_value()
+ }
+ }
+
+ fn parse_value(&mut self) -> Option<Spanned<Expr>> {
+ let Spanned { v: token, span } = self.peek()?;
+ Some(match token {
+ // This could be a function call or an identifier.
+ Token::Ident(id) => {
+ let name = Spanned::new(Ident(id.to_string()), span);
+ self.eat();
+ self.skip_ws();
+ if self.check(Token::LeftParen) {
+ self.parse_paren_call(name).map(Expr::Call)
+ } else {
+ name.map(Expr::Ident)
+ }
+ }
+
+ Token::Str { string, terminated } => {
+ if !terminated {
+ self.expected_at("quote", span.end);
+ }
+ self.with_span(Expr::Str(unescape_string(string)))
+ }
+
+ Token::Bool(b) => self.with_span(Expr::Bool(b)),
+ Token::Number(n) => self.with_span(Expr::Number(n)),
+ Token::Length(s) => self.with_span(Expr::Length(s)),
+ Token::Hex(s) => {
+ if let Ok(color) = RgbaColor::from_str(s) {
+ self.with_span(Expr::Color(color))
+ } else {
+ // Heal color by assuming black.
+ error!(@self.feedback, span, "invalid color");
+ let healed = RgbaColor::new_healed(0, 0, 0, 255);
+ self.with_span(Expr::Color(healed))
+ }
+ }
+
+ // This could be a table or a parenthesized expression. We parse as
+ // a table in any case and coerce the table into a value if it is
+ // coercable (length 1 and no trailing comma).
+ Token::LeftParen => {
+ self.start_group(Group::Paren);
+ let (table, coercable) = self.parse_table_contents();
+ let span = self.end_group();
+
+ let expr = if coercable {
+ table.into_values().next().expect("table is coercable").val.v
+ } else {
+ Expr::Table(table)
+ };
+
+ Spanned::new(expr, span)
+ }
+
+ // This is a content expression.
+ Token::LeftBrace => {
+ self.start_group(Group::Brace);
+ self.tokens.push_mode(TokenMode::Body);
+
+ let tree = self.parse_body_contents();
+
+ self.tokens.pop_mode();
+ let span = self.end_group();
+ Spanned::new(Expr::Tree(tree), span)
+ }
+
+ // This is a bracketed function call.
+ Token::LeftBracket => {
+ let call = self.parse_bracket_call(false);
+ let tree = vec![call.map(SyntaxNode::Call)];
+ Spanned::new(Expr::Tree(tree), span)
+ }
+
+ _ => return None,
+ })
+ }
+
+ fn parse_ident(&mut self) -> Option<Spanned<Ident>> {
+ self.peek().and_then(|token| match token.v {
+ Token::Ident(id) => Some(self.with_span(Ident(id.to_string()))),
+ _ => None,
+ })
+ }
+}
+
+// Error handling.
+impl Parser<'_> {
+ fn expect_at(&mut self, token: Token<'_>, pos: Pos) -> bool {
+ if self.check(token) {
+ self.eat();
+ true
+ } else {
+ self.expected_at(token.name(), pos);
+ false
+ }
+ }
+
+ fn expected(&mut self, thing: &str) {
+ if let Some(found) = self.eat() {
+ error!(
+ @self.feedback, found.span,
+ "expected {}, found {}", thing, found.v.name(),
+ );
+ } else {
+ error!(@self.feedback, Span::at(self.pos()), "expected {}", thing);
+ }
+ }
+
+ fn expected_at(&mut self, thing: &str, pos: Pos) {
+ error!(@self.feedback, Span::at(pos), "expected {}", thing);
+ }
+
+ fn expected_found_or_at(&mut self, thing: &str, pos: Pos) {
+ if self.eof() {
+ self.expected_at(thing, pos)
+ } else {
+ self.expected(thing);
+ }
+ }
+}
+
+// Parsing primitives.
+impl<'s> Parser<'s> {
+ fn start_group(&mut self, group: Group) {
+ let start = self.pos();
+ if let Some(start_token) = group.start() {
+ self.assert(start_token);
+ }
+ self.delimiters.push((start, group.end()));
+ }
+
+ fn end_group(&mut self) -> Span {
+ let peeked = self.peek();
+
+ let (start, end_token) = self.delimiters.pop().expect("group was not started");
+
+ if end_token != Token::Chain && peeked != None {
+ self.delimiters.push((start, end_token));
+ assert_eq!(peeked, None, "unfinished group");
+ }
+
+ match self.peeked.unwrap() {
+ Some(token) if token.v == end_token => {
+ self.peeked = None;
+ Span::new(start, token.span.end)
+ }
+ _ => {
+ let end = self.pos();
+ if end_token != Token::Chain {
+ error!(
+ @self.feedback, Span::at(end),
+ "expected {}", end_token.name(),
+ );
+ }
+ Span::new(start, end)
+ }
+ }
+ }
+
+ fn skip_ws(&mut self) {
+ while matches!(
+ self.peekv(),
+ Some(Token::Space(_)) |
+ Some(Token::LineComment(_)) |
+ Some(Token::BlockComment(_))
+ ) {
+ self.eat();
+ }
+ }
+
+ fn eatv(&mut self) -> Option<Token<'s>> {
+ self.eat().map(Spanned::value)
+ }
+
+ fn peekv(&mut self) -> Option<Token<'s>> {
+ self.peek().map(Spanned::value)
+ }
+
+ fn assert(&mut self, token: Token<'_>) {
+ assert!(self.check_eat(token).is_some());
+ }
+
+ fn check_eat(&mut self, token: Token<'_>) -> Option<Spanned<Token<'s>>> {
+ if self.check(token) { self.eat() } else { None }
+ }
+
+ /// Checks if the next token is of some kind
+ fn check(&mut self, token: Token<'_>) -> bool {
+ self.peekv() == Some(token)
+ }
+
+ fn with_span<T>(&mut self, v: T) -> Spanned<T> {
+ let span = self.eat().expect("expected token").span;
+ Spanned::new(v, span)
+ }
+
+ fn eof(&mut self) -> bool {
+ self.peek().is_none()
+ }
+
+ fn eat(&mut self) -> Option<Spanned<Token<'s>>> {
+ let token = self.peek()?;
+ self.peeked = None;
+ Some(token)
+ }
+
+ fn peek(&mut self) -> Option<Spanned<Token<'s>>> {
+ let tokens = &mut self.tokens;
+ let token = (*self.peeked.get_or_insert_with(|| tokens.next()))?;
+
+ // Check for unclosed groups.
+ if Group::is_delimiter(token.v) {
+ if self.delimiters.iter().rev().any(|&(_, end)| token.v == end) {
+ return None;
+ }
+ }
+
+ Some(token)
+ }
+
+ fn pos(&self) -> Pos {
+ self.peeked
+ .flatten()
+ .map(|s| s.span.start)
+ .unwrap_or_else(|| self.tokens.pos())
+ }
+}
+
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+enum Group {
+ Paren,
+ Bracket,
+ Brace,
+ Subheader,
+}
+
+impl Group {
+ fn is_delimiter(token: Token<'_>) -> bool {
+ matches!(
+ token,
+ Token::RightParen | Token::RightBracket | Token::RightBrace | Token::Chain
+ )
+ }
+
+ fn start(self) -> Option<Token<'static>> {
+ match self {
+ Self::Paren => Some(Token::LeftParen),
+ Self::Bracket => Some(Token::LeftBracket),
+ Self::Brace => Some(Token::LeftBrace),
+ Self::Subheader => None,
+ }
+ }
+
+ fn end(self) -> Token<'static> {
+ match self {
+ Self::Paren => Token::RightParen,
+ Self::Bracket => Token::RightBracket,
+ Self::Brace => Token::RightBrace,
+ Self::Subheader => Token::Chain,
+ }
+ }
+}
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-15 20:27:43 +0000