From 105cda0e698fe86266d706f4e3bacc081e65c2aa Mon Sep 17 00:00:00 2001 From: Laurenz Date: Sun, 10 Jan 2021 21:38:58 +0100 Subject: =?UTF-8?q?Braced=20content=20->=20Bracketed=20templates=20?= =?UTF-8?q?=E2=9C=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/parse/mod.rs | 96 +++++++++++++++++++++++++++-------------------------- src/parse/tests.rs | 87 ++++++++++++++++++++++++------------------------ src/parse/tokens.rs | 46 ++++++++++++------------- 3 files changed, 115 insertions(+), 114 deletions(-) (limited to 'src/parse') diff --git a/src/parse/mod.rs b/src/parse/mod.rs index cce34d8a..ee799159 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -49,32 +49,20 @@ fn tree(p: &mut Parser) -> Tree { /// Parse a syntax node. fn node(p: &mut Parser, at_start: bool) -> Option { let node = match p.peek()? { - Token::Text(text) => Node::Text(text.into()), - Token::Space(newlines) => { - if newlines < 2 { - Node::Space - } else { - Node::Parbreak - } - } - - Token::LineComment(_) | Token::BlockComment(_) => { - p.eat(); - return None; - } - + // Bracket call. Token::LeftBracket => { return Some(Node::Expr(Expr::Call(bracket_call(p)))); } + // Code block. Token::LeftBrace => { - return Some(Node::Expr(block_expr(p)?)); + return Some(Node::Expr(code_block(p)?)); } + // Markup. Token::Star => Node::Strong, Token::Underscore => Node::Emph, Token::Tilde => Node::Text("\u{00A0}".into()), - Token::Backslash => Node::Linebreak, Token::Hash => { if at_start { return Some(Node::Heading(heading(p))); @@ -82,9 +70,24 @@ fn node(p: &mut Parser, at_start: bool) -> Option { Node::Text(p.get(p.peek_span()).into()) } } + Token::Backslash => Node::Linebreak, + Token::Space(newlines) => { + if newlines < 2 { + Node::Space + } else { + Node::Parbreak + } + } + Token::Text(text) => Node::Text(text.into()), Token::Raw(t) => Node::Raw(raw(p, t)), Token::UnicodeEscape(t) => Node::Text(unicode_escape(p, t)), + // Comments. + Token::LineComment(_) | Token::BlockComment(_) => { + p.eat(); + return None; + } + _ => { p.diag_unexpected(); return None; @@ -150,27 +153,6 @@ fn unicode_escape(p: &mut Parser, token: TokenUnicodeEscape) -> String { text } -/// Parse a block expression. -fn block_expr(p: &mut Parser) -> Option { - p.push_mode(TokenMode::Code); - p.start_group(Group::Brace); - let expr = expr(p); - while !p.eof() { - p.diag_unexpected(); - } - p.pop_mode(); - p.end_group(); - expr -} - -/// Parse a parenthesized function call. -fn paren_call(p: &mut Parser, name: Spanned) -> ExprCall { - p.start_group(Group::Paren); - let args = p.span(arguments); - p.end_group(); - ExprCall { name, args } -} - /// Parse a bracketed function call. fn bracket_call(p: &mut Parser) -> ExprCall { p.push_mode(TokenMode::Code); @@ -189,7 +171,7 @@ fn bracket_call(p: &mut Parser) -> ExprCall { p.end_group(); if p.peek() == Some(Token::LeftBracket) { - let body = p.span(|p| Expr::Content(bracket_body(p))); + let body = p.span(|p| Expr::Template(bracket_body(p))); inner.span.expand(body.span); inner.v.args.v.push(Argument::Pos(body)); } @@ -197,7 +179,7 @@ fn bracket_call(p: &mut Parser) -> ExprCall { while let Some(mut top) = outer.pop() { let span = inner.span; let node = inner.map(|c| Node::Expr(Expr::Call(c))); - let expr = Expr::Content(vec![node]).with_span(span); + let expr = Expr::Template(vec![node]).with_span(span); top.v.args.v.push(Argument::Pos(expr)); inner = top; } @@ -236,6 +218,19 @@ fn bracket_body(p: &mut Parser) -> Tree { tree } +/// Parse a code block: `{...}`. +fn code_block(p: &mut Parser) -> Option { + p.push_mode(TokenMode::Code); + p.start_group(Group::Brace); + let expr = expr(p); + while !p.eof() { + p.diag_unexpected(); + } + p.pop_mode(); + p.end_group(); + expr +} + /// Parse an expression: `term (+ term)*`. fn expr(p: &mut Parser) -> Option { binops(p, term, |token| match token { @@ -297,15 +292,14 @@ fn factor(p: &mut Parser) -> Option { /// Parse a value. fn value(p: &mut Parser) -> Option { let expr = match p.peek() { - // Bracketed function call. + // Template. Some(Token::LeftBracket) => { - let node = p.span(|p| Node::Expr(Expr::Call(bracket_call(p)))); - return Some(Expr::Content(vec![node])); + return Some(Expr::Template(template(p))); } - // Content expression. + // Nested block. Some(Token::LeftBrace) => { - return Some(Expr::Content(content(p))); + return code_block(p); } // Dictionary or just a parenthesized expression. @@ -346,16 +340,24 @@ fn value(p: &mut Parser) -> Option { Some(expr) } -// Parse a content value: `{...}`. -fn content(p: &mut Parser) -> Tree { +// Parse a template value: `[...]`. +fn template(p: &mut Parser) -> Tree { p.push_mode(TokenMode::Markup); - p.start_group(Group::Brace); + p.start_group(Group::Bracket); let tree = tree(p); p.pop_mode(); p.end_group(); tree } +/// Parse a parenthesized function call. +fn paren_call(p: &mut Parser, name: Spanned) -> ExprCall { + p.start_group(Group::Paren); + let args = p.span(arguments); + p.end_group(); + ExprCall { name, args } +} + /// Parse an identifier. fn ident(p: &mut Parser) -> Option { p.eat_map(|token| match token { diff --git a/src/parse/tests.rs b/src/parse/tests.rs index 3c57fa1f..a8897c1c 100644 --- a/src/parse/tests.rs +++ b/src/parse/tests.rs @@ -26,7 +26,7 @@ macro_rules! t { $(spans = $spans;)? let Pass { output, feedback } = parse($src); - check($src, Content![@$($node),*], output, spans); + check($src, Template![@$($node),*], output, spans); check( $src, vec![ @@ -158,9 +158,9 @@ macro_rules! Args { }; } -macro_rules! Content { +macro_rules! Template { (@$($node:expr),* $(,)?) => (vec![$(into!($node)),*]); - ($($tts:tt)*) => (Expr::Content(Content![@$($tts)*])); + ($($tts:tt)*) => (Expr::Template(Template![@$($tts)*])); } macro_rules! Call { @@ -227,30 +227,30 @@ fn test_parse_simple_nodes() { fn test_parse_headings() { // Basics with spans. t!("# a" - nodes: [S(0..3, Heading(S(0..1, 0), Content![ + nodes: [S(0..3, Heading(S(0..1, 0), Template![ @S(1..2, Space), S(2..3, Text("a")) ]))], spans: true); // Multiple hashtags. - t!("### three" Heading(2, Content![@Space, Text("three")])); - t!("###### six" Heading(5, Content![@Space, Text("six")])); + t!("### three" Heading(2, Template![@Space, Text("three")])); + t!("###### six" Heading(5, Template![@Space, Text("six")])); // Start of heading. - t!("/**/#" Heading(0, Content![@])); - t!("[f][# ok]" Call!("f", Args![Content![Heading(0, Content![ + t!("/**/#" Heading(0, Template![@])); + t!("[f][# ok]" Call!("f", Args![Template![Heading(0, Template![ @Space, Text("ok") ])]])); // End of heading. - t!("# a\nb" Heading(0, Content![@Space, Text("a")]), Space, Text("b")); + t!("# a\nb" Heading(0, Template![@Space, Text("a")]), Space, Text("b")); // Continued heading. - t!("# a{\n1\n}b" Heading(0, Content![ + t!("# a{\n1\n}b" Heading(0, Template![ @Space, Text("a"), Block(Int(1)), Text("b") ])); - t!("# a[f][\n\n]d" Heading(0, Content![@ - Space, Text("a"), Call!("f", Args![Content![Parbreak]]), Text("d"), + t!("# a[f][\n\n]d" Heading(0, Template![@ + Space, Text("a"), Call!("f", Args![Template![Parbreak]]), Text("d"), ])); // No heading. @@ -260,7 +260,7 @@ fn test_parse_headings() { // Too many hashtags. t!("####### seven" - nodes: [Heading(5, Content![@Space, Text("seven")])], + nodes: [Heading(5, Template![@Space, Text("seven")])], warnings: [S(0..7, "section depth should not exceed 6")]); } @@ -293,13 +293,9 @@ fn test_parse_escape_sequences() { #[test] fn test_parse_groups() { // Test paren group. - t!("{([v 1) + 3}" - nodes: [Block(Binary( - Content![Call!("v", Args![Int(1)])], - Add, - Int(3), - ))], - errors: [S(6..6, "expected closing bracket")]); + t!("{({1) + 3}" + nodes: [Block(Binary(Int(1), Add, Int(3)))], + errors: [S(4..4, "expected closing brace")]); // Test bracket group. t!("[)" @@ -307,19 +303,18 @@ fn test_parse_groups() { errors: [S(1..2, "expected function name, found closing paren"), S(2..2, "expected closing bracket")]); - t!("[v {*]_" - nodes: [Call!("v", Args![Content![Strong]]), Emph], - errors: [S(5..5, "expected closing brace")]); + t!("[v [*]" + nodes: [Call!("v", Args![Template![Strong]])], + errors: [S(6..6, "expected closing bracket")]); // Test brace group. t!("{1 + [}" - nodes: [Block(Binary(Int(1), Add, Content![Call!("")]))], - errors: [S(6..6, "expected function name"), - S(6..6, "expected closing bracket")]); + nodes: [Block(Binary(Int(1), Add, Template![]))], + errors: [S(6..6, "expected closing bracket")]); // Test subheader group. t!("[v (|u )]" - nodes: [Call!("v", Args![Array![], Content![Call!("u")]])], + nodes: [Call!("v", Args![Array![], Template![Call!("u")]])], errors: [S(4..4, "expected closing paren"), S(7..8, "expected expression, found closing paren")]); } @@ -331,7 +326,7 @@ fn test_parse_blocks() { // Function calls. t!("{f()}" Call!("f")); - t!("{[f]}" Block(Content![Call!("f")])); + t!("{[[f]]}" Block(Template![Call!("f")])); // Missing or bad value. t!("{}{1u}" @@ -353,15 +348,15 @@ fn test_parse_bracket_funcs() { t!("[ v ]" Call!("v")); // Body and no body. - t!("[v][[f]]" Call!("v", Args![Content![Call!("f")]])); - t!("[v][v][v]" Call!("v", Args![Content![Text("v")]]), Call!("v")); + t!("[v][[f]]" Call!("v", Args![Template![Call!("f")]])); + t!("[v][v][v]" Call!("v", Args![Template![Text("v")]]), Call!("v")); t!("[v] [f]" Call!("v"), Space, Call!("f")); // Spans. t!("[v 1][📐]" nodes: [S(0..11, Call!(S(1..2, "v"), S(3..4, Args![ S(3..4, Int(1)), - S(5..11, Content![S(6..10, Text("📐"))]), + S(5..11, Template![S(6..10, Text("📐"))]), ])))], spans: true); @@ -389,35 +384,35 @@ fn test_parse_bracket_funcs() { // Raw in body eats closing bracket. t!("[v][`a]`" - nodes: [Call!("v", Args![Content![Raw(None, &["a]"], true)]])], + nodes: [Call!("v", Args![Template![Raw(None, &["a]"], true)]])], errors: [S(8..8, "expected closing bracket")]); } #[test] fn test_parse_chaining() { // Basic. - t!("[a | b]" Call!("a", Args![Content![Call!("b")]])); - t!("[a|b|c]" Call!("a", Args![Content![ - Call!("b", Args![Content![Call!("c")]]) + t!("[a | b]" Call!("a", Args![Template![Call!("b")]])); + t!("[a|b|c]" Call!("a", Args![Template![ + Call!("b", Args![Template![Call!("c")]]) ]])); // With body and spans. t!("[a|b][💕]" nodes: [S(0..11, Call!(S(1..2, "a"), S(2..2, Args![ - S(3..11, Content![S(3..11, Call!(S(3..4, "b"), S(4..4, Args![ - S(5..11, Content![S(6..10, Text("💕"))]) + S(3..11, Template![S(3..11, Call!(S(3..4, "b"), S(4..4, Args![ + S(5..11, Template![S(6..10, Text("💕"))]) ])))]) ])))], spans: true); // No name in second subheader. t!("[a 1|]" - nodes: [Call!("a", Args![Int(1), Content![Call!("")]])], + nodes: [Call!("a", Args![Int(1), Template![Call!("")]])], errors: [S(5..5, "expected function name")]); // No name in first subheader. t!("[|a true]" - nodes: [Call!("", Args![Content![Call!("a", Args![Bool(true)])]])], + nodes: [Call!("", Args![Template![Call!("a", Args![Bool(true)])]])], errors: [S(1..1, "expected function name")]); } @@ -534,12 +529,12 @@ fn test_parse_dictionaries() { errors: [S(8..9, "expected named pair, found expression")]); // Dictionary marker followed by more stuff. - t!("{(:1 b:2, true::)}" - nodes: [Block(Dict!["b" => Int(2)])], + t!("{(:1 b:[], true::)}" + nodes: [Block(Dict!["b" => Template![]])], errors: [S(3..4, "expected named pair, found expression"), S(4..4, "expected comma"), - S(10..14, "name must be identifier"), - S(15..16, "expected expression, found colon")]); + S(11..15, "name must be identifier"), + S(16..17, "expected expression, found colon")]); } #[test] @@ -615,7 +610,11 @@ fn test_parse_values() { errors: [S(1..4, "invalid color")]); // Content. - t!("{{*Hi*}}" Block(Content![Strong, Text("Hi"), Strong])); + t!("{[*Hi*]}" Block(Template![Strong, Text("Hi"), Strong])); + + // Nested blocks. + t!("{{1}}" Block(Int(1))); + t!("{{{1+2}}}" Block(Binary(Int(1), Add, Int(2)))); // Invalid tokens. t!("{1u}" diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs index 28ca6e48..d16cf2ce 100644 --- a/src/parse/tokens.rs +++ b/src/parse/tokens.rs @@ -562,7 +562,7 @@ mod tests { } #[test] - fn test_tokenize_body_symbols() { + fn test_tokenize_markup_symbols() { // Test markup tokens. t!(Markup[" a1"]: "*" => Star); t!(Markup: "_" => Underscore); @@ -572,7 +572,7 @@ mod tests { } #[test] - fn test_tokenize_header_symbols() { + fn test_tokenize_code_symbols() { // Test all symbols. t!(Code: "," => Comma); t!(Code: ":" => Colon); @@ -653,6 +653,27 @@ mod tests { t!(Code[" /"]: "falser" => Ident("falser")); } + #[test] + fn test_tokenize_whitespace() { + // Test basic whitespace. + t!(Both["a1/"]: "" => ); + t!(Both["a1/"]: " " => Space(0)); + t!(Both["a1/"]: " " => Space(0)); + t!(Both["a1/"]: "\t" => Space(0)); + t!(Both["a1/"]: " \t" => Space(0)); + t!(Both["a1/"]: "\u{202F}" => Space(0)); + + // Test newline counting. + t!(Both["a1/"]: "\n" => Space(1)); + t!(Both["a1/"]: "\n " => Space(1)); + t!(Both["a1/"]: " \n" => Space(1)); + t!(Both["a1/"]: " \n " => Space(1)); + t!(Both["a1/"]: "\r\n" => Space(1)); + t!(Both["a1/"]: " \n\t \n " => Space(2)); + t!(Both["a1/"]: "\n\r" => Space(2)); + t!(Both["a1/"]: " \r\r\n \x0D" => Space(3)); + } + #[test] fn test_tokenize_text() { // Test basic text. @@ -738,27 +759,6 @@ mod tests { t!(Markup: r"\u{1🏕}" => UnicodeEscape("1", false), Text("🏕"), RightBrace); } - #[test] - fn test_tokenize_whitespace() { - // Test basic whitespace. - t!(Both["a1/"]: "" => ); - t!(Both["a1/"]: " " => Space(0)); - t!(Both["a1/"]: " " => Space(0)); - t!(Both["a1/"]: "\t" => Space(0)); - t!(Both["a1/"]: " \t" => Space(0)); - t!(Both["a1/"]: "\u{202F}" => Space(0)); - - // Test newline counting. - t!(Both["a1/"]: "\n" => Space(1)); - t!(Both["a1/"]: "\n " => Space(1)); - t!(Both["a1/"]: " \n" => Space(1)); - t!(Both["a1/"]: " \n " => Space(1)); - t!(Both["a1/"]: "\r\n" => Space(1)); - t!(Both["a1/"]: " \n\t \n " => Space(2)); - t!(Both["a1/"]: "\n\r" => Space(2)); - t!(Both["a1/"]: " \r\r\n \x0D" => Space(3)); - } - #[test] fn test_tokenize_idents() { // Test valid identifiers. -- cgit v1.2.3