diff options
| author | Laurenz <laurmaedje@gmail.com> | 2021-01-06 21:06:48 +0100 |
|---|---|---|
| committer | Laurenz <laurmaedje@gmail.com> | 2021-01-06 21:06:48 +0100 |
| commit | 59d811aeba4491d54d2b0220109fd21a8f838b9b (patch) | |
| tree | e3c22a86592252c157cb268404d5e176d60cac55 /src/parse | |
| parent | 7b4d4d6002a9c3da8fafd912f3c7b2da617f19c0 (diff) | |
Inline literal enum into expression enum π
Diffstat (limited to 'src/parse')
| -rw-r--r-- | src/parse/collection.rs | 4 | ||||
| -rw-r--r-- | src/parse/mod.rs | 36 | ||||
| -rw-r--r-- | src/parse/tests.rs | 37 | ||||
| -rw-r--r-- | src/parse/tokens.rs | 226 |
4 files changed, 136 insertions, 167 deletions
diff --git a/src/parse/collection.rs b/src/parse/collection.rs index 889cfb0f..98d4219f 100644 --- a/src/parse/collection.rs +++ b/src/parse/collection.rs @@ -53,7 +53,7 @@ fn collection<T: Collection>(p: &mut Parser, mut collection: T) -> T { fn argument(p: &mut Parser) -> Option<Argument> { let first = p.span_if(expr)?; if p.eat_if(Token::Colon) { - if let Expr::Lit(Lit::Ident(ident)) = first.v { + if let Expr::Ident(ident) = first.v { let expr = p.span_if(expr)?; let name = ident.with_span(first.span); p.deco(Deco::Name.with_span(name.span)); @@ -131,7 +131,7 @@ impl Collection for State { fn take(expr: &mut Spanned<Expr>) -> Spanned<Expr> { // Replace with anything, it's overwritten anyway. - std::mem::replace(expr, Spanned::zero(Expr::Lit(Lit::Bool(false)))) + std::mem::replace(expr, Spanned::zero(Expr::Bool(false))) } fn diag(p: &mut Parser, arg: Spanned<Argument>) { diff --git a/src/parse/mod.rs b/src/parse/mod.rs index ef4ce46f..150b5ed1 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -49,6 +49,7 @@ fn tree(p: &mut Parser) -> Tree { /// Parse a syntax node. fn node(p: &mut Parser, at_start: bool) -> Option<Node> { let node = match p.peek()? { + Token::Text(text) => Node::Text(text.into()), Token::Space(newlines) => { if newlines < 2 { Node::Space @@ -56,13 +57,20 @@ fn node(p: &mut Parser, at_start: bool) -> Option<Node> { Node::Parbreak } } - Token::Text(text) => Node::Text(text.into()), Token::LineComment(_) | Token::BlockComment(_) => { p.eat(); return None; } + Token::LeftBracket => { + return Some(Node::Expr(Expr::Call(bracket_call(p)))); + } + + Token::LeftBrace => { + return Some(Node::Expr(block_expr(p)?)); + } + Token::Star => Node::Strong, Token::Underscore => Node::Emph, Token::Tilde => Node::Text("\u{00A0}".into()), @@ -77,14 +85,6 @@ fn node(p: &mut Parser, at_start: bool) -> Option<Node> { Token::Raw(t) => Node::Raw(raw(p, t)), Token::UnicodeEscape(t) => Node::Text(unicode_escape(p, t)), - Token::LeftBracket => { - return Some(Node::Expr(Expr::Call(bracket_call(p)))); - } - - Token::LeftBrace => { - return Some(Node::Expr(block_expr(p)?)); - } - _ => { p.diag_unexpected(); return None; @@ -321,19 +321,19 @@ fn value(p: &mut Parser) -> Option<Expr> { let name = ident.with_span(p.peek_span()); return Some(Expr::Call(paren_call(p, name))); } else { - return Some(Expr::Lit(Lit::Ident(ident))); + return Some(Expr::Ident(ident)); } } // Basic values. - Some(Token::None) => Expr::Lit(Lit::None), - Some(Token::Bool(b)) => Expr::Lit(Lit::Bool(b)), - Some(Token::Int(i)) => Expr::Lit(Lit::Int(i)), - Some(Token::Float(f)) => Expr::Lit(Lit::Float(f)), - Some(Token::Length(val, unit)) => Expr::Lit(Lit::Length(val, unit)), - Some(Token::Percent(p)) => Expr::Lit(Lit::Percent(p)), - Some(Token::Hex(hex)) => Expr::Lit(Lit::Color(color(p, hex))), - Some(Token::Str(token)) => Expr::Lit(Lit::Str(str(p, token))), + Some(Token::None) => Expr::None, + Some(Token::Bool(b)) => Expr::Bool(b), + Some(Token::Int(i)) => Expr::Int(i), + Some(Token::Float(f)) => Expr::Float(f), + Some(Token::Length(val, unit)) => Expr::Length(val, unit), + Some(Token::Percent(p)) => Expr::Percent(p), + Some(Token::Hex(hex)) => Expr::Color(color(p, hex)), + Some(Token::Str(token)) => Expr::Str(str(p, token)), // No value. _ => { diff --git a/src/parse/tests.rs b/src/parse/tests.rs index 8de03aff..701d2a73 100644 --- a/src/parse/tests.rs +++ b/src/parse/tests.rs @@ -9,7 +9,8 @@ use crate::geom::Unit; use crate::syntax::*; use BinOp::*; -use Node::{Emph, Linebreak, Parbreak, Space, Strong}; +use Expr::{Bool, Color, Float, Int, Length, Percent}; +use Node::{Emph, Expr as Block, Linebreak, Parbreak, Space, Strong}; use UnOp::*; macro_rules! t { @@ -99,39 +100,11 @@ fn Raw(lang: Option<&str>, lines: &[&str], inline: bool) -> Node { } fn Id(ident: &str) -> Expr { - Expr::Lit(Lit::Ident(Ident(ident.to_string()))) -} - -fn Bool(b: bool) -> Expr { - Expr::Lit(Lit::Bool(b)) -} - -fn Int(int: i64) -> Expr { - Expr::Lit(Lit::Int(int)) -} - -fn Float(float: f64) -> Expr { - Expr::Lit(Lit::Float(float)) -} - -fn Percent(percent: f64) -> Expr { - Expr::Lit(Lit::Percent(percent)) -} - -fn Length(val: f64, unit: Unit) -> Expr { - Expr::Lit(Lit::Length(val, unit)) -} - -fn Color(color: RgbaColor) -> Expr { - Expr::Lit(Lit::Color(color)) + Expr::Ident(Ident(ident.to_string())) } fn Str(string: &str) -> Expr { - Expr::Lit(Lit::Str(string.to_string())) -} - -fn Block(expr: Expr) -> Node { - Node::Expr(expr) + Expr::Str(string.to_string()) } fn Binary( @@ -614,7 +587,7 @@ fn test_parse_values() { t!("{name}" Block(Id("name"))); t!("{ke-bab}" Block(Id("ke-bab"))); t!("{Ξ±}" Block(Id("Ξ±"))); - t!("{none}" Block(Expr::Lit(Lit::None))); + t!("{none}" Block(Expr::None)); t!("{true}" Block(Bool(true))); t!("{false}" Block(Bool(false))); t!("{1.0e-4}" Block(Float(1e-4))); diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs index 74ec47e9..dee92168 100644 --- a/src/parse/tokens.rs +++ b/src/parse/tokens.rs @@ -379,25 +379,21 @@ mod tests { use crate::parse::tests::check; use Option::None; - use Token::{ - BlockComment as BC, Ident as Id, LeftBrace as LB, LeftBracket as L, - LeftParen as LP, LineComment as LC, RightBrace as RB, RightBracket as R, - RightParen as RP, Space as S, Text as T, *, - }; + use Token::{Ident, *}; use Unit::*; - fn Str(string: &str, terminated: bool) -> Token { - Token::Str(TokenStr { string, terminated }) - } - fn Raw(text: &str, backticks: usize, terminated: bool) -> Token { Token::Raw(TokenRaw { text, backticks, terminated }) } - fn UE(sequence: &str, terminated: bool) -> Token { + fn UnicodeEscape(sequence: &str, terminated: bool) -> Token { Token::UnicodeEscape(TokenUnicodeEscape { sequence, terminated }) } + fn Str(string: &str, terminated: bool) -> Token { + Token::Str(TokenStr { string, terminated }) + } + /// Building blocks for suffix testing. /// /// We extend each test case with a collection of different suffixes to make @@ -421,27 +417,27 @@ mod tests { /// - the resulting suffix token const SUFFIXES: &[(char, Option<TokenMode>, &str, Token)] = &[ // Whitespace suffixes. - (' ', None, " ", S(0)), - (' ', None, "\n", S(1)), - (' ', None, "\r", S(1)), - (' ', None, "\r\n", S(1)), + (' ', None, " ", Space(0)), + (' ', None, "\n", Space(1)), + (' ', None, "\r", Space(1)), + (' ', None, "\r\n", Space(1)), // Letter suffixes. - ('a', Some(Body), "hello", T("hello")), - ('a', Some(Body), "π", T("π")), - ('a', Some(Header), "val", Id("val")), - ('a', Some(Header), "Ξ±", Id("Ξ±")), - ('a', Some(Header), "_", Id("_")), + ('a', Some(Body), "hello", Text("hello")), + ('a', Some(Body), "π", Text("π")), + ('a', Some(Header), "val", Ident("val")), + ('a', Some(Header), "Ξ±", Ident("Ξ±")), + ('a', Some(Header), "_", Ident("_")), // Number suffixes. ('1', Some(Header), "2", Int(2)), ('1', Some(Header), ".2", Float(0.2)), // Symbol suffixes. - ('/', None, "[", L), - ('/', None, "//", LC("")), - ('/', None, "/**/", BC("")), + ('/', None, "[", LeftBracket), + ('/', None, "//", LineComment("")), + ('/', None, "/**/", BlockComment("")), ('/', Some(Body), "*", Star), ('/', Some(Body), "_", Underscore), - ('/', Some(Body), r"\\", T(r"\")), - ('/', Some(Header), "(", LP), + ('/', Some(Body), r"\\", Text(r"\")), + ('/', Some(Header), "(", LeftParen), ('/', Some(Header), ":", Colon), ('/', Some(Header), "+", Plus), ('/', Some(Header), "#123", Hex("123")), @@ -487,69 +483,69 @@ mod tests { fn test_tokenize_whitespace() { // Test basic whitespace. t!(Both["a1/"]: "" => ); - t!(Both["a1/"]: " " => S(0)); - t!(Both["a1/"]: " " => S(0)); - t!(Both["a1/"]: "\t" => S(0)); - t!(Both["a1/"]: " \t" => S(0)); - t!(Both["a1/"]: "\u{202F}" => S(0)); + t!(Both["a1/"]: " " => Space(0)); + t!(Both["a1/"]: " " => Space(0)); + t!(Both["a1/"]: "\t" => Space(0)); + t!(Both["a1/"]: " \t" => Space(0)); + t!(Both["a1/"]: "\u{202F}" => Space(0)); // Test newline counting. - t!(Both["a1/"]: "\n" => S(1)); - t!(Both["a1/"]: "\n " => S(1)); - t!(Both["a1/"]: " \n" => S(1)); - t!(Both["a1/"]: " \n " => S(1)); - t!(Both["a1/"]: "\r\n" => S(1)); - t!(Both["a1/"]: " \n\t \n " => S(2)); - t!(Both["a1/"]: "\n\r" => S(2)); - t!(Both["a1/"]: " \r\r\n \x0D" => S(3)); + t!(Both["a1/"]: "\n" => Space(1)); + t!(Both["a1/"]: "\n " => Space(1)); + t!(Both["a1/"]: " \n" => Space(1)); + t!(Both["a1/"]: " \n " => Space(1)); + t!(Both["a1/"]: "\r\n" => Space(1)); + t!(Both["a1/"]: " \n\t \n " => Space(2)); + t!(Both["a1/"]: "\n\r" => Space(2)); + t!(Both["a1/"]: " \r\r\n \x0D" => Space(3)); } #[test] fn test_tokenize_line_comments() { // Test line comment with no trailing newline. - t!(Both[""]: "//" => LC("")); + t!(Both[""]: "//" => LineComment("")); // Test line comment ends at newline. - t!(Both["a1/"]: "//bc\n" => LC("bc"), S(1)); - t!(Both["a1/"]: "// bc \n" => LC(" bc "), S(1)); - t!(Both["a1/"]: "//bc\r\n" => LC("bc"), S(1)); + t!(Both["a1/"]: "//bc\n" => LineComment("bc"), Space(1)); + t!(Both["a1/"]: "// bc \n" => LineComment(" bc "), Space(1)); + t!(Both["a1/"]: "//bc\r\n" => LineComment("bc"), Space(1)); // Test nested line comments. - t!(Both["a1/"]: "//a//b\n" => LC("a//b"), S(1)); + t!(Both["a1/"]: "//a//b\n" => LineComment("a//b"), Space(1)); } #[test] fn test_tokenize_block_comments() { // Test basic block comments. - t!(Both[""]: "/*" => BC("")); - t!(Both: "/**/" => BC("")); - t!(Both: "/*π*/" => BC("π")); - t!(Both: "/*\n*/" => BC("\n")); + t!(Both[""]: "/*" => BlockComment("")); + t!(Both: "/**/" => BlockComment("")); + t!(Both: "/*π*/" => BlockComment("π")); + t!(Both: "/*\n*/" => BlockComment("\n")); // Test depth 1 and 2 nested block comments. - t!(Both: "/* /* */ */" => BC(" /* */ ")); - t!(Both: "/*/*/**/*/*/" => BC("/*/**/*/")); + t!(Both: "/* /* */ */" => BlockComment(" /* */ ")); + t!(Both: "/*/*/**/*/*/" => BlockComment("/*/**/*/")); // Test two nested, one unclosed block comments. - t!(Both[""]: "/*/*/**/*/" => BC("/*/**/*/")); + t!(Both[""]: "/*/*/**/*/" => BlockComment("/*/**/*/")); // Test all combinations of up to two following slashes and stars. - t!(Both[""]: "/*" => BC("")); - t!(Both[""]: "/*/" => BC("/")); - t!(Both[""]: "/**" => BC("*")); - t!(Both[""]: "/*//" => BC("//")); - t!(Both[""]: "/*/*" => BC("/*")); - t!(Both[""]: "/**/" => BC("")); - t!(Both[""]: "/***" => BC("**")); + t!(Both[""]: "/*" => BlockComment("")); + t!(Both[""]: "/*/" => BlockComment("/")); + t!(Both[""]: "/**" => BlockComment("*")); + t!(Both[""]: "/*//" => BlockComment("//")); + t!(Both[""]: "/*/*" => BlockComment("/*")); + t!(Both[""]: "/**/" => BlockComment("")); + t!(Both[""]: "/***" => BlockComment("**")); } #[test] fn test_tokenize_body_tokens() { // Test parentheses. - t!(Body: "[" => L); - t!(Body: "]" => R); - t!(Body: "{" => LB); - t!(Body: "}" => RB); + t!(Body: "[" => LeftBracket); + t!(Body: "]" => RightBracket); + t!(Body: "{" => LeftBrace); + t!(Body: "}" => RightBrace); // Test markup tokens. t!(Body[" a1"]: "*" => Star); @@ -559,7 +555,7 @@ mod tests { t!(Body[" "]: r"\" => Backslash); // Test header symbols. - t!(Body[" /"]: ":,=|/+-" => T(":,=|/+-")); + t!(Body[" /"]: ":,=|/+-" => Text(":,=|/+-")); } #[test] @@ -584,62 +580,62 @@ mod tests { #[test] fn test_tokenize_escape_sequences() { // Test escapable symbols. - t!(Body: r"\\" => T(r"\")); - t!(Body: r"\/" => T("/")); - t!(Body: r"\[" => T("[")); - t!(Body: r"\]" => T("]")); - t!(Body: r"\{" => T("{")); - t!(Body: r"\}" => T("}")); - t!(Body: r"\*" => T("*")); - t!(Body: r"\_" => T("_")); - t!(Body: r"\#" => T("#")); - t!(Body: r"\~" => T("~")); - t!(Body: r"\`" => T("`")); + t!(Body: r"\\" => Text(r"\")); + t!(Body: r"\/" => Text("/")); + t!(Body: r"\[" => Text("[")); + t!(Body: r"\]" => Text("]")); + t!(Body: r"\{" => Text("{")); + t!(Body: r"\}" => Text("}")); + t!(Body: r"\*" => Text("*")); + t!(Body: r"\_" => Text("_")); + t!(Body: r"\#" => Text("#")); + t!(Body: r"\~" => Text("~")); + t!(Body: r"\`" => Text("`")); // Test unescapable symbols. - t!(Body[" /"]: r"\a" => T(r"\"), T("a")); - t!(Body[" /"]: r"\u" => T(r"\"), T("u")); - t!(Body[" /"]: r"\1" => T(r"\"), T("1")); - t!(Body[" /"]: r"\:" => T(r"\"), T(":")); - t!(Body[" /"]: r"\=" => T(r"\"), T("=")); - t!(Body[" /"]: r#"\""# => T(r"\"), T("\"")); + t!(Body[" /"]: r"\a" => Text(r"\"), Text("a")); + t!(Body[" /"]: r"\u" => Text(r"\"), Text("u")); + t!(Body[" /"]: r"\1" => Text(r"\"), Text("1")); + t!(Body[" /"]: r"\:" => Text(r"\"), Text(":")); + t!(Body[" /"]: r"\=" => Text(r"\"), Text("=")); + t!(Body[" /"]: r#"\""# => Text(r"\"), Text("\"")); // Test basic unicode escapes. - t!(Body: r"\u{}" => UE("", true)); - t!(Body: r"\u{2603}" => UE("2603", true)); - t!(Body: r"\u{P}" => UE("P", true)); + t!(Body: r"\u{}" => UnicodeEscape("", true)); + t!(Body: r"\u{2603}" => UnicodeEscape("2603", true)); + t!(Body: r"\u{P}" => UnicodeEscape("P", true)); // Test unclosed unicode escapes. - t!(Body[" /"]: r"\u{" => UE("", false)); - t!(Body[" /"]: r"\u{1" => UE("1", false)); - t!(Body[" /"]: r"\u{26A4" => UE("26A4", false)); - t!(Body[" /"]: r"\u{1Q3P" => UE("1Q3P", false)); - t!(Body: r"\u{1π}" => UE("1", false), T("π"), RB); + t!(Body[" /"]: r"\u{" => UnicodeEscape("", false)); + t!(Body[" /"]: r"\u{1" => UnicodeEscape("1", false)); + t!(Body[" /"]: r"\u{26A4" => UnicodeEscape("26A4", false)); + t!(Body[" /"]: r"\u{1Q3P" => UnicodeEscape("1Q3P", false)); + t!(Body: r"\u{1π}" => UnicodeEscape("1", false), Text("π"), RightBrace); } #[test] fn test_tokenize_text() { // Test basic text. - t!(Body[" /"]: "hello" => T("hello")); - t!(Body[" /"]: "hello-world" => T("hello-world")); + t!(Body[" /"]: "hello" => Text("hello")); + t!(Body[" /"]: "hello-world" => Text("hello-world")); // Test header symbols in text. - t!(Body[" /"]: "a():\"b" => T("a():\"b")); + t!(Body[" /"]: "a():\"b" => Text("a():\"b")); // Test text ends. - t!(Body[""]: "hello " => T("hello"), S(0)); - t!(Body[""]: "hello~" => T("hello"), Tilde); + t!(Body[""]: "hello " => Text("hello"), Space(0)); + t!(Body[""]: "hello~" => Text("hello"), Tilde); } #[test] fn test_tokenize_header_tokens() { // Test parentheses. - t!(Header: "[" => L); - t!(Header: "]" => R); - t!(Header: "{" => LB); - t!(Header: "}" => RB); - t!(Header: "(" => LP); - t!(Header: ")" => RP); + t!(Header: "[" => LeftBracket); + t!(Header: "]" => RightBracket); + t!(Header: "{" => LeftBrace); + t!(Header: "}" => RightBrace); + t!(Header: "(" => LeftParen); + t!(Header: ")" => RightParen); // Test structural tokens. t!(Header: ":" => Colon); @@ -652,10 +648,10 @@ mod tests { // Test hyphen parsed as symbol. t!(Header[" /"]: "-1" => Hyphen, Int(1)); - t!(Header[" /"]: "-a" => Hyphen, Id("a")); + t!(Header[" /"]: "-a" => Hyphen, Ident("a")); t!(Header[" /"]: "--1" => Hyphen, Hyphen, Int(1)); - t!(Header[" /"]: "--_a" => Hyphen, Hyphen, Id("_a")); - t!(Header[" /"]: "a-b" => Id("a-b")); + t!(Header[" /"]: "--_a" => Hyphen, Hyphen, Ident("_a")); + t!(Header[" /"]: "a-b" => Ident("a-b")); // Test some operations. t!(Header[" /"]: "1+3" => Int(1), Plus, Int(3)); @@ -666,33 +662,33 @@ mod tests { #[test] fn test_tokenize_idents() { // Test valid identifiers. - t!(Header[" /"]: "x" => Id("x")); - t!(Header[" /"]: "value" => Id("value")); - t!(Header[" /"]: "__main__" => Id("__main__")); - t!(Header[" /"]: "_snake_case" => Id("_snake_case")); + t!(Header[" /"]: "x" => Ident("x")); + t!(Header[" /"]: "value" => Ident("value")); + t!(Header[" /"]: "__main__" => Ident("__main__")); + t!(Header[" /"]: "_snake_case" => Ident("_snake_case")); // Test non-ascii. - t!(Header[" /"]: "Ξ±" => Id("Ξ±")); - t!(Header[" /"]: "ααααΆα" => Id("ααααΆα")); + t!(Header[" /"]: "Ξ±" => Ident("Ξ±")); + t!(Header[" /"]: "ααααΆα" => Ident("ααααΆα")); // Test hyphen parsed as identifier. - t!(Header[" /"]: "kebab-case" => Id("kebab-case")); - t!(Header[" /"]: "one-10" => Id("one-10")); + t!(Header[" /"]: "kebab-case" => Ident("kebab-case")); + t!(Header[" /"]: "one-10" => Ident("one-10")); } #[test] fn test_tokenize_keywords() { // Test none. t!(Header[" /"]: "none" => Token::None); - t!(Header[" /"]: "None" => Id("None")); + t!(Header[" /"]: "None" => Ident("None")); // Test valid bools. t!(Header[" /"]: "false" => Bool(false)); t!(Header[" /"]: "true" => Bool(true)); // Test invalid bools. - t!(Header[" /"]: "True" => Id("True")); - t!(Header[" /"]: "falser" => Id("falser")); + t!(Header[" /"]: "True" => Ident("True")); + t!(Header[" /"]: "falser" => Ident("falser")); } #[test] @@ -775,15 +771,15 @@ mod tests { fn test_tokenize_invalid() { // Test invalidly closed block comments. t!(Both: "*/" => StarSlash); - t!(Both: "/**/*/" => BC(""), StarSlash); + t!(Both: "/**/*/" => BlockComment(""), StarSlash); // Test invalid expressions. t!(Header: r"\" => Invalid(r"\")); t!(Header: "π" => Invalid("π")); t!(Header: r"\:" => Invalid(r"\"), Colon); - t!(Header: "mealβ" => Id("meal"), Invalid("β")); - t!(Header[" /"]: r"\a" => Invalid(r"\"), Id("a")); - t!(Header[" /"]: ">main" => Invalid(">"), Id("main")); + t!(Header: "mealβ" => Ident("meal"), Invalid("β")); + t!(Header[" /"]: r"\a" => Invalid(r"\"), Ident("a")); + t!(Header[" /"]: ">main" => Invalid(">"), Ident("main")); // Test invalid number suffixes. t!(Header[" /"]: "1foo" => Invalid("1foo")); |
