diff options
| author | Laurenz <laurmaedje@gmail.com> | 2019-10-13 13:09:03 +0200 |
|---|---|---|
| committer | Laurenz <laurmaedje@gmail.com> | 2019-10-13 13:09:39 +0200 |
| commit | 5c04185892947969005ffcf6412d7190dafb3a79 (patch) | |
| tree | feef0dcb373261e44a31242584635f11c6cc1701 /src/parsing | |
| parent | e2d17aa9d9491b339e6200c97b52f7ade51fa1d8 (diff) | |
Reformat some things π§Ί
Diffstat (limited to 'src/parsing')
| -rw-r--r-- | src/parsing/mod.rs | 30 | ||||
| -rw-r--r-- | src/parsing/tokens.rs | 56 |
2 files changed, 53 insertions, 33 deletions
diff --git a/src/parsing/mod.rs b/src/parsing/mod.rs index 7d2c3d21..fa71270e 100644 --- a/src/parsing/mod.rs +++ b/src/parsing/mod.rs @@ -1,6 +1,7 @@ -//! Parsing of source code into token streams an syntax trees. +//! Parsing of source code into token streams and syntax trees. use std::collections::HashMap; + use unicode_xid::UnicodeXID; use crate::func::{Function, Scope}; @@ -8,6 +9,7 @@ use crate::syntax::*; use crate::size::Size; mod tokens; + pub use tokens::{tokenize, Tokens}; @@ -539,6 +541,7 @@ mod tests { /// Parse the basic cases. #[test] + #[rustfmt::skip] fn parse_base() { test("", tree! []); test("Hello World!", tree! [ T("Hello"), S, T("World!") ]); @@ -546,6 +549,7 @@ mod tests { /// Test whether newlines generate the correct whitespace. #[test] + #[rustfmt::skip] fn parse_newlines_whitespace() { test("Hello\nWorld", tree! [ T("Hello"), S, T("World") ]); test("Hello \n World", tree! [ T("Hello"), S, T("World") ]); @@ -558,6 +562,7 @@ mod tests { /// Parse things dealing with functions. #[test] + #[rustfmt::skip] fn parse_functions() { let mut scope = Scope::new(); scope.add::<BodylessFn>("test"); @@ -590,6 +595,7 @@ mod tests { /// Parse functions with arguments. #[test] + #[rustfmt::skip] fn parse_function_args() { use Expression::{Number as N, Size as Z, Bool as B}; @@ -599,14 +605,16 @@ mod tests { fn I(string: &str) -> Expression { Expression::Ident(string.to_owned()) } fn func(name: &str, args: Vec<Expression>) -> SyntaxTree { - tree! [ F(FuncCall { - header: FuncHeader { - name: name.to_string(), - args, - kwargs: HashMap::new(), - }, - body: Box::new(BodylessFn) - }) ] + tree! [ + F(FuncCall { + header: FuncHeader { + name: name.to_string(), + args, + kwargs: HashMap::new(), + }, + body: Box::new(BodylessFn) + }) + ] } let mut scope = Scope::new(); @@ -626,6 +634,7 @@ mod tests { /// Parse comments (line and block). #[test] + #[rustfmt::skip] fn parse_comments() { let mut scope = Scope::new(); scope.add::<BodylessFn>("test"); @@ -641,6 +650,7 @@ mod tests { /// Test if escaped, but unbalanced parens are correctly parsed. #[test] + #[rustfmt::skip] fn parse_unbalanced_body_parens() { let mut scope = Scope::new(); scope.add::<TreeFn>("code"); @@ -667,6 +677,7 @@ mod tests { /// Tests if the parser handles non-ASCII stuff correctly. #[test] + #[rustfmt::skip] fn parse_unicode() { let mut scope = Scope::new(); scope.add::<BodylessFn>("func"); @@ -689,6 +700,7 @@ mod tests { /// Tests whether errors get reported correctly. #[test] + #[rustfmt::skip] fn parse_errors() { let mut scope = Scope::new(); scope.add::<TreeFn>("hello"); diff --git a/src/parsing/tokens.rs b/src/parsing/tokens.rs index 74b9c11c..869e3821 100644 --- a/src/parsing/tokens.rs +++ b/src/parsing/tokens.rs @@ -1,7 +1,7 @@ -//! Tokenization of text. - use std::str::CharIndices; + use smallvec::SmallVec; + use crate::syntax::*; @@ -350,6 +350,7 @@ mod tests { /// Tokenizes the basic building blocks. #[test] + #[rustfmt::skip] fn tokenize_base() { test("", vec![]); test("Hallo", vec![T("Hallo")]); @@ -363,17 +364,20 @@ mod tests { /// This test looks if LF- and CRLF-style newlines get both identified correctly. #[test] + #[rustfmt::skip] fn tokenize_whitespace_newlines() { test(" \t", vec![S]); - test("First line\r\nSecond line\nThird line\n", - vec![T("First"), S, T("line"), N, T("Second"), S, T("line"), N, - T("Third"), S, T("line"), N]); + test("First line\r\nSecond line\nThird line\n", vec![ + T("First"), S, T("line"), N, T("Second"), S, T("line"), N, + T("Third"), S, T("line"), N + ]); test("Hello \n ", vec![T("Hello"), S, N, S]); test("Dense\nTimes", vec![T("Dense"), N, T("Times")]); } /// Tests if escaping with backslash works as it should. #[test] + #[rustfmt::skip] fn tokenize_escape() { test(r"\[", vec![T("[")]); test(r"\]", vec![T("]")]); @@ -386,12 +390,14 @@ mod tests { /// Tests if escaped strings work. #[test] + #[rustfmt::skip] fn tokenize_quoted() { test(r#"[align: "hello\"world"]"#, vec![L, T("align"), C, S, Q(r#"hello\"world"#), R]); } /// Tokenizes some more realistic examples. #[test] + #[rustfmt::skip] fn tokenize_examples() { test(r" [function][ @@ -418,29 +424,30 @@ mod tests { /// This test checks whether the colon and equals symbols get parsed correctly depending on the /// context: Either in a function header or in a body. #[test] + #[rustfmt::skip] fn tokenize_symbols_context() { - test("[func: key=value][Answer: 7]", - vec![L, T("func"), C, S, T("key"), E, T("value"), R, L, - T("Answer:"), S, T("7"), R]); - test("[[n: k=v]:x][:[=]]:=", - vec![L, L, T("n"), C, S, T("k"), E, T("v"), R, C, T("x"), R, - L, T(":"), L, E, R, R, T(":=")]); - test("[hi: k=[func][body] v=1][hello]", - vec![L, T("hi"), C, S, T("k"), E, L, T("func"), R, L, T("body"), R, S, - T("v"), E, T("1"), R, L, T("hello"), R]); - test("[func: __key__=value]", - vec![L, T("func"), C, S, T("__key__"), E, T("value"), R]); - test("The /*[*/ answer: 7.", - vec![T("The"), S, BC("["), S, T("answer:"), S, T("7.")]); + test("[func: key=value][Answer: 7]", vec![ + L, T("func"), C, S, T("key"), E, T("value"), R, L, + T("Answer:"), S, T("7"), R + ]); + test("[[n: k=v]:x][:[=]]:=", vec![ + L, L, T("n"), C, S, T("k"), E, T("v"), R, C, T("x"), R, + L, T(":"), L, E, R, R, T(":=") + ]); + test("[hi: k=[func][body] v=1][hello]", vec![ + L, T("hi"), C, S, T("k"), E, L, T("func"), R, L, T("body"), R, S, + T("v"), E, T("1"), R, L, T("hello"), R + ]); + test("[func: __key__=value]", vec![L, T("func"), C, S, T("__key__"), E, T("value"), R]); + test("The /*[*/ answer: 7.", vec![T("The"), S, BC("["), S, T("answer:"), S, T("7.")]); } /// Test if block and line comments get tokenized as expected. #[test] + #[rustfmt::skip] fn tokenize_comments() { - test("These // Line comments.", - vec![T("These"), S, LC(" Line comments.")]); - test("This /* is */ a comment.", - vec![T("This"), S, BC(" is "), S, T("a"), S, T("comment.")]); + test("These // Line comments.", vec![T("These"), S, LC(" Line comments.")]); + test("This /* is */ a comment.", vec![T("This"), S, BC(" is "), S, T("a"), S, T("comment.")]); test("[Head/*of*/][Body]", vec![L, T("Head"), BC("of"), R, L, T("Body"), R]); test("/* Hey */ */", vec![BC(" Hey "), S, SS]); test("Hey\n// Yoo /*\n*/", vec![T("Hey"), N, LC(" Yoo /*"), N, SS]); @@ -449,6 +456,7 @@ mod tests { /// This test has a special look at the underscore syntax. #[test] + #[rustfmt::skip] fn tokenize_underscores() { test("he_llo_world_ __ Now this_ is_ special!", vec![T("he"), TU, T("llo"), TU, T("world"), TU, S, TU, TU, S, T("Now"), S, @@ -457,9 +465,9 @@ mod tests { /// This test is for checking if non-ASCII characters get parsed correctly. #[test] + #[rustfmt::skip] fn tokenize_unicode() { - test("[document][Hello π!]", - vec![L, T("document"), R, L, T("Hello"), S, T("π!"), R]); + test("[document][Hello π!]", vec![L, T("document"), R, L, T("Hello"), S, T("π!"), R]); test("[f]βΊ.", vec![L, T("f"), R, T("βΊ.")]); } } |
