Re-engineer tokenization 🚿

3 files changed, 81 insertions, 80 deletions

diff --git a/tests/parse.rs b/tests/parse.rs
index 953cc959..e00b05d8 100644
--- a/tests/parse.rs
+++ b/tests/parse.rs
@@ -1,9 +1,26 @@
+#![allow(unused_imports)]
+#![allow(non_snake_case)]
+
+use typstc::size::Size;
 use typstc::syntax::*;
 use Token::{
-    Space as S, Newline as N, LeftBracket as LB,
-    RightBracket as RB, Text as T, *
+    Whitespace as W,
+    LineComment as LC, BlockComment as BC, StarSlash as SS,
+    LeftBracket as LB, RightBracket as RB,
+    LeftParen as LP, RightParen as RP,
+    LeftBrace as LBR, RightBrace as RBR,
+    Colon as CL, Comma as CM, Equals as EQ, Expr as E,
+    Star as ST, Underscore as U, Backtick as B, Text as T,
 };
 
+use Expression as Expr;
+fn ID(ident: &str) -> Token { E(Expr::Ident(Ident::new(ident.to_string()).unwrap())) }
+fn STR(ident: &str) -> Token { E(Expr::Str(ident.to_string())) }
+fn SIZE(size: Size) -> Token<'static> { E(Expr::Size(size)) }
+fn NUM(num: f64) -> Token<'static> { E(Expr::Num(num)) }
+fn BOOL(b: bool) -> Token<'static> { E(Expr::Bool(b)) }
+
+
 /// Parses the test syntax.
 macro_rules! tokens {
     ($($src:expr =>($line:expr)=> $tokens:expr)*) => ({
diff --git a/tests/parsing/base.rs b/tests/parsing/base.rs
deleted file mode 100644
index ad7d87c0..00000000
--- a/tests/parsing/base.rs
+++ /dev/null
@@ -1,78 +0,0 @@
-// Spaces, Newlines, Brackets.
-""                => []
-" "               => [S]
-"    "            => [S]
-"\t"              => [S]
-"  \t"            => [S]
-"\n"              => [N]
-"\n "             => [N, S]
-"  \n"            => [S, N]
-"  \n   "         => [S, N, S]
-"["               => [LB]
-"]"               => [RB]
-
-// Header only tokens.
-"[:]"             => [LB, Colon, RB]
-"[=]"             => [LB, Equals, RB]
-"[,]"             => [LB, Comma, RB]
-":"               => [T(":")]
-"="               => [T("=")]
-","               => [T(",")]
-r#"["hi"]"#       => [LB, Quoted("hi"), RB]
-r#""hi""#         => [T(r#""hi""#)]
-
-// Body only tokens.
-"_"               => [Underscore]
-"*"               => [Star]
-"`"               => [Backtick]
-"[_]"             => [LB, T("_"), RB]
-"[*]"             => [LB, T("*"), RB]
-"[`]"             => [LB, T("`"), RB]
-
-// Comments.
-"//line"          => [LineComment("line")]
-"/*block*/"       => [BlockComment("block")]
-"*/"              => [StarSlash]
-
-// Plain text.
-"A"               => [T("A")]
-"Hello"           => [T("Hello")]
-"Hello-World"     => [T("Hello-World")]
-r#"A"B"#          => [T(r#"A"B"#)]
-"🌍"              => [T("🌍")]
-
-// Escapes.
-r"\["             => [T("[")]
-r"\]"             => [T("]")]
-r"\\"             => [T(r"\")]
-r"[\[]"           => [LB, T("["), RB]
-r"[\]]"           => [LB, T("]"), RB]
-r"[\\]"           => [LB, T(r"\"), RB]
-r"\:"             => [T(":")]
-r"\="             => [T("=")]
-r"\/"             => [T("/")]
-r"[\:]"           => [LB, T(":"), RB]
-r"[\=]"           => [LB, T("="), RB]
-r"[\,]"           => [LB, T(","), RB]
-r"\*"             => [T("*")]
-r"\_"             => [T("_")]
-r"\`"             => [T("`")]
-r"[\*]"           => [LB, T("*"), RB]
-r"[\_]"           => [LB, T("_"), RB]
-r"[\`]"           => [LB, T("`"), RB]
-
-// Whitespace.
-"Hello World"     => [T("Hello"), S, T("World")]
-"Hello  World"    => [T("Hello"), S, T("World")]
-"Hello \t World"  => [T("Hello"), S, T("World")]
-
-// Newline.
-"First\n"         => [T("First"), N]
-"First \n"        => [T("First"), S, N]
-"First\n "        => [T("First"), N, S]
-"First \n "       => [T("First"), S, N, S]
-"First\nSecond"   => [T("First"), N, T("Second")]
-"First\r\nSecond" => [T("First"), N, T("Second")]
-"First \nSecond"  => [T("First"), S, N, T("Second")]
-"First\n Second"  => [T("First"), N, S, T("Second")]
-"First \n Second" => [T("First"), S, N, S, T("Second")]
diff --git a/tests/parsing/tokens.rs b/tests/parsing/tokens.rs
new file mode 100644
index 00000000..4f5474bb
--- /dev/null
+++ b/tests/parsing/tokens.rs
@@ -0,0 +1,62 @@
+// Whitespace.
+""             => []
+" "            => [W(0)]
+"    "         => [W(0)]
+"\t"           => [W(0)]
+"  \t"         => [W(0)]
+"\n"           => [W(1)]
+"\n "          => [W(1)]
+"  \n"         => [W(1)]
+"  \n   "      => [W(1)]
+"  \n\t \n  "  => [W(2)]
+"\r\n"         => [W(1)]
+" \r\r\n \x0D" => [W(3)]
+"\n\r"         => [W(2)]
+
+// Comments.
+"a // bc\n "        => [T("a"), W(0), LC(" bc"), W(1)]
+"a //a//b\n "       => [T("a"), W(0), LC("a//b"), W(1)]
+"a //a//b\r\n"      => [T("a"), W(0), LC("a//b"), W(1)]
+"a //a//b\n\nhello" => [T("a"), W(0), LC("a//b"), W(2), T("hello")]
+"/**/"              => [BC("")]
+"_/*_/*a*/*/"       => [U, BC("_/*a*/")]
+"/*/*/"             => [BC("/*/")]
+"abc*/"             => [T("abc"), SS]
+
+// Header only tokens.
+"["                   => [LB]
+"]"                   => [RB]
+"[(){}:=,]"           => [LB, LP, RP, LBR, RBR, CL, EQ, CM, RB]
+"[a:b]"               => [LB, ID("a"), CL, ID("b"), RB]
+"[🌓, 🌍,]"          => [LB, T("🌓"), CM, W(0), T("🌍"), CM, RB]
+"[=]"                 => [LB, EQ, RB]
+"[,]"                 => [LB, CM, RB]
+"a: b"                => [T("a"), T(":"), W(0), T("b")]
+"c=d, "               => [T("c"), T("=d"), T(","), W(0)]
+r#"["hello\"world"]"# => [LB, STR(r#"hello\"world"#), RB]
+r#"["hi", 12pt]"#     => [LB, STR("hi"), CM, W(0), SIZE(Size::pt(12.0)), RB]
+"\"hi\""              => [T("\"hi"), T("\"")]
+"[a: true, x=1]"      => [LB, ID("a"), CL, W(0), BOOL(true), CM, W(0),
+                          ID("x"), EQ, NUM(1.0), RB]
+"[120%]"              => [LB, NUM(1.2), RB]
+
+// Body only tokens.
+"_*`"           => [U, ST, B]
+"[_*`]"         => [LB, T("_"), T("*"), T("`"), RB]
+"hi_you_ there" => [T("hi"), U, T("you"), U, W(0), T("there")]
+
+// Escapes.
+r"\["   => [T("[")]
+r"\]"   => [T("]")]
+r"\\"   => [T(r"\")]
+r"\/"   => [T("/")]
+r"\*"   => [T("*")]
+r"\_"   => [T("_")]
+r"\`"   => [T("`")]
+
+// Unescapable special symbols.
+r"\:"   => [T(r"\"), T(":")]
+r"\="   => [T(r"\"), T("=")]
+r"[\:]" => [LB, T(r"\"), CL, RB]
+r"[\=]" => [LB, T(r"\"), EQ, RB]
+r"[\,]" => [LB, T(r"\"), CM, RB]