Decoupled function parser 🔗 [WIP]

3 files changed, 96 insertions, 87 deletions

diff --git a/tests/layouter/test.typ b/tests/layouter/test.typ
deleted file mode 100644
index b7fd334d..00000000
--- a/tests/layouter/test.typ
+++ /dev/null
@@ -1 +0,0 @@
-[box][hi]
diff --git a/tests/parser/tokens.rs b/tests/parser/tokens.rs
index fb48b32e..66e44ac5 100644
--- a/tests/parser/tokens.rs
+++ b/tests/parser/tokens.rs
@@ -1,80 +1,77 @@
 // Whitespace.
 t ""             => []
-t " "            => [W(0)]
-t "    "         => [W(0)]
-t "\t"           => [W(0)]
-t "  \t"         => [W(0)]
-t "\n"           => [W(1)]
-t "\n "          => [W(1)]
-t "  \n"         => [W(1)]
-t "  \n   "      => [W(1)]
-t "  \n\t \n  "  => [W(2)]
-t "\r\n"         => [W(1)]
-t " \r\r\n \x0D" => [W(3)]
-t "\n\r"         => [W(2)]
+t " "            => [S(0)]
+t "    "         => [S(0)]
+t "\t"           => [S(0)]
+t "  \t"         => [S(0)]
+t "\n"           => [S(1)]
+t "\n "          => [S(1)]
+t "  \n"         => [S(1)]
+t "  \n   "      => [S(1)]
+t "  \n\t \n  "  => [S(2)]
+t "\r\n"         => [S(1)]
+t " \r\r\n \x0D" => [S(3)]
+t "\n\r"         => [S(2)]
 
 // Comments.
-t "a // bc\n "        => [T("a"), W(0), LC(" bc"), W(1)]
-t "a //a//b\n "       => [T("a"), W(0), LC("a//b"), W(1)]
-t "a //a//b\r\n"      => [T("a"), W(0), LC("a//b"), W(1)]
-t "a //a//b\n\nhello" => [T("a"), W(0), LC("a//b"), W(2), T("hello")]
+t "a // bc\n "        => [T("a"), S(0), LC(" bc"),  S(1)]
+t "a //a//b\n "       => [T("a"), S(0), LC("a//b"), S(1)]
+t "a //a//b\r\n"      => [T("a"), S(0), LC("a//b"), S(1)]
+t "a //a//b\n\nhello" => [T("a"), S(0), LC("a//b"), S(2), T("hello")]
 t "/**/"              => [BC("")]
-t "_/*_/*a*/*/"       => [U, BC("_/*a*/")]
+t "_/*_/*a*/*/"       => [Underscore, BC("_/*a*/")]
 t "/*/*/"             => [BC("/*/")]
-t "abc*/"             => [T("abc"), SS]
+t "abc*/"             => [T("abc"), Invalid("*/")]
 
 // Header only tokens.
-t "["                   => [LB]
-t "]"                   => [RB]
-t "[(){}:=,]"           => [LB, LP, RP, LBR, RBR, CL, EQ, CM, RB]
-t "[a:b]"               => [LB, ID("a"), CL, ID("b"), RB]
-t "[🌓, 🌍,]"           => [LB, T("🌓"), CM, W(0), T("🌍"), CM, RB]
-t "[=]"                 => [LB, EQ, RB]
-t "[,]"                 => [LB, CM, RB]
-t "a: b"                => [T("a"), T(":"), W(0), T("b")]
-t "c=d, "               => [T("c"), T("=d"), T(","), W(0)]
-t r#"["hello\"world"]"# => [LB, STR(r#"hello\"world"#), RB]
-t r#"["hi", 12pt]"#     => [LB, STR("hi"), CM, W(0), SIZE(Size::pt(12.0)), RB]
-t "\"hi\""              => [T("\"hi"), T("\"")]
-t "[a: true, x=1]"      => [LB, ID("a"), CL, W(0), BOOL(true), CM, W(0),
-                          ID("x"), EQ, NUM(1.0), RB]
-t "[120%]"              => [LB, NUM(1.2), RB]
+th "["                 => [Func("", None, false)]
+th "]"                 => [Invalid("]")]
+th "(){}:=,"           => [LP, RP, LB, RB, Colon, Equals, Comma]
+th "a:b"               => [Id("a"), Colon, Id("b")]
+th "="                 => [Equals]
+th ","                 => [Comma]
+th r#""hello\"world""# => [Str(r#"hello\"world"#)]
+th r#""hi", 12pt"#     => [Str("hi"), Comma, S(0), Size(12.0)]
+th "\"hi\""            => [T("\"hi"), T("\"")]
+th "a: true, x=1"      => [Id("a"), Colon, S(0), Bool(true), Comma, S(0),
+                           Id("x"), Equals, Num(1.0)]
+th "120%"              => [Num(1.2)]
+th "🌓, 🌍,"           => [T("🌓"), Comma, S(0), T("🌍"), Comma]
+tb "a: b"              => [T("a"), T(":"), S(0), T("b")]
+tb "c=d, "             => [T("c"), T("=d"), T(","), S(0)]
 
 // Body only tokens.
-t "_*`"           => [U, S, B]
-t "[func]*bold*"  => [LB, ID("func"), RB, S, T("bold"), S]
-t "[_*`]"         => [LB, T("_"), T("*"), T("`"), RB]
-t "hi_you_ there" => [T("hi"), U, T("you"), U, W(0), T("there")]
+tb "_*`"           => [Underscore, Star, Backtick]
+tb "[func]*bold*"  => [Func("func", None, true), Star, T("bold"), Star]
+tb "hi_you_ there" => [T("hi"), Underscore, T("you"), Underscore, S(0), T("there")]
+th "_*`"           => [Invalid("_"), Invalid("*"), Invalid("`")]
 
 // Nested functions.
-t "[f: [=][*]]"    => [LB, ID("f"), CL, W(0), LB, EQ, RB, LB, S, RB, RB]
-t "[_][[,],],"     => [LB, T("_"), RB, LB, LB, CM, RB, T(","), RB, T(",")]
-t "[=][=][=]"      => [LB, EQ, RB, LB, T("="), RB, LB, EQ, RB]
-t "[=][[=][=][=]]" => [LB, EQ, RB, LB, LB, EQ, RB, LB, T("="), RB, LB, EQ, RB, RB]
+tb "[f: [=][*]]"    => [Func("f: [=][*]", None, true)]
+tb "[_][[,],],"     => [Func("_", Some("[,],"), true), T(",")]
+tb "[=][=][=]"      => [Func("=", Some("="), true), Func("=", None, true)]
+tb "[=][[=][=][=]]" => [Func("=", Some("[=][=][=]")), true]
 
 // Escapes.
-t r"\["   => [T("[")]
-t r"\]"   => [T("]")]
-t r"\\"   => [T(r"\")]
-t r"\/"   => [T("/")]
-t r"\*"   => [T("*")]
-t r"\_"   => [T("_")]
-t r"\`"   => [T("`")]
+tb r"\[" => [T("[")]
+tb r"\]" => [T("]")]
+tb r"\\" => [T(r"\")]
+tb r"\/" => [T("/")]
+tb r"\*" => [T("*")]
+tb r"\_" => [T("_")]
+tb r"\`" => [T("`")]
 
 // Unescapable special symbols.
-t r"\:"   => [T(r"\"), T(":")]
-t r"\="   => [T(r"\"), T("=")]
-t r"[\:]" => [LB, T(r"\"), CL, RB]
-t r"[\=]" => [LB, T(r"\"), EQ, RB]
-t r"[\,]" => [LB, T(r"\"), CM, RB]
+th r"\:" => [T(r"\"), T(":")]
+th r"\=" => [T(r"\"), T("=")]
+th r"\:" => [T(r"\"), Colon]
+th r"\=" => [T(r"\"), Equals]
+th r"\," => [T(r"\"), Comma]
 
-// Spans
-ts "hello"           => [(0:0, 0:5, T("hello"))]
-ts "ab\r\nc"         => [(0:0, 0:2, T("ab")), (0:2, 1:0, W(1)), (1:0, 1:1, T("c"))]
-ts "[a=10]"          => [(0:0, 0:1, LB), (0:1, 0:2, ID("a")), (0:2, 0:3, EQ),
-                         (0:3, 0:5, NUM(10.0)), (0:5, 0:6, RB)]
-ts r#"[x = "(1)"]*"# => [(0:0, 0:1, LB), (0:1, 0:2, ID("x")), (0:2, 0:3, W(0)),
-                         (0:3, 0:4, EQ), (0:4, 0:5, W(0)), (0:5, 0:10, STR("(1)")),
-                         (0:10, 0:11, RB), (0:11, 0:12, S)]
-ts "// ab\r\n\nf"    => [(0:0, 0:5, LC(" ab")), (0:5, 2:0, W(2)), (2:0, 2:1, T("f"))]
-ts "/*b*/_"          => [(0:0, 0:5, BC("b")), (0:5, 0:6, U)]
+// Spans.
+tbs "hello"          => [(0:0, 0:5, T("hello"))]
+tbs "ab\r\nc"        => [(0:0, 0:2, T("ab")), (0:2, 1:0, S(1)), (1:0, 1:1, T("c"))]
+tbs "[x = \"(1)\"]*" => [(0:0, 0:11, Func("x = \"(1)\"", None, true)), (0:11, 0:12, Star)]
+tbs "// ab\r\n\nf"   => [(0:0, 0:5, LC(" ab")), (0:5, 2:0, S(2)), (2:0, 2:1, T("f"))]
+tbs "/*b*/_"         => [(0:0, 0:5, BC("b")), (0:5, 0:6, Underscore)]
+ths "a=10"           => [(0:0, 0:1, Id("a")), (0:1, 0:2, Equals), (0:2, 0:4, Num(10.0))]
diff --git a/tests/src/layouter.rs b/tests/src/layouter.rs
index eeca1a1b..95eea204 100644
--- a/tests/src/layouter.rs
+++ b/tests/src/layouter.rs
@@ -10,8 +10,8 @@ use futures_executor::block_on;
 
 use typstc::Typesetter;
 use typstc::layout::{MultiLayout, Serialize};
-use typstc::size::{Size, Size2D};
-use typstc::style::PageStyle;
+use typstc::size::{Size, Size2D, ValueBox};
+use typstc::style::{PageStyle, PaperClass};
 use typstc::toddle::query::FileSystemFontProvider;
 use typstc::export::pdf::PdfExporter;
 
@@ -62,14 +62,15 @@ fn main() -> DynResult<()> {
     Ok(())
 }
 
-/// Create a _PDF_ with a name from the source code.
+/// Create a _PDF_ and render with a name from the source code.
 fn test(name: &str, src: &str) -> DynResult<()> {
     println!("Testing: {}.", name);
 
     let mut typesetter = Typesetter::new();
     typesetter.set_page_style(PageStyle {
+        class: PaperClass::Custom,
         dimensions: Size2D::with_all(Size::pt(250.0)),
-        .. PageStyle::default()
+        margins: ValueBox::with_all(None),
     });
 
     let provider = FileSystemFontProvider::from_index("../fonts/index.json")?;
@@ -120,31 +121,43 @@ fn test(name: &str, src: &str) -> DynResult<()> {
 
 /// Compile the source code with the typesetter.
 fn compile(typesetter: &Typesetter, src: &str) -> MultiLayout {
-    #[cfg(not(debug_assertions))] {
-        use std::time::Instant;
+    #![allow(unused_variables)]
+    use std::time::Instant;
 
-        // Warmup.
+    // Warmup.
+    #[cfg(not(debug_assertions))]
+    let warmup = {
         let warmup_start = Instant::now();
         block_on(typesetter.typeset(&src));
-        let warmup_end = Instant::now();
-
-        let start = Instant::now();
-        let tree = typesetter.parse(&src).output;
-        let mid = Instant::now();
-        let layouts = block_on(typesetter.layout(&tree)).output;
-        let end = Instant::now();
-
-        println!(" - cold start:  {:?}", warmup_end - warmup_start);
-        println!(" - warmed up:   {:?}", end - start);
-        println!("   - parsing:   {:?}", mid - start);
-        println!("   - layouting: {:?}", end - mid);
-        println!();
+        Instant::now() - warmup_start
+    };
+
+    let start = Instant::now();
+    let parsed = typesetter.parse(&src);
+    let parse = Instant::now() - start;
+
+    if !parsed.errors.is_empty() {
+        println!("parse errors: {:#?}", parsed.errors);
+    }
 
-        layouts
+    let start_layout = Instant::now();
+    let layouted = block_on(typesetter.layout(&parsed.output));
+    let layout = Instant::now() - start_layout;
+    let total = Instant::now() - start;
+
+    if !layouted.errors.is_empty() {
+        println!("layout errors: {:#?}", layouted.errors);
+    }
+
+    #[cfg(not(debug_assertions))] {
+        println!(" - cold start:  {:?}", warmup);
+        println!(" - warmed up:   {:?}", total);
+        println!("   - parsing:   {:?}", parse);
+        println!("   - layouting: {:?}", layout);
+        println!();
     }
 
-    #[cfg(debug_assertions)]
-    block_on(typesetter.typeset(&src))
+    layouted.output
 }
 
 /// Command line options.