Refactor Parser (#5310)

11 files changed, 223 insertions, 41 deletions

diff --git a/tests/Cargo.toml b/tests/Cargo.toml
index b1855b49..eed093eb 100644
--- a/tests/Cargo.toml
+++ b/tests/Cargo.toml
@@ -11,14 +11,32 @@ name = "tests"
 path = "src/tests.rs"
 harness = false
 
+[features]
+# Allow just compiling the parser when only testing typst-syntax. To do so,
+# pass '--no-default-features' to 'cargo test'.
+default = [
+    # "typst-syntax" intentionally not present
+    "typst",
+    "typst-assets",
+    "typst-dev-assets",
+    "typst-library",
+    "typst-pdf",
+    "typst-render",
+    "typst-svg",
+    "typst-svg",
+]
+
 [dependencies]
-typst = { workspace = true }
-typst-assets = { workspace = true, features = ["fonts"] }
-typst-dev-assets = { workspace = true }
-typst-library = { workspace = true }
-typst-pdf = { workspace = true }
-typst-render = { workspace = true }
-typst-svg = { workspace = true }
+typst-syntax = { workspace = true }
+# Mark other Typst crates as optional so we can use '--no-default-features'
+# to decrease compile times for parser testing.
+typst = { workspace = true, optional = true }
+typst-assets = { workspace = true, features = ["fonts"], optional = true }
+typst-dev-assets = { workspace = true, optional = true }
+typst-library = { workspace = true, optional = true }
+typst-pdf = { workspace = true, optional = true }
+typst-render = { workspace = true, optional = true }
+typst-svg = { workspace = true, optional = true }
 clap = { workspace = true }
 comemo = { workspace = true }
 ecow = { workspace = true }
diff --git a/tests/ref/single-right-bracket.png b/tests/ref/single-right-bracket.png
new file mode 100644
index 00000000..9867424d
--- /dev/null
+++ b/tests/ref/single-right-bracket.png
diff --git a/tests/src/args.rs b/tests/src/args.rs
index 786733cc..db5d1a9b 100644
--- a/tests/src/args.rs
+++ b/tests/src/args.rs
@@ -43,7 +43,9 @@ pub struct CliArguments {
     /// Runs SVG export.
     #[arg(long)]
     pub svg: bool,
-    /// Displays the syntax tree.
+    /// Displays the syntax tree before running tests.
+    ///
+    /// Note: This is ignored if using '--syntax-compare'.
     #[arg(long)]
     pub syntax: bool,
     /// Displays only one line per test, hiding details about failures.
@@ -55,6 +57,29 @@ pub struct CliArguments {
     /// How many threads to spawn when running the tests.
     #[arg(short = 'j', long)]
     pub num_threads: Option<usize>,
+    /// Changes testing behavior for debugging the parser: With no argument,
+    /// outputs the concrete syntax trees of tests as files in
+    /// 'tests/store/syntax/'. With a directory as argument, will treat it as a
+    /// reference of correct syntax tree files and will print which output
+    /// syntax trees differ (viewing the diffs is on you).
+    ///
+    /// This overrides the normal testing system. It parses, but does not run
+    /// the test suite.
+    ///
+    /// If `cargo test` is run with `--no-default-features`, then compiling will
+    /// not include Typst's core crates, only typst-syntax, greatly speeding up
+    /// debugging when changing the parser.
+    ///
+    /// You can generate a correct reference directory by running on a known
+    /// good commit and copying the generated outputs to a new directory.
+    /// `_things` may be a good location as it is in the top-level gitignore.
+    ///
+    /// You can view diffs in VS Code with: `code --diff <ref_dir>/<test>.syntax
+    /// tests/store/syntax/<test>.syntax`
+    #[arg(long)]
+    pub parser_compare: Option<Option<PathBuf>>,
+    // ^ I'm not using a subcommand here because then test patterns don't parse
+    // how you would expect and I'm too lazy to try to fix it.
 }
 
 impl CliArguments {
diff --git a/tests/src/collect.rs b/tests/src/collect.rs
index 80e5e5a8..5c7327f1 100644
--- a/tests/src/collect.rs
+++ b/tests/src/collect.rs
@@ -6,8 +6,8 @@ use std::str::FromStr;
 use std::sync::LazyLock;
 
 use ecow::{eco_format, EcoString};
-use typst::syntax::package::PackageVersion;
-use typst::syntax::{is_id_continue, is_ident, is_newline, FileId, Source, VirtualPath};
+use typst_syntax::package::PackageVersion;
+use typst_syntax::{is_id_continue, is_ident, is_newline, FileId, Source, VirtualPath};
 use unscanny::Scanner;
 
 /// Collects all tests from all files.
diff --git a/tests/src/logger.rs b/tests/src/logger.rs
index 45c9f098..48bad451 100644
--- a/tests/src/logger.rs
+++ b/tests/src/logger.rs
@@ -2,7 +2,16 @@ use std::io::{self, IsTerminal, StderrLock, Write};
 use std::time::{Duration, Instant};
 
 use crate::collect::Test;
-use crate::run::TestResult;
+
+/// The result of running a single test.
+pub struct TestResult {
+    /// The error log for this test. If empty, the test passed.
+    pub errors: String,
+    /// The info log for this test.
+    pub infos: String,
+    /// Whether the image was mismatched.
+    pub mismatched_image: bool,
+}
 
 /// Receives status updates by individual test runs.
 pub struct Logger<'a> {
@@ -58,7 +67,7 @@ impl<'a> Logger<'a> {
             }
         };
 
-        if result.is_ok() {
+        if result.errors.is_empty() {
             self.passed += 1;
         } else {
             self.failed += 1;
diff --git a/tests/src/run.rs b/tests/src/run.rs
index caa078c4..1ea19a16 100644
--- a/tests/src/run.rs
+++ b/tests/src/run.rs
@@ -12,6 +12,7 @@ use typst::WorldExt;
 use typst_pdf::PdfOptions;
 
 use crate::collect::{FileSize, NoteKind, Test};
+use crate::logger::TestResult;
 use crate::world::TestWorld;
 
 /// Runs a single test.
@@ -21,23 +22,6 @@ pub fn run(test: &Test) -> TestResult {
     Runner::new(test).run()
 }
 
-/// The result of running a single test.
-pub struct TestResult {
-    /// The error log for this test. If empty, the test passed.
-    pub errors: String,
-    /// The info log for this test.
-    pub infos: String,
-    /// Whether the image was mismatched.
-    pub mismatched_image: bool,
-}
-
-impl TestResult {
-    /// Whether the test passed.
-    pub fn is_ok(&self) -> bool {
-        self.errors.is_empty()
-    }
-}
-
 /// Write a line to a log sink, defaulting to the test's error log.
 macro_rules! log {
     (into: $sink:expr, $($tts:tt)*) => {
diff --git a/tests/src/tests.rs b/tests/src/tests.rs
index 940c9e3c..2b09b29c 100644
--- a/tests/src/tests.rs
+++ b/tests/src/tests.rs
@@ -1,13 +1,19 @@
 //! Typst's test runner.
 
+#![cfg_attr(not(feature = "default"), allow(dead_code, unused_imports))]
+
 mod args;
 mod collect;
-mod custom;
 mod logger;
+
+#[cfg(feature = "default")]
+mod custom;
+#[cfg(feature = "default")]
 mod run;
+#[cfg(feature = "default")]
 mod world;
 
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::sync::LazyLock;
 use std::time::Duration;
 
@@ -16,7 +22,8 @@ use parking_lot::Mutex;
 use rayon::iter::{ParallelBridge, ParallelIterator};
 
 use crate::args::{CliArguments, Command};
-use crate::logger::Logger;
+use crate::collect::Test;
+use crate::logger::{Logger, TestResult};
 
 /// The parsed command line arguments.
 static ARGS: LazyLock<CliArguments> = LazyLock::new(CliArguments::parse);
@@ -27,6 +34,9 @@ const SUITE_PATH: &str = "tests/suite";
 /// The directory where the full test results are stored.
 const STORE_PATH: &str = "tests/store";
 
+/// The directory where syntax trees are stored.
+const SYNTAX_PATH: &str = "tests/store/syntax";
+
 /// The directory where the reference images are stored.
 const REF_PATH: &str = "tests/ref";
 
@@ -89,6 +99,21 @@ fn test() {
         return;
     }
 
+    let parser_dirs = ARGS.parser_compare.clone().map(create_syntax_store);
+    #[cfg(not(feature = "default"))]
+    let parser_dirs = parser_dirs.or_else(|| Some(create_syntax_store(None)));
+
+    let runner = |test: &Test| {
+        if let Some((live_path, ref_path)) = &parser_dirs {
+            run_parser_test(test, live_path, ref_path)
+        } else {
+            #[cfg(feature = "default")]
+            return run::run(test);
+            #[cfg(not(feature = "default"))]
+            unreachable!();
+        }
+    };
+
     // Run the tests.
     let logger = Mutex::new(Logger::new(selected, skipped));
     std::thread::scope(|scope| {
@@ -112,7 +137,7 @@ fn test() {
         // to `typst::utils::Deferred` yielding.
         tests.iter().par_bridge().for_each(|test| {
             logger.lock().start(test);
-            let result = std::panic::catch_unwind(|| run::run(test));
+            let result = std::panic::catch_unwind(|| runner(test));
             logger.lock().end(test, result);
         });
 
@@ -142,3 +167,46 @@ fn undangle() {
         }
     }
 }
+
+fn create_syntax_store(ref_path: Option<PathBuf>) -> (&'static Path, Option<PathBuf>) {
+    if ref_path.as_ref().is_some_and(|p| !p.exists()) {
+        eprintln!("syntax reference path doesn't exist");
+        std::process::exit(1);
+    }
+
+    let live_path = Path::new(SYNTAX_PATH);
+    std::fs::remove_dir_all(live_path).ok();
+    std::fs::create_dir_all(live_path).unwrap();
+    (live_path, ref_path)
+}
+
+fn run_parser_test(
+    test: &Test,
+    live_path: &Path,
+    ref_path: &Option<PathBuf>,
+) -> TestResult {
+    let mut result = TestResult {
+        errors: String::new(),
+        infos: String::new(),
+        mismatched_image: false,
+    };
+
+    let syntax_file = live_path.join(format!("{}.syntax", test.name));
+    let tree = format!("{:#?}\n", test.source.root());
+    std::fs::write(syntax_file, &tree).unwrap();
+
+    let Some(ref_path) = ref_path else { return result };
+    let ref_file = ref_path.join(format!("{}.syntax", test.name));
+    match std::fs::read_to_string(&ref_file) {
+        Ok(ref_tree) => {
+            if tree != ref_tree {
+                result.errors = "differs".to_string();
+            }
+        }
+        Err(_) => {
+            result.errors = format!("missing reference: {}", ref_file.display());
+        }
+    }
+
+    result
+}
diff --git a/tests/suite/math/symbols.typ b/tests/suite/math/symbols.typ
new file mode 100644
index 00000000..65a48316
--- /dev/null
+++ b/tests/suite/math/symbols.typ
@@ -0,0 +1,29 @@
+// Test math symbol edge cases.
+
+--- math-symbol-basic ---
+#let sym = symbol("s", ("basic", "s"))
+#test($sym.basic$, $#"s"$)
+
+--- math-symbol-underscore ---
+#let sym = symbol("s", ("test_underscore", "s"))
+// Error: 6-10 unknown symbol modifier
+$sym.test_underscore$
+
+--- math-symbol-dash ---
+#let sym = symbol("s", ("test-dash", "s"))
+// Error: 6-10 unknown symbol modifier
+$sym.test-dash$
+
+--- math-symbol-double ---
+#let sym = symbol("s", ("test.basic", "s"))
+#test($sym.test.basic$, $#"s"$)
+
+--- math-symbol-double-underscore ---
+#let sym = symbol("s", ("one.test_underscore", "s"))
+// Error: 10-14 unknown symbol modifier
+$sym.one.test_underscore$
+
+--- math-symbol-double-dash ---
+#let sym = symbol("s", ("one.test-dash", "s"))
+// Error: 10-14 unknown symbol modifier
+$sym.one.test-dash$
diff --git a/tests/suite/model/heading.typ b/tests/suite/model/heading.typ
index 884f203d..d182724c 100644
--- a/tests/suite/model/heading.typ
+++ b/tests/suite/model/heading.typ
@@ -38,7 +38,7 @@ multiline.
 --- heading-trailing-whitespace ---
 // Whether headings contain trailing whitespace with or without comments/labels.
 // Labels are special cased to immediately end headings in the parser, but also
-// have unique whitespace behavior.
+// #strike[have unique whitespace behavior] Now their behavior is consistent!
 
 #let join(..xs) = xs.pos().join()
 #let head(h) = heading(depth: 1, h)
@@ -49,19 +49,20 @@ multiline.
 #test(head[h], [= h<a>])
 #test(head[h], [= h/**/<b>])
 
-// Label behaves differently than normal trailing space and comment.
-#test(head(join[h][ ]), [= h  ])
-#test(head(join[h][ ]), [= h  /**/])
+// #strike[Label behaves differently than normal trailing space and comment.]
+// Now they behave the same!
+#test(join(head[h])[ ], [= h  ])
+#test(join(head[h])[ ], [= h  /**/])
 #test(join(head[h])[ ], [= h  <c>])
 
 // Combinations.
-#test(head(join[h][ ][ ]), [= h  /**/  ])
+#test(join(head[h])[ ][ ], [= h  /**/  ])
 #test(join(head[h])[ ][ ], [= h  <d>  ])
-#test(head(join[h][ ]), [= h  /**/<e>])
+#test(join(head[h])[ ], [= h  /**/<e>])
 #test(join(head[h])[ ], [= h/**/  <f>])
 
-// The first space attaches, but not the second
-#test(join(head(join[h][ ]))[ ], [= h  /**/  <g>])
+// #strike[The first space attaches, but not the second] Now neither attaches!
+#test(join(head(join[h]))[ ][ ], [= h  /**/  <g>])
 
 --- heading-leading-whitespace ---
 // Test that leading whitespace and comments don't matter.
diff --git a/tests/suite/model/list.typ b/tests/suite/model/list.typ
index 46f4621f..c3c123de 100644
--- a/tests/suite/model/list.typ
+++ b/tests/suite/model/list.typ
@@ -34,6 +34,51 @@ _Shopping list_
    - C
 - D
 
+--- list-indent-trivia-nesting ---
+// Test indent nesting behavior with odd trivia (comments and spaces).
+
+#let indented = [
+- a
+ /**/- b
+/**/ - c
+   /*spanning
+     multiple
+      lines */ - d
+    - e
+/**/       - f
+/**/  - g
+]
+// Current behavior is that list columns are based on the first non-whitespace
+// element in their line, so the block comments here determine the column the
+// list starts at
+
+#let item = list.item
+#let manual = {
+  [ ]
+  item({
+    [a]
+    [ ]
+    item[b]
+    [ ]; [ ]
+    item({
+      [c]
+      [ ]; [ ]
+      item[d]
+    })
+    [ ]
+    item({
+      [e]
+      [ ]; [ ]
+      item[f]
+      [ ]; [ ]
+      item[g]
+    })
+  })
+  [ ]
+}
+
+#test(indented, manual)
+
 --- list-tabs ---
 // This works because tabs are used consistently.
 	- A with 1 tab
diff --git a/tests/suite/scripting/blocks.typ b/tests/suite/scripting/blocks.typ
index f139b8c6..ba1d9c89 100644
--- a/tests/suite/scripting/blocks.typ
+++ b/tests/suite/scripting/blocks.typ
@@ -135,6 +135,9 @@
 // Error: 2-3 unexpected closing brace
 #}
 
+--- single-right-bracket ---
+]
+
 --- content-block-in-markup-scope ---
 // Content blocks also create a scope.
 #[#let x = 1]