Better test runner (#3922)

7 files changed, 1360 insertions, 1431 deletions

diff --git a/tests/src/args.rs b/tests/src/args.rs
new file mode 100644
index 00000000..fcd4ead1
--- /dev/null
+++ b/tests/src/args.rs
@@ -0,0 +1,46 @@
+use clap::{Parser, Subcommand};
+
+/// Typst's test runner.
+#[derive(Debug, Clone, Parser)]
+#[clap(name = "typst-test", author)]
+pub struct CliArguments {
+    /// The command to run.
+    #[command(subcommand)]
+    pub command: Option<Command>,
+    /// All the tests that contain the filter string will be run.
+    pub filter: Vec<String>,
+    /// Runs only the tests with the exact specified `filter` names.
+    #[arg(short, long)]
+    pub exact: bool,
+    /// Whether to update the reference images of non-passing tests.
+    #[arg(short, long)]
+    pub update: bool,
+    /// The scaling factor to render the output image with.
+    ///
+    /// Does not affect the comparison or the reference image.
+    #[arg(short, long, default_value_t = 1.0)]
+    pub scale: f32,
+    /// Exports PDF outputs into the artifact store.
+    #[arg(long)]
+    pub pdf: bool,
+    /// Exports SVG outputs into the artifact store.
+    #[arg(long)]
+    pub svg: bool,
+    /// Whether to display the syntax tree.
+    #[arg(long)]
+    pub syntax: bool,
+    /// Prevents the terminal from being cleared of test names.
+    #[arg(short, long)]
+    pub verbose: bool,
+    /// How many threads to spawn when running the tests.
+    #[arg(short = 'j', long)]
+    pub num_threads: Option<usize>,
+}
+
+/// What to do.
+#[derive(Debug, Clone, Subcommand)]
+#[command()]
+pub enum Command {
+    /// Clears the on-disk test artifact store.
+    Clean,
+}
diff --git a/tests/src/collect.rs b/tests/src/collect.rs
new file mode 100644
index 00000000..44a325f2
--- /dev/null
+++ b/tests/src/collect.rs
@@ -0,0 +1,420 @@
+use std::collections::{HashMap, HashSet};
+use std::fmt::{self, Display, Formatter};
+use std::ops::Range;
+use std::path::{Path, PathBuf};
+use std::str::FromStr;
+
+use ecow::{eco_format, EcoString};
+use typst::syntax::package::PackageVersion;
+use typst::syntax::{is_id_continue, is_ident, is_newline, FileId, Source, VirtualPath};
+use unscanny::Scanner;
+
+/// Collects all tests from all files.
+///
+/// Returns:
+/// - the tests and the number of skipped tests in the success case.
+/// - parsing errors in the failure case.
+pub fn collect() -> Result<(Vec<Test>, usize), Vec<TestParseError>> {
+    Collector::new().collect()
+}
+
+/// A single test.
+pub struct Test {
+    pub pos: FilePos,
+    pub name: EcoString,
+    pub source: Source,
+    pub notes: Vec<Note>,
+    pub large: bool,
+}
+
+impl Display for Test {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        write!(f, "{} ({})", self.name, self.pos)
+    }
+}
+
+/// A position in a file.
+#[derive(Clone)]
+pub struct FilePos {
+    pub path: PathBuf,
+    pub line: usize,
+}
+
+impl FilePos {
+    fn new(path: impl Into<PathBuf>, line: usize) -> Self {
+        Self { path: path.into(), line }
+    }
+}
+
+impl Display for FilePos {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        write!(f, "{}:{}", self.path.display(), self.line)
+    }
+}
+
+/// The size of a file.
+pub struct FileSize(pub usize);
+
+impl Display for FileSize {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        write!(f, "{:.2} KiB", (self.0 as f64) / 1024.0)
+    }
+}
+
+/// An annotation like `// Error: 2-6 message` in a test.
+pub struct Note {
+    pub pos: FilePos,
+    pub kind: NoteKind,
+    pub range: Option<Range<usize>>,
+    pub message: String,
+}
+
+/// A kind of annotation in a test.
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+pub enum NoteKind {
+    Error,
+    Warning,
+    Hint,
+}
+
+impl FromStr for NoteKind {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(match s {
+            "Error" => Self::Error,
+            "Warning" => Self::Warning,
+            "Hint" => Self::Hint,
+            _ => return Err(()),
+        })
+    }
+}
+
+impl Display for NoteKind {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        f.pad(match self {
+            Self::Error => "Error",
+            Self::Warning => "Warning",
+            Self::Hint => "Hint",
+        })
+    }
+}
+
+/// Collects all tests from all files.
+struct Collector {
+    tests: Vec<Test>,
+    errors: Vec<TestParseError>,
+    seen: HashMap<EcoString, FilePos>,
+    large: HashSet<EcoString>,
+    skipped: usize,
+}
+
+impl Collector {
+    /// Creates a new test collector.
+    fn new() -> Self {
+        Self {
+            tests: vec![],
+            errors: vec![],
+            seen: HashMap::new(),
+            large: HashSet::new(),
+            skipped: 0,
+        }
+    }
+
+    /// Collects tests from all files.
+    fn collect(mut self) -> Result<(Vec<Test>, usize), Vec<TestParseError>> {
+        self.walk_files();
+        self.walk_references();
+
+        if self.errors.is_empty() {
+            Ok((self.tests, self.skipped))
+        } else {
+            Err(self.errors)
+        }
+    }
+
+    /// Walks through all test files and collects the tests.
+    fn walk_files(&mut self) {
+        for entry in walkdir::WalkDir::new(crate::SUITE_PATH).sort_by_file_name() {
+            let entry = entry.unwrap();
+            let path = entry.path();
+            if !path.extension().is_some_and(|ext| ext == "typ") {
+                continue;
+            }
+
+            let text = std::fs::read_to_string(path).unwrap();
+            if text.starts_with("// SKIP") {
+                continue;
+            }
+
+            Parser::new(self, path, &text).parse();
+        }
+    }
+
+    /// Walks through all reference images and ensure that a test exists for
+    /// each one.
+    fn walk_references(&mut self) {
+        for entry in walkdir::WalkDir::new(crate::REF_PATH).sort_by_file_name() {
+            let entry = entry.unwrap();
+            let path = entry.path();
+            if !path.extension().is_some_and(|ext| ext == "png") {
+                continue;
+            }
+
+            let stem = path.file_stem().unwrap().to_string_lossy();
+            let name = &*stem;
+
+            let Some(pos) = self.seen.get(name) else {
+                self.errors.push(TestParseError {
+                    pos: FilePos::new(path, 0),
+                    message: "dangling reference image".into(),
+                });
+                continue;
+            };
+
+            let len = path.metadata().unwrap().len() as usize;
+            if !self.large.contains(name) && len > crate::REF_LIMIT {
+                self.errors.push(TestParseError {
+                    pos: pos.clone(),
+                    message: format!(
+                        "reference image size exceeds {}, but the test is not marked as `// LARGE`",
+                        FileSize(crate::REF_LIMIT),
+                    ),
+                });
+            }
+        }
+    }
+}
+
+/// Parses a single test file.
+struct Parser<'a> {
+    collector: &'a mut Collector,
+    path: &'a Path,
+    s: Scanner<'a>,
+    test_start_line: usize,
+    line: usize,
+}
+
+impl<'a> Parser<'a> {
+    /// Creates a new parser for a file.
+    fn new(collector: &'a mut Collector, path: &'a Path, source: &'a str) -> Self {
+        Self {
+            collector,
+            path,
+            s: Scanner::new(source),
+            test_start_line: 1,
+            line: 1,
+        }
+    }
+
+    /// Parses an individual file.
+    fn parse(&mut self) {
+        self.skip_preamble();
+
+        while !self.s.done() {
+            let mut name = EcoString::new();
+            let mut notes = vec![];
+            if self.s.eat_if("---") {
+                self.s.eat_while(' ');
+                name = self.s.eat_until(char::is_whitespace).into();
+                self.s.eat_while(' ');
+
+                if name.is_empty() {
+                    self.error("expected test name");
+                } else if !is_ident(&name) {
+                    self.error(format!("test name `{name}` is not a valid identifier"));
+                } else if !self.s.eat_if("---") {
+                    self.error("expected closing ---");
+                }
+            } else {
+                self.error("expected opening ---");
+            }
+
+            if self.collector.seen.contains_key(&name) {
+                self.error(format!("duplicate test {name}"));
+            }
+
+            if self.s.eat_newline() {
+                self.line += 1;
+            }
+
+            let start = self.s.cursor();
+            self.test_start_line = self.line;
+
+            let pos = FilePos::new(self.path, self.test_start_line);
+            self.collector.seen.insert(name.clone(), pos.clone());
+
+            while !self.s.done() && !self.s.at("---") {
+                self.s.eat_until(is_newline);
+                if self.s.eat_newline() {
+                    self.line += 1;
+                }
+            }
+
+            let text = self.s.from(start);
+            let large = text.starts_with("// LARGE");
+            if large {
+                self.collector.large.insert(name.clone());
+            }
+
+            if !filtered(&name) {
+                self.collector.skipped += 1;
+                continue;
+            }
+
+            let vpath = VirtualPath::new(self.path);
+            let source = Source::new(FileId::new(None, vpath), text.into());
+
+            self.s.jump(start);
+            self.line = self.test_start_line;
+
+            while !self.s.done() && !self.s.at("---") {
+                self.s.eat_while(' ');
+                if self.s.eat_if("// ") {
+                    notes.extend(self.parse_note(&source));
+                }
+
+                self.s.eat_until(is_newline);
+                if self.s.eat_newline() {
+                    self.line += 1;
+                }
+            }
+
+            self.collector.tests.push(Test { pos, name, source, notes, large });
+        }
+    }
+
+    /// Skips the preamble of a test.
+    fn skip_preamble(&mut self) {
+        let mut errored = false;
+        while !self.s.done() && !self.s.at("---") {
+            let line = self.s.eat_until(is_newline).trim();
+            if !errored && !line.is_empty() && !line.starts_with("//") {
+                self.error("test preamble may only contain comments and blank lines");
+                errored = true;
+            }
+            if self.s.eat_newline() {
+                self.line += 1;
+            }
+        }
+    }
+
+    /// Parses an annotation in a test.
+    fn parse_note(&mut self, source: &Source) -> Option<Note> {
+        let head = self.s.eat_while(is_id_continue);
+        if !self.s.eat_if(':') {
+            return None;
+        }
+
+        let kind: NoteKind = head.parse().ok()?;
+        self.s.eat_if(' ');
+
+        let mut range = None;
+        if self.s.at('-') || self.s.at(char::is_numeric) {
+            range = self.parse_range(source);
+            if range.is_none() {
+                self.error("range is malformed");
+                return None;
+            }
+        }
+
+        let message = self
+            .s
+            .eat_until(is_newline)
+            .trim()
+            .replace("VERSION", &eco_format!("{}", PackageVersion::compiler()));
+
+        Some(Note {
+            pos: FilePos::new(self.path, self.line),
+            kind,
+            range,
+            message,
+        })
+    }
+
+    /// Parse a range, optionally abbreviated as just a position if the range
+    /// is empty.
+    fn parse_range(&mut self, source: &Source) -> Option<Range<usize>> {
+        let start = self.parse_position(source)?;
+        let end = if self.s.eat_if('-') { self.parse_position(source)? } else { start };
+        Some(start..end)
+    }
+
+    /// Parses a relative `(line:)?column` position.
+    fn parse_position(&mut self, source: &Source) -> Option<usize> {
+        let first = self.parse_number()?;
+        let (line_delta, column) =
+            if self.s.eat_if(':') { (first, self.parse_number()?) } else { (1, first) };
+
+        let text = source.text();
+        let line_idx_in_test = self.line - self.test_start_line;
+        let comments = text
+            .lines()
+            .skip(line_idx_in_test + 1)
+            .take_while(|line| line.trim().starts_with("//"))
+            .count();
+
+        let line_idx = (line_idx_in_test + comments).checked_add_signed(line_delta)?;
+        let column_idx = if column < 0 {
+            // Negative column index is from the back.
+            let range = source.line_to_range(line_idx)?;
+            text[range].chars().count().saturating_add_signed(column)
+        } else {
+            usize::try_from(column).ok()?.checked_sub(1)?
+        };
+
+        source.line_column_to_byte(line_idx, column_idx)
+    }
+
+    /// Parse a number.
+    fn parse_number(&mut self) -> Option<isize> {
+        let start = self.s.cursor();
+        self.s.eat_if('-');
+        self.s.eat_while(char::is_numeric);
+        self.s.from(start).parse().ok()
+    }
+
+    /// Stores a test parsing error.
+    fn error(&mut self, message: impl Into<String>) {
+        self.collector.errors.push(TestParseError {
+            pos: FilePos::new(self.path, self.line),
+            message: message.into(),
+        });
+    }
+}
+
+/// Whether a test is within the filtered set.
+fn filtered(name: &str) -> bool {
+    let exact = crate::ARGS.exact;
+    let filter = &crate::ARGS.filter;
+    filter.is_empty()
+        || filter
+            .iter()
+            .any(|v| if exact { name == v } else { name.contains(v) })
+}
+
+/// An error in a test file.
+pub struct TestParseError {
+    pos: FilePos,
+    message: String,
+}
+
+impl Display for TestParseError {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        write!(f, "{} ({})", self.message, self.pos)
+    }
+}
+
+trait ScannerExt {
+    fn eat_newline(&mut self) -> bool;
+}
+
+impl ScannerExt for Scanner<'_> {
+    fn eat_newline(&mut self) -> bool {
+        let ate = self.eat_if(is_newline);
+        if ate && self.before().ends_with('\r') {
+            self.eat_if('\n');
+        }
+        ate
+    }
+}
diff --git a/tests/src/logger.rs b/tests/src/logger.rs
new file mode 100644
index 00000000..c48650a7
--- /dev/null
+++ b/tests/src/logger.rs
@@ -0,0 +1,141 @@
+use std::io::{self, IsTerminal, StderrLock, Write};
+use std::time::{Duration, Instant};
+
+use crate::collect::Test;
+use crate::run::TestResult;
+
+/// Receives status updates by individual test runs.
+pub struct Logger<'a> {
+    filtered: usize,
+    passed: usize,
+    failed: usize,
+    skipped: usize,
+    mismatched_image: bool,
+    active: Vec<&'a Test>,
+    last_change: Instant,
+    temp_lines: usize,
+    terminal: bool,
+}
+
+impl<'a> Logger<'a> {
+    /// Create a new logger.
+    pub fn new(filtered: usize, skipped: usize) -> Self {
+        Self {
+            filtered,
+            passed: 0,
+            failed: 0,
+            skipped,
+            mismatched_image: false,
+            active: vec![],
+            temp_lines: 0,
+            last_change: Instant::now(),
+            terminal: std::io::stderr().is_terminal(),
+        }
+    }
+
+    /// Register the start of a test.
+    pub fn start(&mut self, test: &'a Test) {
+        self.active.push(test);
+        self.last_change = Instant::now();
+        self.refresh();
+    }
+
+    /// Register a finished test.
+    pub fn end(&mut self, test: &'a Test, result: std::thread::Result<TestResult>) {
+        self.active.retain(|t| t.name != test.name);
+
+        let result = match result {
+            Ok(result) => result,
+            Err(_) => {
+                self.failed += 1;
+                self.temp_lines = 0;
+                self.print(move |out| {
+                    writeln!(out, "❌ {test} panicked")?;
+                    Ok(())
+                })
+                .unwrap();
+                return;
+            }
+        };
+
+        if result.is_ok() {
+            self.passed += 1;
+        } else {
+            self.failed += 1;
+        }
+
+        self.mismatched_image |= result.mismatched_image;
+        self.last_change = Instant::now();
+
+        self.print(move |out| {
+            if !result.errors.is_empty() {
+                writeln!(out, "❌ {test}")?;
+                for line in result.errors.lines() {
+                    writeln!(out, "  {line}")?;
+                }
+            } else if crate::ARGS.verbose || !result.infos.is_empty() {
+                writeln!(out, "✅ {test}")?;
+            }
+            for line in result.infos.lines() {
+                writeln!(out, "  {line}")?;
+            }
+            Ok(())
+        })
+        .unwrap();
+    }
+
+    /// Prints a summary and returns whether the test suite passed.
+    pub fn finish(&self) -> bool {
+        let Self { filtered, passed, failed, skipped, .. } = *self;
+
+        eprintln!("{passed} passed, {failed} failed, {skipped} skipped");
+        assert_eq!(filtered, passed + failed, "not all tests were executed succesfully");
+
+        if self.mismatched_image {
+            eprintln!("  pass the --update flag to update the reference images");
+        }
+
+        self.failed == 0
+    }
+
+    /// Refresh the status.
+    pub fn refresh(&mut self) {
+        self.print(|_| Ok(())).unwrap();
+    }
+
+    /// Refresh the status print.
+    fn print(
+        &mut self,
+        inner: impl FnOnce(&mut StderrLock<'_>) -> io::Result<()>,
+    ) -> io::Result<()> {
+        let mut out = std::io::stderr().lock();
+
+        // Clear the status lines.
+        for _ in 0..self.temp_lines {
+            write!(out, "\x1B[1F\x1B[0J")?;
+            self.temp_lines = 0;
+        }
+
+        // Print the result of a finished test.
+        inner(&mut out)?;
+
+        // Print the status line.
+        let done = self.failed + self.passed;
+        if done < self.filtered {
+            if self.last_change.elapsed() > Duration::from_secs(2) {
+                for test in &self.active {
+                    writeln!(out, "⏰ {test} is taking a long time ...")?;
+                    if self.terminal {
+                        self.temp_lines += 1;
+                    }
+                }
+            }
+            if self.terminal {
+                writeln!(out, "💨 {done} / {}", self.filtered)?;
+                self.temp_lines += 1;
+            }
+        }
+
+        Ok(())
+    }
+}
diff --git a/tests/src/metadata.rs b/tests/src/metadata.rs
deleted file mode 100644
index 53cbbdff..00000000
--- a/tests/src/metadata.rs
+++ /dev/null
@@ -1,334 +0,0 @@
-use std::collections::HashSet;
-use std::fmt::{self, Display, Formatter};
-use std::ops::Range;
-use std::str::FromStr;
-
-use ecow::EcoString;
-use typst::syntax::package::PackageVersion;
-use typst::syntax::Source;
-use unscanny::Scanner;
-
-/// Each test and subset may contain metadata.
-#[derive(Debug)]
-pub struct TestMetadata {
-    /// Configures how the test is run.
-    pub config: TestConfig,
-    /// Declares properties that must hold for a test.
-    ///
-    /// For instance, `// Warning: 1-3 no text within underscores`
-    /// will fail the test if the warning isn't generated by your test.
-    pub annotations: HashSet<Annotation>,
-}
-
-/// Configuration of a test or subtest.
-#[derive(Debug, Default)]
-pub struct TestConfig {
-    /// Reference images will be generated and compared.
-    ///
-    /// Defaults to `true`, can be disabled with `Ref: false`.
-    pub compare_ref: Option<bool>,
-    /// Hint annotations will be compared to compiler hints.
-    ///
-    /// Defaults to `true`, can be disabled with `Hints: false`.
-    pub validate_hints: Option<bool>,
-    /// Autocompletion annotations will be validated against autocompletions.
-    /// Mutually exclusive with error and hint annotations.
-    ///
-    /// Defaults to `false`, can be enabled with `Autocomplete: true`.
-    pub validate_autocomplete: Option<bool>,
-}
-
-/// Parsing error when the metadata is invalid.
-pub(crate) enum InvalidMetadata {
-    /// An invalid annotation and it's error message.
-    InvalidAnnotation(Annotation, String),
-    /// Setting metadata can only be done with `true` or `false` as a value.
-    InvalidSet(String),
-}
-
-impl InvalidMetadata {
-    pub(crate) fn write(
-        invalid_data: Vec<InvalidMetadata>,
-        output: &mut String,
-        print_annotation: &mut impl FnMut(&Annotation, &mut String),
-    ) {
-        use std::fmt::Write;
-        for data in invalid_data.into_iter() {
-            let (annotation, error) = match data {
-                InvalidMetadata::InvalidAnnotation(a, e) => (Some(a), e),
-                InvalidMetadata::InvalidSet(e) => (None, e),
-            };
-            write!(output, "{error}",).unwrap();
-            if let Some(annotation) = annotation {
-                print_annotation(&annotation, output)
-            } else {
-                writeln!(output).unwrap();
-            }
-        }
-    }
-}
-
-/// Annotation of the form `// KIND: RANGE TEXT`.
-#[derive(Debug, Clone, Eq, PartialEq, Hash)]
-pub struct Annotation {
-    /// Which kind of annotation this is.
-    pub kind: AnnotationKind,
-    /// May be written as:
-    /// - `{line}:{col}-{line}:{col}`, e.g. `0:4-0:6`.
-    /// - `{col}-{col}`, e.g. `4-6`:
-    ///    The line is assumed to be the line after the annotation.
-    /// - `-1`: Produces a range of length zero at the end of the next line.
-    ///   Mostly useful for autocompletion tests which require an index.
-    pub range: Option<Range<usize>>,
-    /// The raw text after the annotation.
-    pub text: EcoString,
-}
-
-/// The different kinds of in-test annotations.
-#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
-pub enum AnnotationKind {
-    Error,
-    Warning,
-    Hint,
-    AutocompleteContains,
-    AutocompleteExcludes,
-}
-
-impl AnnotationKind {
-    /// Returns the user-facing string for this annotation.
-    pub fn as_str(self) -> &'static str {
-        match self {
-            AnnotationKind::Error => "Error",
-            AnnotationKind::Warning => "Warning",
-            AnnotationKind::Hint => "Hint",
-            AnnotationKind::AutocompleteContains => "Autocomplete contains",
-            AnnotationKind::AutocompleteExcludes => "Autocomplete excludes",
-        }
-    }
-}
-
-impl FromStr for AnnotationKind {
-    type Err = &'static str;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        Ok(match s {
-            "Error" => AnnotationKind::Error,
-            "Warning" => AnnotationKind::Warning,
-            "Hint" => AnnotationKind::Hint,
-            "Autocomplete contains" => AnnotationKind::AutocompleteContains,
-            "Autocomplete excludes" => AnnotationKind::AutocompleteExcludes,
-            _ => return Err("invalid annotatino"),
-        })
-    }
-}
-
-impl Display for AnnotationKind {
-    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
-        f.pad(self.as_str())
-    }
-}
-
-/// Parse metadata for a test.
-pub fn parse_part_metadata(
-    source: &Source,
-    is_header: bool,
-) -> Result<TestMetadata, Vec<InvalidMetadata>> {
-    let mut config = TestConfig::default();
-    let mut annotations = HashSet::default();
-    let mut invalid_data = vec![];
-
-    let lines = source_to_lines(source);
-
-    for (i, line) in lines.iter().enumerate() {
-        if let Some((key, value)) = parse_metadata_line(line) {
-            let key = key.trim();
-            match key {
-                "Ref" => validate_set_annotation(
-                    value,
-                    &mut config.compare_ref,
-                    &mut invalid_data,
-                ),
-                "Hints" => validate_set_annotation(
-                    value,
-                    &mut config.validate_hints,
-                    &mut invalid_data,
-                ),
-                "Autocomplete" => validate_set_annotation(
-                    value,
-                    &mut config.validate_autocomplete,
-                    &mut invalid_data,
-                ),
-                annotation_key => {
-                    let Ok(kind) = AnnotationKind::from_str(annotation_key) else {
-                        continue;
-                    };
-                    let mut s = Scanner::new(value);
-                    let range = parse_range(&mut s, i, source);
-                    let rest = if range.is_some() { s.after() } else { s.string() };
-                    let message = rest
-                        .trim()
-                        .replace("VERSION", &PackageVersion::compiler().to_string())
-                        .into();
-
-                    let annotation =
-                        Annotation { kind, range: range.clone(), text: message };
-
-                    if is_header {
-                        invalid_data.push(InvalidMetadata::InvalidAnnotation(
-                            annotation,
-                            format!(
-                                "Error: header may not contain annotations of type {kind}"
-                            ),
-                        ));
-                        continue;
-                    }
-
-                    if matches!(
-                        kind,
-                        AnnotationKind::AutocompleteContains
-                            | AnnotationKind::AutocompleteExcludes
-                    ) {
-                        if let Some(range) = range {
-                            if range.start != range.end {
-                                invalid_data.push(InvalidMetadata::InvalidAnnotation(
-                                    annotation,
-                                    "Error: found range in Autocomplete annotation where range.start != range.end, range.end would be ignored."
-                                        .to_string()
-                                    ));
-                                continue;
-                            }
-                        } else {
-                            invalid_data.push(InvalidMetadata::InvalidAnnotation(
-                                annotation,
-                                "Error: autocomplete annotation but no range specified"
-                                    .to_string(),
-                            ));
-                            continue;
-                        }
-                    }
-                    annotations.insert(annotation);
-                }
-            }
-        }
-    }
-    if invalid_data.is_empty() {
-        Ok(TestMetadata { config, annotations })
-    } else {
-        Err(invalid_data)
-    }
-}
-
-/// Extract key and value for a metadata line of the form: `// KEY: VALUE`.
-fn parse_metadata_line(line: &str) -> Option<(&str, &str)> {
-    let mut s = Scanner::new(line);
-    if !s.eat_if("// ") {
-        return None;
-    }
-
-    let key = s.eat_until(':').trim();
-    if !s.eat_if(':') {
-        return None;
-    }
-
-    let value = s.eat_until('\n').trim();
-    Some((key, value))
-}
-
-/// Parse a quoted string.
-fn parse_string<'a>(s: &mut Scanner<'a>) -> Option<&'a str> {
-    if !s.eat_if('"') {
-        return None;
-    }
-    let sub = s.eat_until('"');
-    if !s.eat_if('"') {
-        return None;
-    }
-
-    Some(sub)
-}
-
-/// Parse a number.
-fn parse_num(s: &mut Scanner) -> Option<isize> {
-    let mut first = true;
-    let n = &s.eat_while(|c: char| {
-        let valid = first && c == '-' || c.is_numeric();
-        first = false;
-        valid
-    });
-    n.parse().ok()
-}
-
-/// Parse a comma-separated list of strings.
-pub fn parse_string_list(text: &str) -> HashSet<&str> {
-    let mut s = Scanner::new(text);
-    let mut result = HashSet::new();
-    while let Some(sub) = parse_string(&mut s) {
-        result.insert(sub);
-        s.eat_whitespace();
-        if !s.eat_if(',') {
-            break;
-        }
-        s.eat_whitespace();
-    }
-    result
-}
-
-/// Parse a position.
-fn parse_pos(s: &mut Scanner, i: usize, source: &Source) -> Option<usize> {
-    let first = parse_num(s)? - 1;
-    let (delta, column) =
-        if s.eat_if(':') { (first, parse_num(s)? - 1) } else { (0, first) };
-    let line = (i + comments_until_code(source, i)).checked_add_signed(delta)?;
-    source.line_column_to_byte(line, usize::try_from(column).ok()?)
-}
-
-/// Parse a range.
-fn parse_range(s: &mut Scanner, i: usize, source: &Source) -> Option<Range<usize>> {
-    let lines = source_to_lines(source);
-    s.eat_whitespace();
-    if s.eat_if("-1") {
-        let mut add = 1;
-        while let Some(line) = lines.get(i + add) {
-            if !line.starts_with("//") {
-                break;
-            }
-            add += 1;
-        }
-        let next_line = lines.get(i + add)?;
-        let col = next_line.chars().count();
-
-        let index = source.line_column_to_byte(i + add, col)?;
-        s.eat_whitespace();
-        return Some(index..index);
-    }
-    let start = parse_pos(s, i, source)?;
-    let end = if s.eat_if('-') { parse_pos(s, i, source)? } else { start };
-    s.eat_whitespace();
-    Some(start..end)
-}
-
-/// Returns the number of lines of comment from line i to next line of code.
-fn comments_until_code(source: &Source, i: usize) -> usize {
-    source_to_lines(source)[i..]
-        .iter()
-        .take_while(|line| line.starts_with("//"))
-        .count()
-}
-
-fn source_to_lines(source: &Source) -> Vec<&str> {
-    source.text().lines().map(str::trim).collect()
-}
-
-fn validate_set_annotation(
-    value: &str,
-    flag: &mut Option<bool>,
-    invalid_data: &mut Vec<InvalidMetadata>,
-) {
-    let value = value.trim();
-    if value != "false" && value != "true" {
-        invalid_data.push(
-            InvalidMetadata::InvalidSet(format!("Error: trying to set Ref, Hints, or Autocomplete with value {value:?} != true, != false.")))
-    } else {
-        *flag = Some(value == "true")
-    }
-}
diff --git a/tests/src/run.rs b/tests/src/run.rs
new file mode 100644
index 00000000..f797147f
--- /dev/null
+++ b/tests/src/run.rs
@@ -0,0 +1,442 @@
+use std::fmt::Write;
+use std::ops::Range;
+use std::path::Path;
+
+use ecow::eco_vec;
+use tiny_skia as sk;
+use typst::diag::SourceDiagnostic;
+use typst::eval::Tracer;
+use typst::foundations::Smart;
+use typst::introspection::Meta;
+use typst::layout::{Abs, Frame, FrameItem, Page, Transform};
+use typst::model::Document;
+use typst::visualize::Color;
+use typst::WorldExt;
+
+use crate::collect::{FileSize, NoteKind, Test};
+use crate::world::TestWorld;
+
+/// Runs a single test.
+///
+/// Returns whether the test passed.
+pub fn run(test: &Test) -> TestResult {
+    Runner::new(test).run()
+}
+
+/// The result of running a single test.
+pub struct TestResult {
+    /// The error log for this test. If empty, the test passed.
+    pub errors: String,
+    /// The info log for this test.
+    pub infos: String,
+    /// Whether the image was mismatched.
+    pub mismatched_image: bool,
+}
+
+impl TestResult {
+    /// Whether the test passed.
+    pub fn is_ok(&self) -> bool {
+        self.errors.is_empty()
+    }
+}
+
+/// Write a line to a log sink, defaulting to the test's error log.
+macro_rules! log {
+    (into: $sink:expr, $($tts:tt)*) => {
+        writeln!($sink, $($tts)*).unwrap();
+    };
+    ($runner:expr, $($tts:tt)*) => {
+        writeln!(&mut $runner.result.errors, $($tts)*).unwrap();
+    };
+}
+
+/// Runs a single test.
+pub struct Runner<'a> {
+    test: &'a Test,
+    world: TestWorld,
+    seen: Vec<bool>,
+    result: TestResult,
+    not_annotated: String,
+}
+
+impl<'a> Runner<'a> {
+    /// Create a new test runner.
+    fn new(test: &'a Test) -> Self {
+        Self {
+            test,
+            world: TestWorld::new(test.source.clone()),
+            seen: vec![false; test.notes.len()],
+            result: TestResult {
+                errors: String::new(),
+                infos: String::new(),
+                mismatched_image: false,
+            },
+            not_annotated: String::new(),
+        }
+    }
+
+    /// Run the test.
+    fn run(mut self) -> TestResult {
+        if crate::ARGS.syntax {
+            log!(into: self.result.infos, "tree: {:#?}", self.test.source.root());
+        }
+
+        let mut tracer = Tracer::new();
+        let (doc, errors) = match typst::compile(&self.world, &mut tracer) {
+            Ok(doc) => (Some(doc), eco_vec![]),
+            Err(errors) => (None, errors),
+        };
+
+        let warnings = tracer.warnings();
+        if doc.is_none() && errors.is_empty() {
+            log!(self, "no document, but also no errors");
+        }
+
+        self.check_document(doc.as_ref());
+
+        for error in &errors {
+            self.check_diagnostic(NoteKind::Error, error);
+        }
+
+        for warning in &warnings {
+            self.check_diagnostic(NoteKind::Warning, warning);
+        }
+
+        self.handle_not_emitted();
+        self.handle_not_annotated();
+
+        self.result
+    }
+
+    /// Handle errors that weren't annotated.
+    fn handle_not_annotated(&mut self) {
+        if !self.not_annotated.is_empty() {
+            log!(self, "not annotated");
+            self.result.errors.push_str(&self.not_annotated);
+        }
+    }
+
+    /// Handle notes that weren't handled before.
+    fn handle_not_emitted(&mut self) {
+        let mut first = true;
+        for (note, &seen) in self.test.notes.iter().zip(&self.seen) {
+            if seen {
+                continue;
+            }
+            let note_range = self.format_range(&note.range);
+            if first {
+                log!(self, "not emitted");
+                first = false;
+            }
+            log!(self, "  {}: {note_range} {} ({})", note.kind, note.message, note.pos,);
+        }
+    }
+
+    /// Check that the document output is correct.
+    fn check_document(&mut self, document: Option<&Document>) {
+        let live_path = format!("{}/render/{}.png", crate::STORE_PATH, self.test.name);
+        let ref_path = format!("{}/{}.png", crate::REF_PATH, self.test.name);
+        let has_ref = Path::new(&ref_path).exists();
+
+        let Some(document) = document else {
+            if has_ref {
+                log!(self, "missing document");
+                log!(self, "  ref       | {ref_path}");
+            }
+            return;
+        };
+
+        let skippable = match document.pages.as_slice() {
+            [page] => skippable(page),
+            _ => false,
+        };
+
+        // Tests without visible output and no reference image don't need to be
+        // compared.
+        if skippable && !has_ref {
+            std::fs::remove_file(&live_path).ok();
+            return;
+        }
+
+        // Render the live version.
+        let pixmap = render(document, 1.0);
+
+        // Save live version, possibly rerendering if different scale is
+        // requested.
+        let mut pixmap_live = &pixmap;
+        let slot;
+        let scale = crate::ARGS.scale;
+        if scale != 1.0 {
+            slot = render(document, scale);
+            pixmap_live = &slot;
+        }
+        let data = pixmap_live.encode_png().unwrap();
+        std::fs::write(&live_path, data).unwrap();
+
+        // Write PDF if requested.
+        if crate::ARGS.pdf {
+            let pdf_path = format!("{}/pdf/{}.pdf", crate::STORE_PATH, self.test.name);
+            let pdf = typst_pdf::pdf(document, Smart::Auto, None);
+            std::fs::write(pdf_path, pdf).unwrap();
+        }
+
+        // Write SVG if requested.
+        if crate::ARGS.svg {
+            let svg_path = format!("{}/svg/{}.svg", crate::STORE_PATH, self.test.name);
+            let svg = typst_svg::svg_merged(document, Abs::pt(5.0));
+            std::fs::write(svg_path, svg).unwrap();
+        }
+
+        // Compare against reference image if available.
+        let equal = has_ref && {
+            let ref_data = std::fs::read(&ref_path).unwrap();
+            let ref_pixmap = sk::Pixmap::decode_png(&ref_data).unwrap();
+            approx_equal(&pixmap, &ref_pixmap)
+        };
+
+        // Test that is ok doesn't need to be updated.
+        if equal {
+            return;
+        }
+
+        if crate::ARGS.update {
+            if skippable {
+                std::fs::remove_file(&ref_path).unwrap();
+                log!(
+                    into: self.result.infos,
+                    "removed reference image ({ref_path})"
+                );
+            } else {
+                let opts = oxipng::Options::max_compression();
+                let data = pixmap.encode_png().unwrap();
+                let ref_data = oxipng::optimize_from_memory(&data, &opts).unwrap();
+                if !self.test.large && ref_data.len() > crate::REF_LIMIT {
+                    log!(self, "reference image would exceed maximum size");
+                    log!(self, "  maximum   | {}", FileSize(crate::REF_LIMIT));
+                    log!(self, "  size      | {}", FileSize(ref_data.len()));
+                    log!(self, "please try to minimize the size of the test (smaller pages, less text, etc.)");
+                    log!(self, "if you think the test cannot be reasonably minimized, mark it as `// LARGE`");
+                    return;
+                }
+                std::fs::write(&ref_path, &ref_data).unwrap();
+                log!(
+                    into: self.result.infos,
+                    "Updated reference image ({ref_path}, {})",
+                    FileSize(ref_data.len()),
+                );
+            }
+        } else {
+            self.result.mismatched_image = true;
+            if has_ref {
+                log!(self, "mismatched rendering");
+                log!(self, "  live      | {live_path}");
+                log!(self, "  ref       | {ref_path}");
+            } else {
+                log!(self, "missing reference image");
+                log!(self, "  live      | {live_path}");
+            }
+        }
+    }
+
+    /// Compare a subset of notes with a given kind against diagnostics of
+    /// that same kind.
+    fn check_diagnostic(&mut self, kind: NoteKind, diag: &SourceDiagnostic) {
+        // Ignore diagnostics from other sources than the test file itself.
+        if diag.span.id().is_some_and(|id| id != self.test.source.id()) {
+            return;
+        }
+
+        let message = diag.message.replace("\\", "/");
+        let range = self.world.range(diag.span);
+        self.validate_note(kind, range.clone(), &message);
+
+        // Check hints.
+        for hint in &diag.hints {
+            self.validate_note(NoteKind::Hint, range.clone(), hint);
+        }
+    }
+
+    /// Try to find a matching note for the given `kind`, `range`, and
+    /// `message`.
+    ///
+    /// - If found, marks it as seen and returns it.
+    /// - If none was found, emits a "Not annotated" error and returns nothing.
+    fn validate_note(
+        &mut self,
+        kind: NoteKind,
+        range: Option<Range<usize>>,
+        message: &str,
+    ) {
+        // Try to find perfect match.
+        if let Some((i, _)) = self.test.notes.iter().enumerate().find(|&(i, note)| {
+            !self.seen[i]
+                && note.kind == kind
+                && note.range == range
+                && note.message == message
+        }) {
+            self.seen[i] = true;
+            return;
+        }
+
+        // Try to find closely matching annotation. If the note has the same
+        // range or message, it's most likely the one we're interested in.
+        let Some((i, note)) = self.test.notes.iter().enumerate().find(|&(i, note)| {
+            !self.seen[i]
+                && note.kind == kind
+                && (note.range == range || note.message == message)
+        }) else {
+            // Not even a close match, diagnostic is not annotated.
+            let diag_range = self.format_range(&range);
+            log!(into: self.not_annotated, "  {kind}: {diag_range} {}", message);
+            return;
+        };
+
+        // Mark this annotation as visited and return it.
+        self.seen[i] = true;
+
+        // Range is wrong.
+        if range != note.range {
+            let note_range = self.format_range(&note.range);
+            let note_text = self.text_for_range(&note.range);
+            let diag_range = self.format_range(&range);
+            let diag_text = self.text_for_range(&range);
+            log!(self, "mismatched range ({}):", note.pos);
+            log!(self, "  message   | {}", note.message);
+            log!(self, "  annotated | {note_range:<9} | {note_text}");
+            log!(self, "  emitted   | {diag_range:<9} | {diag_text}");
+        }
+
+        // Message is wrong.
+        if message != note.message {
+            log!(self, "mismatched message ({}):", note.pos);
+            log!(self, "  annotated | {}", note.message);
+            log!(self, "  emitted   | {message}");
+        }
+    }
+
+    /// Display the text for a range.
+    fn text_for_range(&self, range: &Option<Range<usize>>) -> String {
+        let Some(range) = range else { return "No text".into() };
+        if range.is_empty() {
+            "(empty)".into()
+        } else {
+            format!("`{}`", self.test.source.text()[range.clone()].replace('\n', "\\n"))
+        }
+    }
+
+    /// Display a byte range as a line:column range.
+    fn format_range(&self, range: &Option<Range<usize>>) -> String {
+        let Some(range) = range else { return "No range".into() };
+        if range.start == range.end {
+            self.format_pos(range.start)
+        } else {
+            format!("{}-{}", self.format_pos(range.start,), self.format_pos(range.end,))
+        }
+    }
+
+    /// Display a position as a line:column pair.
+    fn format_pos(&self, pos: usize) -> String {
+        if let (Some(line_idx), Some(column_idx)) =
+            (self.test.source.byte_to_line(pos), self.test.source.byte_to_column(pos))
+        {
+            let line = self.test.pos.line + line_idx;
+            let column = column_idx + 1;
+            if line == 1 {
+                format!("{column}")
+            } else {
+                format!("{line}:{column}")
+            }
+        } else {
+            "oob".into()
+        }
+    }
+}
+
+/// Draw all frames into one image with padding in between.
+fn render(document: &Document, pixel_per_pt: f32) -> sk::Pixmap {
+    for page in &document.pages {
+        let limit = Abs::cm(100.0);
+        if page.frame.width() > limit || page.frame.height() > limit {
+            panic!("overlarge frame: {:?}", page.frame.size());
+        }
+    }
+
+    let gap = Abs::pt(1.0);
+    let mut pixmap = typst_render::render_merged(
+        document,
+        pixel_per_pt,
+        Color::WHITE,
+        gap,
+        Color::BLACK,
+    );
+
+    let gap = (pixel_per_pt * gap.to_pt() as f32).round();
+
+    let mut y = 0.0;
+    for page in &document.pages {
+        let ts =
+            sk::Transform::from_scale(pixel_per_pt, pixel_per_pt).post_translate(0.0, y);
+        render_links(&mut pixmap, ts, &page.frame);
+        y += (pixel_per_pt * page.frame.height().to_pt() as f32).round().max(1.0) + gap;
+    }
+
+    pixmap
+}
+
+/// Draw extra boxes for links so we can see whether they are there.
+fn render_links(canvas: &mut sk::Pixmap, ts: sk::Transform, frame: &Frame) {
+    for (pos, item) in frame.items() {
+        let ts = ts.pre_translate(pos.x.to_pt() as f32, pos.y.to_pt() as f32);
+        match *item {
+            FrameItem::Group(ref group) => {
+                let ts = ts.pre_concat(to_sk_transform(&group.transform));
+                render_links(canvas, ts, &group.frame);
+            }
+            FrameItem::Meta(Meta::Link(_), size) => {
+                let w = size.x.to_pt() as f32;
+                let h = size.y.to_pt() as f32;
+                let rect = sk::Rect::from_xywh(0.0, 0.0, w, h).unwrap();
+                let mut paint = sk::Paint::default();
+                paint.set_color_rgba8(40, 54, 99, 40);
+                canvas.fill_rect(rect, &paint, ts, None);
+            }
+            _ => {}
+        }
+    }
+}
+
+/// Whether rendering of a frame can be skipped.
+fn skippable(page: &Page) -> bool {
+    page.frame.width().approx_eq(Abs::pt(120.0))
+        && page.frame.height().approx_eq(Abs::pt(20.0))
+        && skippable_frame(&page.frame)
+}
+
+/// Whether rendering of a frame can be skipped.
+fn skippable_frame(frame: &Frame) -> bool {
+    frame.items().all(|(_, item)| match item {
+        FrameItem::Group(group) => skippable_frame(&group.frame),
+        FrameItem::Meta(..) => true,
+        _ => false,
+    })
+}
+
+/// Whether to pixel images are approximately equal.
+fn approx_equal(a: &sk::Pixmap, b: &sk::Pixmap) -> bool {
+    a.width() == b.width()
+        && a.height() == b.height()
+        && a.data().iter().zip(b.data()).all(|(&a, &b)| a.abs_diff(b) <= 1)
+}
+
+/// Convert a Typst transform to a tiny-skia transform.
+fn to_sk_transform(transform: &Transform) -> sk::Transform {
+    let Transform { sx, ky, kx, sy, tx, ty } = *transform;
+    sk::Transform::from_row(
+        sx.get() as _,
+        ky.get() as _,
+        kx.get() as _,
+        sy.get() as _,
+        tx.to_pt() as f32,
+        ty.to_pt() as f32,
+    )
+}
diff --git a/tests/src/tests.rs b/tests/src/tests.rs
index e4f60bb6..6d58e969 100644
--- a/tests/src/tests.rs
+++ b/tests/src/tests.rs
@@ -1,1127 +1,112 @@
-/*! This is Typst's test runner.
+//! Typst's test runner.
 
-Tests are Typst files composed of a header part followed by subtests.
+mod args;
+mod collect;
+mod logger;
+mod run;
+mod world;
 
-The header may contain:
-- a small description `// tests that features X works well`
-- metadata (see [metadata::TestConfiguration])
-
-The subtests may use extra testing functions defined in [library], most
-importantly, `test(x, y)` which will fail the test `if x != y`.
-*/
-
-#![allow(clippy::comparison_chain)]
-mod metadata;
-
-use self::metadata::*;
-
-use std::borrow::Cow;
-use std::collections::{HashMap, HashSet};
-use std::ffi::OsStr;
-use std::fmt::Write as _;
-use std::io::{self, IsTerminal, Write as _};
-use std::ops::Range;
-use std::path::{Path, PathBuf, MAIN_SEPARATOR_STR};
-use std::sync::{OnceLock, RwLock};
-use std::{env, fs};
+use std::path::Path;
+use std::time::Duration;
 
 use clap::Parser;
-use comemo::{Prehashed, Track};
-use oxipng::{InFile, Options, OutFile};
+use once_cell::sync::Lazy;
+use parking_lot::Mutex;
 use rayon::iter::{ParallelBridge, ParallelIterator};
-use tiny_skia as sk;
-use typst::diag::{bail, FileError, FileResult, Severity, SourceDiagnostic, StrResult};
-use typst::eval::Tracer;
-use typst::foundations::{func, Bytes, Datetime, NoneValue, Repr, Smart, Value};
-use typst::introspection::Meta;
-use typst::layout::{Abs, Frame, FrameItem, Margin, Page, PageElem, Transform};
-use typst::model::Document;
-use typst::syntax::{FileId, Source, SyntaxNode, VirtualPath};
-use typst::text::{Font, FontBook, TextElem, TextSize};
-use typst::visualize::Color;
-use typst::{Library, World, WorldExt};
-use walkdir::WalkDir;
 
-// These directories are all relative to the tests/ directory.
-const TYP_DIR: &str = "typ";
-const REF_DIR: &str = "ref";
-const PNG_DIR: &str = "png";
-const PDF_DIR: &str = "pdf";
-const SVG_DIR: &str = "svg";
+use crate::args::{CliArguments, Command};
+use crate::logger::Logger;
 
-/// Arguments that modify test behaviour.
-///
-/// Specify them like this when developing:
-/// `cargo test --workspace --test tests -- --help`
-#[derive(Debug, Clone, Parser)]
-#[clap(name = "typst-test", author)]
-struct Args {
-    /// All the tests that contains a filter string will be run (unless
-    /// `--exact` is specified, which is even stricter).
-    filter: Vec<String>,
-    /// Runs only the specified subtest.
-    #[arg(short, long)]
-    #[arg(allow_hyphen_values = true)]
-    subtest: Option<isize>,
-    /// Runs only the test with the exact name specified in your command.
-    ///
-    /// Example:
-    /// `cargo test --workspace --test tests  -- compiler/bytes.typ --exact`
-    #[arg(long)]
-    exact: bool,
-    /// Updates the reference images in `tests/ref`.
-    #[arg(long, default_value_t = env::var_os("UPDATE_EXPECT").is_some())]
-    update: bool,
-    /// Exports the tests as PDF into `tests/pdf`.
-    #[arg(long)]
-    pdf: bool,
-    /// Configuration of what to print.
-    #[command(flatten)]
-    print: PrintConfig,
-    /// Running `cargo test --workspace -- --nocapture` for the unit tests would
-    /// fail the test runner without argument.
-    // TODO: would it really still happen?
-    #[arg(long)]
-    nocapture: bool,
-    /// Prevents the terminal from being cleared of test names and includes
-    /// non-essential test messages.
-    #[arg(short, long)]
-    verbose: bool,
-}
+/// The parsed command line arguments.
+static ARGS: Lazy<CliArguments> = Lazy::new(CliArguments::parse);
 
-/// Which things to print out for debugging.
-#[derive(Default, Debug, Copy, Clone, Eq, PartialEq, Parser)]
-struct PrintConfig {
-    /// Print the syntax tree.
-    #[arg(long)]
-    syntax: bool,
-    /// Print the content model.
-    #[arg(long)]
-    model: bool,
-    /// Print the layouted frames.
-    #[arg(long)]
-    frames: bool,
-}
+/// The directory where the test suite is located.
+const SUITE_PATH: &str = "tests/suite";
 
-impl Args {
-    fn matches(&self, canonicalized_path: &Path) -> bool {
-        let path = canonicalized_path.to_string_lossy();
-        if !self.exact {
-            return self.filter.is_empty()
-                || self.filter.iter().any(|v| path.contains(v));
-        }
+/// The directory where the full test results are stored.
+const STORE_PATH: &str = "tests/store";
 
-        self.filter.iter().any(|v| match path.strip_suffix(v) {
-            None => false,
-            Some(residual) => {
-                residual.is_empty() || residual.ends_with(MAIN_SEPARATOR_STR)
-            }
-        })
-    }
-}
-
-/// Tests all test files and prints a summary.
-fn main() {
-    let args = Args::parse();
-
-    // Create loader and context.
-    let world = TestWorld::new(args.print);
-
-    println!("Running tests...");
-    let results = WalkDir::new(TYP_DIR)
-        .sort_by_file_name()
-        .into_iter()
-        .par_bridge()
-        .filter_map(|entry| {
-            let entry = entry.unwrap();
-            if entry.depth() == 0 {
-                return None;
-            }
+/// The directory where the reference images are stored.
+const REF_PATH: &str = "tests/ref";
 
-            if entry.path().starts_with("typ/benches") {
-                return None;
-            }
-
-            let src_path = entry.into_path(); // Relative to TYP_DIR.
-            if src_path.extension() != Some(OsStr::new("typ")) {
-                return None;
-            }
-
-            if args.matches(&src_path.canonicalize().unwrap()) {
-                Some(src_path)
-            } else {
-                None
-            }
-        })
-        .map_with(world, |world, src_path| {
-            let path = src_path.strip_prefix(TYP_DIR).unwrap();
-            let png_path = Path::new(PNG_DIR).join(path).with_extension("png");
-            let ref_path = Path::new(REF_DIR).join(path).with_extension("png");
-            let svg_path = Path::new(SVG_DIR).join(path).with_extension("svg");
-            let pdf_path =
-                args.pdf.then(|| Path::new(PDF_DIR).join(path).with_extension("pdf"));
-
-            test(
-                world,
-                &src_path,
-                &png_path,
-                &ref_path,
-                pdf_path.as_deref(),
-                &svg_path,
-                &args,
-            ) as usize
-        })
-        .collect::<Vec<_>>();
-
-    let len = results.len();
-    let ok = results.iter().sum::<usize>();
-    if len > 0 {
-        println!("{ok} / {len} test{} passed.", if len > 1 { "s" } else { "" });
-    } else {
-        println!("No test ran.");
-    }
-
-    if ok != len {
-        println!(
-            "Set the UPDATE_EXPECT environment variable or pass the \
-             --update flag to update the reference image(s)."
-        );
-    }
-
-    if ok < len {
-        std::process::exit(1);
-    }
-}
+/// The maximum size of reference images that aren't marked as `// LARGE`.
+const REF_LIMIT: usize = 20 * 1024;
 
-fn library() -> Library {
-    #[func]
-    fn test(lhs: Value, rhs: Value) -> StrResult<NoneValue> {
-        if lhs != rhs {
-            bail!("Assertion failed: {} != {}", lhs.repr(), rhs.repr());
-        }
-        Ok(NoneValue)
-    }
-
-    #[func]
-    fn test_repr(lhs: Value, rhs: Value) -> StrResult<NoneValue> {
-        if lhs.repr() != rhs.repr() {
-            bail!("Assertion failed: {} != {}", lhs.repr(), rhs.repr());
-        }
-        Ok(NoneValue)
-    }
-
-    #[func]
-    fn print(#[variadic] values: Vec<Value>) -> NoneValue {
-        let mut stdout = io::stdout().lock();
-        write!(stdout, "> ").unwrap();
-        for (i, value) in values.into_iter().enumerate() {
-            if i > 0 {
-                write!(stdout, ", ").unwrap();
-            }
-            write!(stdout, "{value:?}").unwrap();
-        }
-        writeln!(stdout).unwrap();
-        NoneValue
-    }
-
-    // Set page width to 120pt with 10pt margins, so that the inner page is
-    // exactly 100pt wide. Page height is unbounded and font size is 10pt so
-    // that it multiplies to nice round numbers.
-    let mut lib = Library::default();
-    lib.styles
-        .set(PageElem::set_width(Smart::Custom(Abs::pt(120.0).into())));
-    lib.styles.set(PageElem::set_height(Smart::Auto));
-    lib.styles.set(PageElem::set_margin(Margin::splat(Some(Smart::Custom(
-        Abs::pt(10.0).into(),
-    )))));
-    lib.styles.set(TextElem::set_size(TextSize(Abs::pt(10.0).into())));
-
-    // Hook up helpers into the global scope.
-    lib.global.scope_mut().define_func::<test>();
-    lib.global.scope_mut().define_func::<test_repr>();
-    lib.global.scope_mut().define_func::<print>();
-    lib.global
-        .scope_mut()
-        .define("conifer", Color::from_u8(0x9f, 0xEB, 0x52, 0xFF));
-    lib.global
-        .scope_mut()
-        .define("forest", Color::from_u8(0x43, 0xA1, 0x27, 0xFF));
-
-    lib
-}
-
-/// A world that provides access to the tests environment.
-struct TestWorld {
-    print: PrintConfig,
-    main: FileId,
-    library: Prehashed<Library>,
-    book: Prehashed<FontBook>,
-    fonts: Vec<Font>,
-    slots: RwLock<HashMap<FileId, FileSlot>>,
-}
-
-#[derive(Clone)]
-struct FileSlot {
-    source: OnceLock<FileResult<Source>>,
-    buffer: OnceLock<FileResult<Bytes>>,
-}
-
-impl TestWorld {
-    fn new(print: PrintConfig) -> Self {
-        let fonts: Vec<_> = typst_assets::fonts()
-            .chain(typst_dev_assets::fonts())
-            .flat_map(|data| Font::iter(Bytes::from_static(data)))
-            .collect();
-
-        Self {
-            print,
-            main: FileId::new(None, VirtualPath::new("main.typ")),
-            library: Prehashed::new(library()),
-            book: Prehashed::new(FontBook::from_fonts(&fonts)),
-            fonts,
-            slots: RwLock::new(HashMap::new()),
-        }
-    }
-}
-
-impl World for TestWorld {
-    fn library(&self) -> &Prehashed<Library> {
-        &self.library
-    }
-
-    fn book(&self) -> &Prehashed<FontBook> {
-        &self.book
-    }
-
-    fn main(&self) -> Source {
-        self.source(self.main).unwrap()
-    }
-
-    fn source(&self, id: FileId) -> FileResult<Source> {
-        self.slot(id, |slot| {
-            slot.source
-                .get_or_init(|| {
-                    let buf = read(&system_path(id)?)?;
-                    let text = String::from_utf8(buf.into_owned())?;
-                    Ok(Source::new(id, text))
-                })
-                .clone()
-        })
-    }
-
-    fn file(&self, id: FileId) -> FileResult<Bytes> {
-        self.slot(id, |slot| {
-            slot.buffer
-                .get_or_init(|| {
-                    read(&system_path(id)?).map(|cow| match cow {
-                        Cow::Owned(buf) => buf.into(),
-                        Cow::Borrowed(buf) => Bytes::from_static(buf),
-                    })
-                })
-                .clone()
-        })
-    }
-
-    fn font(&self, id: usize) -> Option<Font> {
-        Some(self.fonts[id].clone())
-    }
-
-    fn today(&self, _: Option<i64>) -> Option<Datetime> {
-        Some(Datetime::from_ymd(1970, 1, 1).unwrap())
-    }
-}
-
-impl TestWorld {
-    fn set(&mut self, path: &Path, text: String) -> Source {
-        self.main = FileId::new(None, VirtualPath::new(path));
-        let source = Source::new(self.main, text);
-        self.slot(self.main, |slot| {
-            slot.source = OnceLock::from(Ok(source.clone()));
-            source
-        })
-    }
-
-    fn slot<F, T>(&self, id: FileId, f: F) -> T
-    where
-        F: FnOnce(&mut FileSlot) -> T,
-    {
-        f(self.slots.write().unwrap().entry(id).or_insert_with(|| FileSlot {
-            source: OnceLock::new(),
-            buffer: OnceLock::new(),
-        }))
-    }
-}
+fn main() {
+    setup();
 
-impl Clone for TestWorld {
-    fn clone(&self) -> Self {
-        Self {
-            print: self.print,
-            main: self.main,
-            library: self.library.clone(),
-            book: self.book.clone(),
-            fonts: self.fonts.clone(),
-            slots: RwLock::new(self.slots.read().unwrap().clone()),
-        }
+    match &ARGS.command {
+        None => test(),
+        Some(Command::Clean) => std::fs::remove_dir_all(STORE_PATH).unwrap(),
     }
 }
 
-/// The file system path for a file ID.
-fn system_path(id: FileId) -> FileResult<PathBuf> {
-    let root: PathBuf = match id.package() {
-        Some(spec) => format!("packages/{}-{}", spec.name, spec.version).into(),
-        None => PathBuf::new(),
-    };
-
-    id.vpath().resolve(&root).ok_or(FileError::AccessDenied)
-}
-
-/// Read a file.
-fn read(path: &Path) -> FileResult<Cow<'static, [u8]>> {
-    // Basically symlinks `assets/files` to `tests/files` so that the assets
-    // are within the test project root.
-    let resolved = path.to_path_buf();
-    if let Ok(suffix) = path.strip_prefix("assets/") {
-        return typst_dev_assets::get(&suffix.to_string_lossy())
-            .map(Cow::Borrowed)
-            .ok_or_else(|| FileError::NotFound(path.into()));
-    }
-
-    let f = |e| FileError::from_io(e, path);
-    if fs::metadata(&resolved).map_err(f)?.is_dir() {
-        Err(FileError::IsDirectory)
-    } else {
-        fs::read(&resolved).map(Cow::Owned).map_err(f)
-    }
-}
+fn setup() {
+    // Make all paths relative to the workspace. That's nicer for IDEs when
+    // clicking on paths printed to the terminal.
+    std::env::set_current_dir("..").unwrap();
 
-/// Tests a test file and prints the result.
-///
-/// Also tests that the header of each test is written correctly.
-/// See [parse_part_metadata] for more details.
-fn test(
-    world: &mut TestWorld,
-    src_path: &Path,
-    png_path: &Path,
-    ref_path: &Path,
-    pdf_path: Option<&Path>,
-    svg_path: &Path,
-    args: &Args,
-) -> bool {
-    struct PanicGuard<'a>(&'a Path);
-    impl Drop for PanicGuard<'_> {
-        fn drop(&mut self) {
-            if std::thread::panicking() {
-                println!("Panicked in {}", self.0.display());
-            }
-        }
+    // Create the storage.
+    for ext in ["render", "pdf", "svg"] {
+        std::fs::create_dir_all(Path::new(STORE_PATH).join(ext)).unwrap();
     }
 
-    let name = src_path.strip_prefix(TYP_DIR).unwrap_or(src_path);
-    let text = fs::read_to_string(src_path).unwrap();
-    let _guard = PanicGuard(name);
-
-    let mut output = String::new();
-    let mut ok = true;
-    let mut updated = false;
-    let mut pages = vec![];
-    let mut line = 0;
-    let mut header_configuration = None;
-    let mut compare_ever = false;
-    let mut rng = LinearShift::new();
-
-    let parts: Vec<_> = text
-        .split("\n---")
-        .map(|s| s.strip_suffix('\r').unwrap_or(s))
-        .collect();
-
-    for (i, &part) in parts.iter().enumerate() {
-        if let Some(x) = args.subtest {
-            let x = usize::try_from(
-                x.rem_euclid(isize::try_from(parts.len()).unwrap_or_default()),
-            )
+    // Set up the thread pool.
+    if let Some(num_threads) = ARGS.num_threads {
+        rayon::ThreadPoolBuilder::new()
+            .num_threads(num_threads)
+            .build_global()
             .unwrap();
-            if x != i {
-                writeln!(output, "  Skipped subtest {i}.").unwrap();
-                continue;
-            }
-        }
-        let is_header = i == 0
-            && parts.len() > 1
-            && part
-                .lines()
-                .all(|s| s.starts_with("//") || s.chars().all(|c| c.is_whitespace()));
-
-        if is_header {
-            let source = Source::detached(part.to_string());
-            let metadata = parse_part_metadata(&source, true);
-            match metadata {
-                Ok(metadata) => {
-                    header_configuration = Some(metadata.config);
-                }
-                Err(invalid_data) => {
-                    ok = false;
-                    writeln!(
-                        output,
-                        " Test {}: invalid metadata in header, failing the test:",
-                        name.display()
-                    )
-                    .unwrap();
-                    InvalidMetadata::write(
-                        invalid_data,
-                        &mut output,
-                        &mut |annotation, output| {
-                            print_annotation(output, &source, line, annotation)
-                        },
-                    );
-                }
-            }
-        } else {
-            let (part_ok, compare_here, part_frames) = test_part(
-                &mut output,
-                world,
-                src_path,
-                part.into(),
-                line,
-                i,
-                header_configuration.as_ref().unwrap_or(&Default::default()),
-                &mut rng,
-                args.verbose,
-            );
-
-            ok &= part_ok;
-            compare_ever |= compare_here;
-            pages.extend(part_frames);
-        }
-
-        line += part.lines().count() + 1;
-    }
-
-    let document = Document { pages, ..Default::default() };
-    if compare_ever {
-        if let Some(pdf_path) = pdf_path {
-            let pdf_data = typst_pdf::pdf(
-                &document,
-                Smart::Custom(&format!("typst-test: {}", name.display())),
-                world.today(Some(0)),
-            );
-            fs::create_dir_all(pdf_path.parent().unwrap()).unwrap();
-            fs::write(pdf_path, pdf_data).unwrap();
-        }
-
-        if world.print.frames {
-            for frame in &document.pages {
-                writeln!(output, "{frame:#?}\n").unwrap();
-            }
-        }
-
-        let canvas = render(&document);
-        fs::create_dir_all(png_path.parent().unwrap()).unwrap();
-        canvas.save_png(png_path).unwrap();
-
-        let svg = typst_svg::svg_merged(&document, Abs::pt(5.0));
-
-        fs::create_dir_all(svg_path.parent().unwrap()).unwrap();
-        std::fs::write(svg_path, svg.as_bytes()).unwrap();
-
-        if let Ok(ref_pixmap) = sk::Pixmap::load_png(ref_path) {
-            if canvas.width() != ref_pixmap.width()
-                || canvas.height() != ref_pixmap.height()
-                || canvas
-                    .data()
-                    .iter()
-                    .zip(ref_pixmap.data())
-                    .any(|(&a, &b)| a.abs_diff(b) > 2)
-            {
-                if args.update {
-                    update_image(png_path, ref_path);
-                    updated = true;
-                } else {
-                    writeln!(output, "  Does not match reference image.").unwrap();
-                    ok = false;
-                }
-            }
-        } else if !document.pages.is_empty() {
-            if args.update {
-                update_image(png_path, ref_path);
-                updated = true;
-            } else {
-                writeln!(output, "  Failed to open reference image.").unwrap();
-                ok = false;
-            }
-        }
     }
-
-    {
-        let mut stdout = io::stdout().lock();
-        stdout.write_all(name.to_string_lossy().as_bytes()).unwrap();
-        if ok {
-            writeln!(stdout, " ✔").unwrap();
-            // Don't clear the line when in verbose mode or when the reference image
-            // was updated, to show in the output which test had its image updated.
-            if !updated && !args.verbose && stdout.is_terminal() {
-                // ANSI escape codes: cursor moves up and clears the line.
-                write!(stdout, "\x1b[1A\x1b[2K").unwrap();
-            }
-        } else {
-            writeln!(stdout, " ❌").unwrap();
-        }
-        if updated {
-            writeln!(stdout, "  Updated reference image.").unwrap();
-        }
-        if !output.is_empty() {
-            stdout.write_all(output.as_bytes()).unwrap();
-        }
-    }
-
-    ok
 }
 
-fn update_image(png_path: &Path, ref_path: &Path) {
-    oxipng::optimize(
-        &InFile::Path(png_path.to_owned()),
-        &OutFile::from_path(ref_path.to_owned()),
-        &Options::max_compression(),
-    )
-    .unwrap();
-}
-
-#[allow(clippy::too_many_arguments)]
-fn test_part(
-    output: &mut String,
-    world: &mut TestWorld,
-    src_path: &Path,
-    text: String,
-    line: usize,
-    i: usize,
-    header_configuration: &TestConfig,
-    rng: &mut LinearShift,
-    verbose: bool,
-) -> (bool, bool, Vec<Page>) {
-    let source = world.set(src_path, text);
-    if world.print.syntax {
-        writeln!(output, "Syntax Tree:\n{:#?}\n", source.root()).unwrap();
-    }
-
-    if world.print.model {
-        print_model(world, &source, output);
-    }
-
-    let mut tracer = Tracer::new();
-    let (mut frames, diagnostics) = match typst::compile(world, &mut tracer) {
-        Ok(document) => (document.pages, tracer.warnings()),
+fn test() {
+    let (tests, skipped) = match crate::collect::collect() {
+        Ok(output) => output,
         Err(errors) => {
-            let mut warnings = tracer.warnings();
-            warnings.extend(errors);
-            (vec![], warnings)
-        }
-    };
-
-    let metadata = parse_part_metadata(&source, false);
-    match metadata {
-        Ok(metadata) => {
-            let mut ok = true;
-            let compare_ref = metadata
-                .config
-                .compare_ref
-                .unwrap_or(header_configuration.compare_ref.unwrap_or(true));
-            let validate_hints = metadata
-                .config
-                .validate_hints
-                .unwrap_or(header_configuration.validate_hints.unwrap_or(true));
-            let validate_autocomplete = metadata
-                .config
-                .validate_autocomplete
-                .unwrap_or(header_configuration.validate_autocomplete.unwrap_or(false));
-
-            if verbose {
-                writeln!(output, "Subtest {i} runs with compare_ref={compare_ref}; validate_hints={validate_hints}; validate_autocomplete={validate_autocomplete};").unwrap();
-            }
-            ok &= test_spans(output, source.root());
-            ok &= test_reparse(output, source.text(), i, rng);
-
-            // Don't retain frames if we don't want to compare with reference images.
-            if !compare_ref {
-                frames.clear();
-            }
-
-            // we never check autocomplete and error at the same time
-
-            let diagnostic_annotations = metadata
-                .annotations
-                .iter()
-                .filter(|a| {
-                    !matches!(
-                        a.kind,
-                        AnnotationKind::AutocompleteContains
-                            | AnnotationKind::AutocompleteExcludes
-                    )
-                })
-                .cloned()
-                .collect::<HashSet<_>>();
-
-            if validate_autocomplete {
-                // warns and ignores diagnostics
-                if !diagnostic_annotations.is_empty() {
-                    writeln!(
-                        output,
-                        "  Subtest {i} contains diagnostics but is in autocomplete mode."
-                    )
-                    .unwrap();
-                    for annotation in diagnostic_annotations {
-                        write!(output, "    Ignored | ").unwrap();
-                        print_annotation(output, &source, line, &annotation);
-                    }
-                }
-
-                test_autocomplete(
-                    output,
-                    world,
-                    &source,
-                    line,
-                    i,
-                    &mut ok,
-                    metadata.annotations.iter(),
-                );
-            } else {
-                test_diagnostics(
-                    output,
-                    world,
-                    &source,
-                    line,
-                    i,
-                    &mut ok,
-                    validate_hints,
-                    diagnostics.iter(),
-                    &diagnostic_annotations,
-                );
-            }
-
-            (ok, compare_ref, frames)
-        }
-        Err(invalid_data) => {
-            writeln!(output, "  Subtest {i} has invalid metadata, failing the test:")
-                .unwrap();
-            InvalidMetadata::write(
-                invalid_data,
-                output,
-                &mut |annotation: &Annotation, output: &mut String| {
-                    print_annotation(output, &source, line, annotation)
-                },
-            );
-
-            (false, false, frames)
-        }
-    }
-}
-
-#[allow(clippy::too_many_arguments)]
-fn test_autocomplete<'a>(
-    output: &mut String,
-    world: &mut TestWorld,
-    source: &Source,
-    line: usize,
-    i: usize,
-    ok: &mut bool,
-    annotations: impl Iterator<Item = &'a Annotation>,
-) {
-    for annotation in annotations.filter(|a| {
-        matches!(
-            a.kind,
-            AnnotationKind::AutocompleteContains | AnnotationKind::AutocompleteExcludes
-        )
-    }) {
-        // Ok cause we checked in parsing that range was Some for this annotation
-        let cursor = annotation.range.as_ref().unwrap().start;
-
-        // todo, use document if is_some to test labels autocomplete
-        let completions = typst_ide::autocomplete(world, None, source, cursor, true)
-            .map(|(_, c)| c)
-            .unwrap_or_default()
-            .into_iter()
-            .map(|c| c.label.to_string())
-            .collect::<HashSet<_>>();
-        let completions =
-            completions.iter().map(|s| s.as_str()).collect::<HashSet<&str>>();
-
-        let must_contain_or_exclude = parse_string_list(&annotation.text);
-        let missing =
-            must_contain_or_exclude.difference(&completions).collect::<Vec<_>>();
-
-        if !missing.is_empty()
-            && matches!(annotation.kind, AnnotationKind::AutocompleteContains)
-        {
-            writeln!(output, "  Subtest {i} does not match expected completions.")
-                .unwrap();
-            write!(output, "  for annotation  | ").unwrap();
-            print_annotation(output, source, line, annotation);
-
-            write!(output, "    Not contained  // ").unwrap();
-            for item in missing {
-                write!(output, "{item:?}, ").unwrap()
+            eprintln!("failed to collect tests");
+            for error in errors {
+                eprintln!("❌ {error}");
             }
-            writeln!(output).unwrap();
-            *ok = false;
+            std::process::exit(1);
         }
-
-        let undesired =
-            must_contain_or_exclude.intersection(&completions).collect::<Vec<_>>();
-
-        if !undesired.is_empty()
-            && matches!(annotation.kind, AnnotationKind::AutocompleteExcludes)
-        {
-            writeln!(output, "  Subtest {i} does not match expected completions.")
-                .unwrap();
-            write!(output, "  for annotation  | ").unwrap();
-            print_annotation(output, source, line, annotation);
-
-            write!(output, "    Not excluded  // ").unwrap();
-            for item in undesired {
-                write!(output, "{item:?}, ").unwrap()
-            }
-            writeln!(output).unwrap();
-            *ok = false;
-        }
-    }
-}
-
-#[allow(clippy::too_many_arguments)]
-fn test_diagnostics<'a>(
-    output: &mut String,
-    world: &mut TestWorld,
-    source: &Source,
-    line: usize,
-    i: usize,
-    ok: &mut bool,
-    validate_hints: bool,
-    diagnostics: impl Iterator<Item = &'a SourceDiagnostic>,
-    diagnostic_annotations: &HashSet<Annotation>,
-) {
-    // Map diagnostics to range and message format, discard traces and errors from
-    // other files, collect hints.
-    //
-    // This has one caveat: due to the format of the expected hints, we can not
-    // verify if a hint belongs to a diagnostic or not. That should be irrelevant
-    // however, as the line of the hint is still verified.
-    let mut actual_diagnostics = HashSet::new();
-    for diagnostic in diagnostics {
-        // Ignore diagnostics from other files.
-        if diagnostic.span.id().is_some_and(|id| id != source.id()) {
-            continue;
-        }
-
-        let annotation = Annotation {
-            kind: match diagnostic.severity {
-                Severity::Error => AnnotationKind::Error,
-                Severity::Warning => AnnotationKind::Warning,
-            },
-            range: world.range(diagnostic.span),
-            text: diagnostic.message.replace("\\", "/"),
-        };
-
-        if validate_hints {
-            for hint in &diagnostic.hints {
-                actual_diagnostics.insert(Annotation {
-                    kind: AnnotationKind::Hint,
-                    text: hint.clone(),
-                    range: annotation.range.clone(),
-                });
-            }
-        }
-
-        actual_diagnostics.insert(annotation);
-    }
-
-    // Basically symmetric_difference, but we need to know where an item is coming from.
-    let mut unexpected_outputs = actual_diagnostics
-        .difference(diagnostic_annotations)
-        .collect::<Vec<_>>();
-    let mut missing_outputs = diagnostic_annotations
-        .difference(&actual_diagnostics)
-        .collect::<Vec<_>>();
-
-    unexpected_outputs.sort_by_key(|&v| v.range.as_ref().map(|r| r.start));
-    missing_outputs.sort_by_key(|&v| v.range.as_ref().map(|r| r.start));
-
-    // This prints all unexpected emits first, then all missing emits.
-    // Is this reasonable or subject to change?
-    if !(unexpected_outputs.is_empty() && missing_outputs.is_empty()) {
-        writeln!(output, "  Subtest {i} does not match expected errors.").unwrap();
-        *ok = false;
-
-        for unexpected in unexpected_outputs {
-            write!(output, "    Not annotated // ").unwrap();
-            print_annotation(output, source, line, unexpected)
-        }
-
-        for missing in missing_outputs {
-            write!(output, "    Not emitted   // ").unwrap();
-            print_annotation(output, source, line, missing)
-        }
-    }
-}
-
-fn print_model(world: &mut TestWorld, source: &Source, output: &mut String) {
-    let world = (world as &dyn World).track();
-    let route = typst::engine::Route::default();
-    let mut tracer = typst::eval::Tracer::new();
-
-    let module =
-        typst::eval::eval(world, route.track(), tracer.track_mut(), source).unwrap();
-    writeln!(output, "Model:\n{:#?}\n", module.content()).unwrap();
-}
-
-fn print_annotation(
-    output: &mut String,
-    source: &Source,
-    line: usize,
-    annotation: &Annotation,
-) {
-    let Annotation { range, text, kind } = annotation;
-    write!(output, "{kind}: ").unwrap();
-    if let Some(range) = range {
-        let start_line = 1 + line + source.byte_to_line(range.start).unwrap();
-        let start_col = 1 + source.byte_to_column(range.start).unwrap();
-        let end_line = 1 + line + source.byte_to_line(range.end).unwrap();
-        let end_col = 1 + source.byte_to_column(range.end).unwrap();
-        write!(output, "{start_line}:{start_col}-{end_line}:{end_col} ").unwrap();
-    }
-    writeln!(output, "{text}").unwrap();
-}
-
-/// Pseudorandomly edit the source file and test whether a reparse produces the
-/// same result as a clean parse.
-///
-/// The method will first inject 10 strings once every 400 source characters
-/// and then select 5 leaf node boundaries to inject an additional, randomly
-/// chosen string from the injection list.
-fn test_reparse(
-    output: &mut String,
-    text: &str,
-    i: usize,
-    rng: &mut LinearShift,
-) -> bool {
-    let supplements = [
-        "[",
-        "]",
-        "{",
-        "}",
-        "(",
-        ")",
-        "#rect()",
-        "a word",
-        ", a: 1",
-        "10.0",
-        ":",
-        "if i == 0 {true}",
-        "for",
-        "* hello *",
-        "//",
-        "/*",
-        "\\u{12e4}",
-        "```typst",
-        " ",
-        "trees",
-        "\\",
-        "$ a $",
-        "2.",
-        "-",
-        "5",
-    ];
-
-    let mut ok = true;
-    let mut apply = |replace: Range<usize>, with| {
-        let mut incr_source = Source::detached(text);
-        if incr_source.root().len() != text.len() {
-            println!(
-                "    Subtest {i} tree length {} does not match string length {} ❌",
-                incr_source.root().len(),
-                text.len(),
-            );
-            return false;
-        }
-
-        incr_source.edit(replace.clone(), with);
-
-        let edited_src = incr_source.text();
-        let ref_source = Source::detached(edited_src);
-        let ref_root = ref_source.root();
-        let incr_root = incr_source.root();
-
-        // Ensures that the span numbering invariants hold.
-        let spans_ok = test_spans(output, ref_root) && test_spans(output, incr_root);
-
-        // Ensure that the reference and incremental trees are the same.
-        let tree_ok = ref_root.spanless_eq(incr_root);
-
-        if !tree_ok {
-            writeln!(
-                output,
-                "    Subtest {i} reparse differs from clean parse when inserting '{with}' at {}-{} ❌\n",
-                replace.start, replace.end,
-            ).unwrap();
-            writeln!(output, "    Expected reference tree:\n{ref_root:#?}\n").unwrap();
-            writeln!(output, "    Found incremental tree:\n{incr_root:#?}").unwrap();
-            writeln!(
-                output,
-                "    Full source ({}):\n\"{edited_src:?}\"",
-                edited_src.len()
-            )
-            .unwrap();
-        }
-
-        spans_ok && tree_ok
-    };
-
-    let mut pick = |range: Range<usize>| {
-        let ratio = rng.next();
-        (range.start as f64 + ratio * (range.end - range.start) as f64).floor() as usize
     };
 
-    let insertions = (text.len() as f64 / 400.0).ceil() as usize;
-    for _ in 0..insertions {
-        let supplement = supplements[pick(0..supplements.len())];
-        let start = pick(0..text.len());
-        let end = pick(start..text.len());
-
-        if !text.is_char_boundary(start) || !text.is_char_boundary(end) {
-            continue;
-        }
-
-        ok &= apply(start..end, supplement);
-    }
-
-    let source = Source::detached(text);
-    let leafs = leafs(source.root());
-    let start = source.find(leafs[pick(0..leafs.len())].span()).unwrap().offset();
-    let supplement = supplements[pick(0..supplements.len())];
-    ok &= apply(start..start, supplement);
-
-    ok
-}
-
-/// Returns all leaf descendants of a node (may include itself).
-fn leafs(node: &SyntaxNode) -> Vec<SyntaxNode> {
-    if node.children().len() == 0 {
-        vec![node.clone()]
-    } else {
-        node.children().flat_map(leafs).collect()
-    }
-}
-
-/// Ensure that all spans are properly ordered (and therefore unique).
-#[track_caller]
-fn test_spans(output: &mut String, root: &SyntaxNode) -> bool {
-    test_spans_impl(output, root, 0..u64::MAX)
-}
-
-#[track_caller]
-fn test_spans_impl(output: &mut String, node: &SyntaxNode, within: Range<u64>) -> bool {
-    if !within.contains(&node.span().number()) {
-        writeln!(output, "    Node: {node:#?}").unwrap();
-        writeln!(
-            output,
-            "    Wrong span order: {} not in {within:?} ❌",
-            node.span().number()
-        )
-        .unwrap();
-    }
-
-    let start = node.span().number() + 1;
-    let mut children = node.children().peekable();
-    while let Some(child) = children.next() {
-        let end = children.peek().map_or(within.end, |next| next.span().number());
-        if !test_spans_impl(output, child, start..end) {
-            return false;
-        }
-    }
-
-    true
-}
-
-/// Draw all frames into one image with padding in between.
-fn render(document: &Document) -> sk::Pixmap {
-    let pixel_per_pt = 2.0;
-    let padding = Abs::pt(5.0);
-
-    for page in &document.pages {
-        let limit = Abs::cm(100.0);
-        if page.frame.width() > limit || page.frame.height() > limit {
-            panic!("overlarge frame: {:?}", page.frame.size());
-        }
-    }
-
-    let mut pixmap = typst_render::render_merged(
-        document,
-        pixel_per_pt,
-        Color::WHITE,
-        padding,
-        Color::BLACK,
-    );
-
-    let padding = (pixel_per_pt * padding.to_pt() as f32).round();
-    let [x, mut y] = [padding; 2];
-    for page in &document.pages {
-        let ts =
-            sk::Transform::from_scale(pixel_per_pt, pixel_per_pt).post_translate(x, y);
-        render_links(&mut pixmap, ts, &page.frame);
-        y += (pixel_per_pt * page.frame.height().to_pt() as f32).round().max(1.0)
-            + padding;
-    }
-
-    pixmap
-}
-
-/// Draw extra boxes for links so we can see whether they are there.
-fn render_links(canvas: &mut sk::Pixmap, ts: sk::Transform, frame: &Frame) {
-    for (pos, item) in frame.items() {
-        let ts = ts.pre_translate(pos.x.to_pt() as f32, pos.y.to_pt() as f32);
-        match *item {
-            FrameItem::Group(ref group) => {
-                let ts = ts.pre_concat(to_sk_transform(&group.transform));
-                render_links(canvas, ts, &group.frame);
-            }
-            FrameItem::Meta(Meta::Link(_), size) => {
-                let w = size.x.to_pt() as f32;
-                let h = size.y.to_pt() as f32;
-                let rect = sk::Rect::from_xywh(0.0, 0.0, w, h).unwrap();
-                let mut paint = sk::Paint::default();
-                paint.set_color_rgba8(40, 54, 99, 40);
-                canvas.fill_rect(rect, &paint, ts, None);
-            }
-            _ => {}
-        }
-    }
-}
-
-fn to_sk_transform(transform: &Transform) -> sk::Transform {
-    let Transform { sx, ky, kx, sy, tx, ty } = *transform;
-    sk::Transform::from_row(
-        sx.get() as _,
-        ky.get() as _,
-        kx.get() as _,
-        sy.get() as _,
-        tx.to_pt() as f32,
-        ty.to_pt() as f32,
-    )
-}
-
-/// A Linear-feedback shift register using XOR as its shifting function.
-/// Can be used as PRNG.
-struct LinearShift(u64);
-
-impl LinearShift {
-    /// Initialize the shift register with a pre-set seed.
-    pub fn new() -> Self {
-        Self(0xACE5)
-    }
-
-    /// Return a pseudo-random number between `0.0` and `1.0`.
-    pub fn next(&mut self) -> f64 {
-        self.0 ^= self.0 >> 3;
-        self.0 ^= self.0 << 14;
-        self.0 ^= self.0 >> 28;
-        self.0 ^= self.0 << 36;
-        self.0 ^= self.0 >> 52;
-        self.0 as f64 / u64::MAX as f64
+    let filtered = tests.len();
+    if filtered == 0 {
+        eprintln!("no test selected");
+        return;
+    }
+
+    // Run the tests.
+    let logger = Mutex::new(Logger::new(filtered, skipped));
+    std::thread::scope(|scope| {
+        let logger = &logger;
+        let (sender, receiver) = std::sync::mpsc::channel();
+
+        // Regularly refresh the logger in case we make no progress.
+        scope.spawn(move || {
+            while receiver.recv_timeout(Duration::from_millis(500)).is_err() {
+                logger.lock().refresh();
+            }
+        });
+
+        // Run the tests.
+        //
+        // We use `par_bridge` instead of `par_iter` because the former
+        // results in a stack overflow during PDF export. Probably related
+        // to `typst::util::Deferred` yielding.
+        tests.iter().par_bridge().for_each(|test| {
+            logger.lock().start(test);
+            let result = std::panic::catch_unwind(|| run::run(test));
+            logger.lock().end(test, result);
+        });
+
+        sender.send(()).unwrap();
+    });
+
+    let passed = logger.into_inner().finish();
+    if !passed {
+        std::process::exit(1);
     }
 }
diff --git a/tests/src/world.rs b/tests/src/world.rs
new file mode 100644
index 00000000..86ee8da6
--- /dev/null
+++ b/tests/src/world.rs
@@ -0,0 +1,229 @@
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::fs;
+use std::io::Write;
+use std::path::{Path, PathBuf};
+use std::sync::OnceLock;
+
+use comemo::Prehashed;
+use once_cell::sync::Lazy;
+use parking_lot::Mutex;
+use typst::diag::{bail, FileError, FileResult, StrResult};
+use typst::foundations::{func, Bytes, Datetime, NoneValue, Repr, Smart, Value};
+use typst::layout::{Abs, Margin, PageElem};
+use typst::syntax::{FileId, Source};
+use typst::text::{Font, FontBook, TextElem, TextSize};
+use typst::visualize::Color;
+use typst::{Library, World};
+
+/// A world that provides access to the tests environment.
+#[derive(Clone)]
+pub struct TestWorld {
+    main: Source,
+    base: &'static TestBase,
+}
+
+impl TestWorld {
+    /// Create a new world for a single test.
+    ///
+    /// This is cheap because the shared base for all test runs is lazily
+    /// initialized just once.
+    pub fn new(source: Source) -> Self {
+        static BASE: Lazy<TestBase> = Lazy::new(TestBase::default);
+        Self { main: source, base: &*BASE }
+    }
+}
+
+impl World for TestWorld {
+    fn library(&self) -> &Prehashed<Library> {
+        &self.base.library
+    }
+
+    fn book(&self) -> &Prehashed<FontBook> {
+        &self.base.book
+    }
+
+    fn main(&self) -> Source {
+        self.main.clone()
+    }
+
+    fn source(&self, id: FileId) -> FileResult<Source> {
+        if id == self.main.id() {
+            Ok(self.main.clone())
+        } else {
+            self.slot(id, FileSlot::source)
+        }
+    }
+
+    fn file(&self, id: FileId) -> FileResult<Bytes> {
+        self.slot(id, FileSlot::file)
+    }
+
+    fn font(&self, index: usize) -> Option<Font> {
+        Some(self.base.fonts[index].clone())
+    }
+
+    fn today(&self, _: Option<i64>) -> Option<Datetime> {
+        Some(Datetime::from_ymd(1970, 1, 1).unwrap())
+    }
+}
+
+impl TestWorld {
+    /// Access the canonical slot for the given file id.
+    fn slot<F, T>(&self, id: FileId, f: F) -> T
+    where
+        F: FnOnce(&mut FileSlot) -> T,
+    {
+        let mut map = self.base.slots.lock();
+        f(map.entry(id).or_insert_with(|| FileSlot::new(id)))
+    }
+}
+
+/// Shared foundation of all test worlds.
+struct TestBase {
+    library: Prehashed<Library>,
+    book: Prehashed<FontBook>,
+    fonts: Vec<Font>,
+    slots: Mutex<HashMap<FileId, FileSlot>>,
+}
+
+impl Default for TestBase {
+    fn default() -> Self {
+        let fonts: Vec<_> = typst_assets::fonts()
+            .chain(typst_dev_assets::fonts())
+            .flat_map(|data| Font::iter(Bytes::from_static(data)))
+            .collect();
+
+        Self {
+            library: Prehashed::new(library()),
+            book: Prehashed::new(FontBook::from_fonts(&fonts)),
+            fonts,
+            slots: Mutex::new(HashMap::new()),
+        }
+    }
+}
+
+/// Holds the processed data for a file ID.
+#[derive(Clone)]
+struct FileSlot {
+    id: FileId,
+    source: OnceLock<FileResult<Source>>,
+    file: OnceLock<FileResult<Bytes>>,
+}
+
+impl FileSlot {
+    /// Create a new file slot.
+    fn new(id: FileId) -> Self {
+        Self { id, file: OnceLock::new(), source: OnceLock::new() }
+    }
+
+    /// Retrieve the source for this file.
+    fn source(&mut self) -> FileResult<Source> {
+        self.source
+            .get_or_init(|| {
+                let buf = read(&system_path(self.id)?)?;
+                let text = String::from_utf8(buf.into_owned())?;
+                Ok(Source::new(self.id, text))
+            })
+            .clone()
+    }
+
+    /// Retrieve the file's bytes.
+    fn file(&mut self) -> FileResult<Bytes> {
+        self.file
+            .get_or_init(|| {
+                read(&system_path(self.id)?).map(|cow| match cow {
+                    Cow::Owned(buf) => buf.into(),
+                    Cow::Borrowed(buf) => Bytes::from_static(buf),
+                })
+            })
+            .clone()
+    }
+}
+
+/// The file system path for a file ID.
+fn system_path(id: FileId) -> FileResult<PathBuf> {
+    let root: PathBuf = match id.package() {
+        Some(spec) => format!("tests/packages/{}-{}", spec.name, spec.version).into(),
+        None => PathBuf::new(),
+    };
+
+    id.vpath().resolve(&root).ok_or(FileError::AccessDenied)
+}
+
+/// Read a file.
+fn read(path: &Path) -> FileResult<Cow<'static, [u8]>> {
+    // Resolve asset.
+    if let Ok(suffix) = path.strip_prefix("assets/") {
+        return typst_dev_assets::get(&suffix.to_string_lossy())
+            .map(Cow::Borrowed)
+            .ok_or_else(|| FileError::NotFound(path.into()));
+    }
+
+    let f = |e| FileError::from_io(e, path);
+    if fs::metadata(path).map_err(f)?.is_dir() {
+        Err(FileError::IsDirectory)
+    } else {
+        fs::read(path).map(Cow::Owned).map_err(f)
+    }
+}
+
+/// The extended standard library for testing.
+fn library() -> Library {
+    // Set page width to 120pt with 10pt margins, so that the inner page is
+    // exactly 100pt wide. Page height is unbounded and font size is 10pt so
+    // that it multiplies to nice round numbers.
+    let mut lib = Library::default();
+
+    #[func]
+    fn test(lhs: Value, rhs: Value) -> StrResult<NoneValue> {
+        if lhs != rhs {
+            bail!("Assertion failed: {} != {}", lhs.repr(), rhs.repr());
+        }
+        Ok(NoneValue)
+    }
+
+    #[func]
+    fn test_repr(lhs: Value, rhs: Value) -> StrResult<NoneValue> {
+        if lhs.repr() != rhs.repr() {
+            bail!("Assertion failed: {} != {}", lhs.repr(), rhs.repr());
+        }
+        Ok(NoneValue)
+    }
+
+    #[func]
+    fn print(#[variadic] values: Vec<Value>) -> NoneValue {
+        let mut out = std::io::stdout().lock();
+        write!(out, "> ").unwrap();
+        for (i, value) in values.into_iter().enumerate() {
+            if i > 0 {
+                write!(out, ", ").unwrap();
+            }
+            write!(out, "{value:?}").unwrap();
+        }
+        writeln!(out).unwrap();
+        NoneValue
+    }
+
+    // Hook up helpers into the global scope.
+    lib.global.scope_mut().define_func::<test>();
+    lib.global.scope_mut().define_func::<test_repr>();
+    lib.global.scope_mut().define_func::<print>();
+    lib.global
+        .scope_mut()
+        .define("conifer", Color::from_u8(0x9f, 0xEB, 0x52, 0xFF));
+    lib.global
+        .scope_mut()
+        .define("forest", Color::from_u8(0x43, 0xA1, 0x27, 0xFF));
+
+    // Hook up default styles.
+    lib.styles
+        .set(PageElem::set_width(Smart::Custom(Abs::pt(120.0).into())));
+    lib.styles.set(PageElem::set_height(Smart::Auto));
+    lib.styles.set(PageElem::set_margin(Margin::splat(Some(Smart::Custom(
+        Abs::pt(10.0).into(),
+    )))));
+    lib.styles.set(TextElem::set_size(TextSize(Abs::pt(10.0).into())));
+
+    lib
+}