diff options
| -rw-r--r-- | Cargo.lock | 1 | ||||
| -rw-r--r-- | tests/Cargo.toml | 1 | ||||
| -rw-r--r-- | tests/README.md | 49 | ||||
| -rw-r--r-- | tests/src/args.rs | 23 | ||||
| -rw-r--r-- | tests/src/collect.rs | 25 | ||||
| -rw-r--r-- | tests/src/logger.rs | 14 | ||||
| -rw-r--r-- | tests/src/tests.rs | 12 |
7 files changed, 85 insertions, 40 deletions
@@ -2780,6 +2780,7 @@ dependencies = [ "oxipng", "parking_lot", "rayon", + "regex", "tiny-skia", "ttf-parser", "typst", diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 62e7a493..2923f4d0 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -25,6 +25,7 @@ once_cell = { workspace = true } oxipng = { workspace = true } parking_lot = { workspace = true } rayon = { workspace = true } +regex = { workspace = true } tiny-skia = { workspace = true } ttf-parser = { workspace = true } unscanny = { workspace = true } diff --git a/tests/README.md b/tests/README.md index f817e8f9..e7713eed 100644 --- a/tests/README.md +++ b/tests/README.md @@ -3,7 +3,9 @@ ## Directory structure Top level directory structure: - `src`: Testing code. -- `suite`: Input files. Mostly organize in parallel to the code. +- `suite`: Input files. Mostly organized in parallel to the code. Each file can + contain multiple tests, each of which is a section of Typst code + following `--- {name} ---`. - `ref`: Reference images which the output is compared with to determine whether a test passed or failed. - `store`: Store for PNG, PDF, and SVG output files produced by the tests. @@ -19,17 +21,27 @@ Running just the integration tests (the tests in this directory): cargo test --workspace --test tests ``` -You may want to [make yourself an alias](#making-an-alias) like: +You may want to [make yourself an alias](#making-an-alias) `testit` so that you can +write shorter commands. In the examples below, we will use this alias. + +Running all tests with the given name pattern. You can use +[regular expression](https://docs.rs/regex/latest/regex/)s. +```bash +testit math # The name has "math" anywhere +testit math page # The name has "math" or "page" anywhere +testit "^math" "^page" # The name begins with "math" or "page" +testit "^(math|page)" # Same as above. +``` + +Running all tests discovered under given paths: ```bash -testit +testit -p tests/suite/math/attach.typ +testit -p tests/suite/model -p tests/suite/text ``` -Running all tests whose names contain the string `page` or `stack`. Note each -`.typ` file in this directory can contain multiple tests, each of which is a -section of Typst code following `--- {name} ---`. +Running tests that begin with `issue` under a given path: ```bash -# Add --verbose to list which tests were run. -testit page stack +testit "^issue" -p tests/suite/model ``` Running a test with the exact test name `math-attach-mixed`. @@ -37,6 +49,11 @@ Running a test with the exact test name `math-attach-mixed`. testit --exact math-attach-mixed ``` +You may find more options in the help message: +```bash +testit --help +``` + To make the integration tests go faster they don't generate PDFs by default. Pass the `--pdf` flag to generate those. Mind that PDFs are not tested automatically at the moment, so you should always check the output manually when @@ -56,12 +73,12 @@ There are, broadly speaking, three kinds of tests: use of `test` or `assert.eq` (both are very similar, `test` is just shorter) to ensure certain properties hold when executing the Typst code. -- Tests that ensure the code fails with a particular error: Those have inline - annotations like `// Error: 2-7 thing was wrong`. An annotation can be - either an "Error", a "Warning", or a "Hint". The range designates where - in the next non-comment line the error is and after it follows the message. - If you the error is in a line further below, you can also write ranges like - `3:2-3:7` to indicate the 2-7 column in the 3rd non-comment line. +- Tests that ensure the code emits particular diagnostic messages: Those have + inline annotations like `// Error: 2-7 thing was wrong`. An annotation can + start with either "Error", "Warning", or "Hint". The range designates the + code span the diagnostic message refers to in the first non-comment line + below. If the code span is in a line further below, you can write ranges + like `3:2-3:7` to indicate the 2-7 column in the 3rd non-comment line. - Tests that ensure certain visual output is produced: Those render the result of the test with the `typst-render` crate and compare against a reference @@ -82,7 +99,7 @@ If you created a new test or fixed a bug in an existing test, you need to update the reference image used for comparison. For this, you can use the `--update` flag: ```bash -testit mytest --update +testit --exact my-test-name --update ``` If you use the VS Code test helper extension (see the `tools` folder), you can @@ -92,7 +109,7 @@ alternatively use the save button to update the reference image. If you want to have a quicker way to run the tests, consider adding a shortcut to your shell profile so that you can simply write something like: ```bash -testit empty.typ +testit --exact my-test-name ``` ### Bash diff --git a/tests/src/args.rs b/tests/src/args.rs index fcd4ead1..33935edf 100644 --- a/tests/src/args.rs +++ b/tests/src/args.rs @@ -1,19 +1,30 @@ +use std::path::PathBuf; + use clap::{Parser, Subcommand}; +use regex::Regex; /// Typst's test runner. #[derive(Debug, Clone, Parser)] +#[command(bin_name = "cargo test --workspace --test tests --")] #[clap(name = "typst-test", author)] pub struct CliArguments { /// The command to run. #[command(subcommand)] pub command: Option<Command>, - /// All the tests that contain the filter string will be run. - pub filter: Vec<String>, - /// Runs only the tests with the exact specified `filter` names. + /// All the tests whose names match the test name pattern will be run. + #[arg(value_parser = Regex::new)] + pub pattern: Vec<Regex>, + /// Restricts test selection within the given path. + #[arg(short, long, value_parser = |s: &str| PathBuf::from(s).canonicalize())] + pub path: Vec<PathBuf>, + /// Only selects the test that matches with the test name verbatim. #[arg(short, long)] pub exact: bool, - /// Whether to update the reference images of non-passing tests. - #[arg(short, long)] + /// Lists what tests will be run, without actually running them. + #[arg(long, group = "action")] + pub list: bool, + /// Updates the reference images of non-passing tests. + #[arg(short, long, group = "action")] pub update: bool, /// The scaling factor to render the output image with. /// @@ -26,7 +37,7 @@ pub struct CliArguments { /// Exports SVG outputs into the artifact store. #[arg(long)] pub svg: bool, - /// Whether to display the syntax tree. + /// Displays the syntax tree. #[arg(long)] pub syntax: bool, /// Prevents the terminal from being cleared of test names. diff --git a/tests/src/collect.rs b/tests/src/collect.rs index 44a325f2..ee4f9db9 100644 --- a/tests/src/collect.rs +++ b/tests/src/collect.rs @@ -257,7 +257,7 @@ impl<'a> Parser<'a> { self.collector.large.insert(name.clone()); } - if !filtered(&name) { + if !selected(&name, self.path.canonicalize().unwrap()) { self.collector.skipped += 1; continue; } @@ -383,14 +383,23 @@ impl<'a> Parser<'a> { } } -/// Whether a test is within the filtered set. -fn filtered(name: &str) -> bool { +/// Whether a test is within the selected set to run. +fn selected(name: &str, abs: PathBuf) -> bool { + let paths = &crate::ARGS.path; + if !paths.is_empty() && !paths.iter().any(|path| abs.starts_with(path)) { + return false; + } + let exact = crate::ARGS.exact; - let filter = &crate::ARGS.filter; - filter.is_empty() - || filter - .iter() - .any(|v| if exact { name == v } else { name.contains(v) }) + let patterns = &crate::ARGS.pattern; + patterns.is_empty() + || patterns.iter().any(|pattern: ®ex::Regex| { + if exact { + name == pattern.as_str() + } else { + pattern.is_match(name) + } + }) } /// An error in a test file. diff --git a/tests/src/logger.rs b/tests/src/logger.rs index c48650a7..1acf7c14 100644 --- a/tests/src/logger.rs +++ b/tests/src/logger.rs @@ -6,7 +6,7 @@ use crate::run::TestResult; /// Receives status updates by individual test runs. pub struct Logger<'a> { - filtered: usize, + selected: usize, passed: usize, failed: usize, skipped: usize, @@ -19,9 +19,9 @@ pub struct Logger<'a> { impl<'a> Logger<'a> { /// Create a new logger. - pub fn new(filtered: usize, skipped: usize) -> Self { + pub fn new(selected: usize, skipped: usize) -> Self { Self { - filtered, + selected, passed: 0, failed: 0, skipped, @@ -86,10 +86,10 @@ impl<'a> Logger<'a> { /// Prints a summary and returns whether the test suite passed. pub fn finish(&self) -> bool { - let Self { filtered, passed, failed, skipped, .. } = *self; + let Self { selected, passed, failed, skipped, .. } = *self; eprintln!("{passed} passed, {failed} failed, {skipped} skipped"); - assert_eq!(filtered, passed + failed, "not all tests were executed succesfully"); + assert_eq!(selected, passed + failed, "not all tests were executed succesfully"); if self.mismatched_image { eprintln!(" pass the --update flag to update the reference images"); @@ -121,7 +121,7 @@ impl<'a> Logger<'a> { // Print the status line. let done = self.failed + self.passed; - if done < self.filtered { + if done < self.selected { if self.last_change.elapsed() > Duration::from_secs(2) { for test in &self.active { writeln!(out, "⏰ {test} is taking a long time ...")?; @@ -131,7 +131,7 @@ impl<'a> Logger<'a> { } } if self.terminal { - writeln!(out, "💨 {done} / {}", self.filtered)?; + writeln!(out, "💨 {done} / {}", self.selected)?; self.temp_lines += 1; } } diff --git a/tests/src/tests.rs b/tests/src/tests.rs index 6d58e969..19d9e5e9 100644 --- a/tests/src/tests.rs +++ b/tests/src/tests.rs @@ -72,14 +72,20 @@ fn test() { } }; - let filtered = tests.len(); - if filtered == 0 { + let selected = tests.len(); + if ARGS.list { + for test in tests.iter() { + println!("{test}"); + } + eprintln!("{selected} selected, {skipped} skipped"); + return; + } else if selected == 0 { eprintln!("no test selected"); return; } // Run the tests. - let logger = Mutex::new(Logger::new(filtered, skipped)); + let logger = Mutex::new(Logger::new(selected, skipped)); std::thread::scope(|scope| { let logger = &logger; let (sender, receiver) = std::sync::mpsc::channel(); |