diff options
Diffstat (limited to 'tests/src')
| -rw-r--r-- | tests/src/args.rs | 4 | ||||
| -rw-r--r-- | tests/src/collect.rs | 60 | ||||
| -rw-r--r-- | tests/src/logger.rs | 14 | ||||
| -rw-r--r-- | tests/src/run.rs | 312 | ||||
| -rw-r--r-- | tests/src/tests.rs | 12 | ||||
| -rw-r--r-- | tests/src/world.rs | 6 |
6 files changed, 270 insertions, 138 deletions
diff --git a/tests/src/args.rs b/tests/src/args.rs index db5d1a9b..3d1bc28c 100644 --- a/tests/src/args.rs +++ b/tests/src/args.rs @@ -23,7 +23,7 @@ pub struct CliArguments { /// Lists what tests will be run, without actually running them. #[arg(long, group = "action")] pub list: bool, - /// Updates the reference images of non-passing tests. + /// Updates the reference output of non-passing tests. #[arg(short, long, group = "action")] pub update: bool, /// The scaling factor to render the output image with. @@ -100,6 +100,6 @@ impl CliArguments { pub enum Command { /// Clears the on-disk test artifact store. Clean, - /// Deletes all dangling reference images. + /// Deletes all dangling reference output. Undangle, } diff --git a/tests/src/collect.rs b/tests/src/collect.rs index 5c7327f1..c6deba77 100644 --- a/tests/src/collect.rs +++ b/tests/src/collect.rs @@ -23,9 +23,9 @@ pub fn collect() -> Result<(Vec<Test>, usize), Vec<TestParseError>> { pub struct Test { pub pos: FilePos, pub name: EcoString, + pub attrs: Vec<Attr>, pub source: Source, pub notes: Vec<Note>, - pub large: bool, } impl Display for Test { @@ -57,6 +57,14 @@ impl Display for FilePos { } } +/// A test attribute, given after the test name. +#[derive(Clone, Debug, PartialEq)] +pub enum Attr { + Html, + Render, + Large, +} + /// The size of a file. pub struct FileSize(pub usize); @@ -109,8 +117,7 @@ impl Display for NoteKind { struct Collector { tests: Vec<Test>, errors: Vec<TestParseError>, - seen: HashMap<EcoString, FilePos>, - large: HashSet<EcoString>, + seen: HashMap<EcoString, (FilePos, Vec<Attr>)>, skipped: usize, } @@ -121,7 +128,6 @@ impl Collector { tests: vec![], errors: vec![], seen: HashMap::new(), - large: HashSet::new(), skipped: 0, } } @@ -156,7 +162,7 @@ impl Collector { } } - /// Walks through all reference images and ensure that a test exists for + /// Walks through all reference output and ensures that a test exists for /// each one. fn walk_references(&mut self) { for entry in walkdir::WalkDir::new(crate::REF_PATH).sort_by_file_name() { @@ -169,20 +175,20 @@ impl Collector { let stem = path.file_stem().unwrap().to_string_lossy(); let name = &*stem; - let Some(pos) = self.seen.get(name) else { + let Some((pos, attrs)) = self.seen.get(name) else { self.errors.push(TestParseError { pos: FilePos::new(path, 0), - message: "dangling reference image".into(), + message: "dangling reference output".into(), }); continue; }; let len = path.metadata().unwrap().len() as usize; - if !self.large.contains(name) && len > crate::REF_LIMIT { + if !attrs.contains(&Attr::Large) && len > crate::REF_LIMIT { self.errors.push(TestParseError { pos: pos.clone(), message: format!( - "reference image size exceeds {}, but the test is not marked as `// LARGE`", + "reference output size exceeds {}, but the test is not marked as `large`", FileSize(crate::REF_LIMIT), ), }); @@ -218,6 +224,7 @@ impl<'a> Parser<'a> { while !self.s.done() { let mut name = EcoString::new(); + let mut attrs = Vec::new(); let mut notes = vec![]; if self.s.eat_if("---") { self.s.eat_while(' '); @@ -228,8 +235,8 @@ impl<'a> Parser<'a> { self.error("expected test name"); } else if !is_ident(&name) { self.error(format!("test name `{name}` is not a valid identifier")); - } else if !self.s.eat_if("---") { - self.error("expected closing ---"); + } else { + attrs = self.parse_attrs(); } } else { self.error("expected opening ---"); @@ -247,7 +254,7 @@ impl<'a> Parser<'a> { self.test_start_line = self.line; let pos = FilePos::new(self.path, self.test_start_line); - self.collector.seen.insert(name.clone(), pos.clone()); + self.collector.seen.insert(name.clone(), (pos.clone(), attrs.clone())); while !self.s.done() && !self.s.at("---") { self.s.eat_until(is_newline); @@ -257,10 +264,6 @@ impl<'a> Parser<'a> { } let text = self.s.from(start); - let large = text.starts_with("// LARGE"); - if large { - self.collector.large.insert(name.clone()); - } if !selected(&name, self.path.canonicalize().unwrap()) { self.collector.skipped += 1; @@ -285,8 +288,31 @@ impl<'a> Parser<'a> { } } - self.collector.tests.push(Test { pos, name, source, notes, large }); + self.collector.tests.push(Test { pos, name, source, notes, attrs }); + } + } + + fn parse_attrs(&mut self) -> Vec<Attr> { + let mut attrs = vec![]; + while !self.s.eat_if("---") { + let attr = match self.s.eat_until(char::is_whitespace) { + "large" => Attr::Large, + "html" => Attr::Html, + "render" => Attr::Render, + found => { + self.error(format!( + "expected attribute or closing ---, found `{found}`" + )); + break; + } + }; + if attrs.contains(&attr) { + self.error(format!("duplicate attribute {attr:?}")); + } + attrs.push(attr); + self.s.eat_while(' '); } + attrs } /// Skips the preamble of a test. diff --git a/tests/src/logger.rs b/tests/src/logger.rs index 48bad451..18eefa5f 100644 --- a/tests/src/logger.rs +++ b/tests/src/logger.rs @@ -9,8 +9,8 @@ pub struct TestResult { pub errors: String, /// The info log for this test. pub infos: String, - /// Whether the image was mismatched. - pub mismatched_image: bool, + /// Whether the output was mismatched. + pub mismatched_output: bool, } /// Receives status updates by individual test runs. @@ -19,7 +19,7 @@ pub struct Logger<'a> { passed: usize, failed: usize, skipped: usize, - mismatched_image: bool, + mismatched_output: bool, active: Vec<&'a Test>, last_change: Instant, temp_lines: usize, @@ -34,7 +34,7 @@ impl<'a> Logger<'a> { passed: 0, failed: 0, skipped, - mismatched_image: false, + mismatched_output: false, active: vec![], temp_lines: 0, last_change: Instant::now(), @@ -73,7 +73,7 @@ impl<'a> Logger<'a> { self.failed += 1; } - self.mismatched_image |= result.mismatched_image; + self.mismatched_output |= result.mismatched_output; self.last_change = Instant::now(); self.print(move |out| { @@ -102,8 +102,8 @@ impl<'a> Logger<'a> { eprintln!("{passed} passed, {failed} failed, {skipped} skipped"); assert_eq!(selected, passed + failed, "not all tests were executed successfully"); - if self.mismatched_image { - eprintln!(" pass the --update flag to update the reference images"); + if self.mismatched_output { + eprintln!(" pass the --update flag to update the reference output"); } self.failed == 0 diff --git a/tests/src/run.rs b/tests/src/run.rs index 1aa70204..f9a3c043 100644 --- a/tests/src/run.rs +++ b/tests/src/run.rs @@ -1,16 +1,17 @@ use std::fmt::Write; use std::ops::Range; -use std::path::Path; +use std::path::PathBuf; use ecow::eco_vec; use tiny_skia as sk; use typst::diag::{SourceDiagnostic, Warned}; -use typst::layout::{Abs, Frame, FrameItem, Page, PagedDocument, Transform}; +use typst::html::HtmlDocument; +use typst::layout::{Abs, Frame, FrameItem, PagedDocument, Transform}; use typst::visualize::Color; -use typst::WorldExt; +use typst::{Document, WorldExt}; use typst_pdf::PdfOptions; -use crate::collect::{FileSize, NoteKind, Test}; +use crate::collect::{Attr, FileSize, NoteKind, Test}; use crate::logger::TestResult; use crate::world::TestWorld; @@ -50,7 +51,7 @@ impl<'a> Runner<'a> { result: TestResult { errors: String::new(), infos: String::new(), - mismatched_image: false, + mismatched_output: false, }, not_annotated: String::new(), } @@ -62,6 +63,23 @@ impl<'a> Runner<'a> { log!(into: self.result.infos, "tree: {:#?}", self.test.source.root()); } + let html = self.test.attrs.contains(&Attr::Html); + let render = !html || self.test.attrs.contains(&Attr::Render); + if render { + self.run_test::<PagedDocument>(); + } + if html { + self.run_test::<HtmlDocument>(); + } + + self.handle_not_emitted(); + self.handle_not_annotated(); + + self.result + } + + /// Run test specific to document format. + fn run_test<D: OutputType>(&mut self) { let Warned { output, warnings } = typst::compile(&self.world); let (doc, errors) = match output { Ok(doc) => (Some(doc), eco_vec![]), @@ -72,8 +90,8 @@ impl<'a> Runner<'a> { log!(self, "no document, but also no errors"); } - self.check_custom(doc.as_ref()); - self.check_document(doc.as_ref()); + D::check_custom(self, doc.as_ref()); + self.check_output(doc.as_ref()); for error in &errors { self.check_diagnostic(NoteKind::Error, error); @@ -82,11 +100,6 @@ impl<'a> Runner<'a> { for warning in &warnings { self.check_diagnostic(NoteKind::Warning, warning); } - - self.handle_not_emitted(); - self.handle_not_annotated(); - - self.result } /// Handle errors that weren't annotated. @@ -113,86 +126,42 @@ impl<'a> Runner<'a> { } } - /// Run custom checks for which it is not worth to create special - /// annotations. - fn check_custom(&mut self, doc: Option<&PagedDocument>) { - let errors = crate::custom::check(self.test, &self.world, doc); - if !errors.is_empty() { - log!(self, "custom check failed"); - for line in errors.lines() { - log!(self, " {line}"); - } - } - } - /// Check that the document output is correct. - fn check_document(&mut self, document: Option<&PagedDocument>) { - let live_path = format!("{}/render/{}.png", crate::STORE_PATH, self.test.name); - let ref_path = format!("{}/{}.png", crate::REF_PATH, self.test.name); - let has_ref = Path::new(&ref_path).exists(); + fn check_output<D: OutputType>(&mut self, document: Option<&D>) { + let live_path = D::live_path(&self.test.name); + let ref_path = D::ref_path(&self.test.name); + let ref_data = std::fs::read(&ref_path); let Some(document) = document else { - if has_ref { + if ref_data.is_ok() { log!(self, "missing document"); - log!(self, " ref | {ref_path}"); + log!(self, " ref | {}", ref_path.display()); } return; }; - let skippable = match document.pages.as_slice() { - [] => { + let skippable = match D::is_skippable(document) { + Ok(skippable) => skippable, + Err(()) => { log!(self, "document has zero pages"); return; } - [page] => skippable(page), - _ => false, }; - // Tests without visible output and no reference image don't need to be + // Tests without visible output and no reference output don't need to be // compared. - if skippable && !has_ref { + if skippable && ref_data.is_err() { std::fs::remove_file(&live_path).ok(); return; } - // Render the live version. - let pixmap = render(document, 1.0); - - // Save live version, possibly rerendering if different scale is - // requested. - let mut pixmap_live = &pixmap; - let slot; - let scale = crate::ARGS.scale; - if scale != 1.0 { - slot = render(document, scale); - pixmap_live = &slot; - } - let data = pixmap_live.encode_png().unwrap(); - std::fs::write(&live_path, data).unwrap(); - - // Write PDF if requested. - if crate::ARGS.pdf() { - let pdf_path = format!("{}/pdf/{}.pdf", crate::STORE_PATH, self.test.name); - let pdf = typst_pdf::pdf(document, &PdfOptions::default()).unwrap(); - std::fs::write(pdf_path, pdf).unwrap(); - } - - // Write SVG if requested. - if crate::ARGS.svg() { - let svg_path = format!("{}/svg/{}.svg", crate::STORE_PATH, self.test.name); - let svg = typst_svg::svg_merged(document, Abs::pt(5.0)); - std::fs::write(svg_path, svg).unwrap(); - } - - // Compare against reference image if available. - let equal = has_ref && { - let ref_data = std::fs::read(&ref_path).unwrap(); - let ref_pixmap = sk::Pixmap::decode_png(&ref_data).unwrap(); - approx_equal(&pixmap, &ref_pixmap) - }; + // Render and save live version. + let live = document.make_live(); + document.save_live(&self.test.name, &live); + // Compare against reference output if available. // Test that is ok doesn't need to be updated. - if equal { + if ref_data.as_ref().map_or(false, |r| D::matches(&live, r)) { return; } @@ -201,36 +170,37 @@ impl<'a> Runner<'a> { std::fs::remove_file(&ref_path).unwrap(); log!( into: self.result.infos, - "removed reference image ({ref_path})" + "removed reference output ({})", ref_path.display() ); } else { - let opts = oxipng::Options::max_compression(); - let data = pixmap.encode_png().unwrap(); - let ref_data = oxipng::optimize_from_memory(&data, &opts).unwrap(); - if !self.test.large && ref_data.len() > crate::REF_LIMIT { - log!(self, "reference image would exceed maximum size"); + let ref_data = D::make_ref(live); + if !self.test.attrs.contains(&Attr::Large) + && ref_data.len() > crate::REF_LIMIT + { + log!(self, "reference output would exceed maximum size"); log!(self, " maximum | {}", FileSize(crate::REF_LIMIT)); log!(self, " size | {}", FileSize(ref_data.len())); log!(self, "please try to minimize the size of the test (smaller pages, less text, etc.)"); - log!(self, "if you think the test cannot be reasonably minimized, mark it as `// LARGE`"); + log!(self, "if you think the test cannot be reasonably minimized, mark it as `large`"); return; } std::fs::write(&ref_path, &ref_data).unwrap(); log!( into: self.result.infos, - "updated reference image ({ref_path}, {})", + "updated reference output ({}, {})", + ref_path.display(), FileSize(ref_data.len()), ); } } else { - self.result.mismatched_image = true; - if has_ref { - log!(self, "mismatched rendering"); - log!(self, " live | {live_path}"); - log!(self, " ref | {ref_path}"); + self.result.mismatched_output = true; + if ref_data.is_ok() { + log!(self, "mismatched output"); + log!(self, " live | {}", live_path.display()); + log!(self, " ref | {}", ref_path.display()); } else { - log!(self, "missing reference image"); - log!(self, " live | {live_path}"); + log!(self, "missing reference output"); + log!(self, " live | {}", live_path.display()); } } } @@ -242,6 +212,10 @@ impl<'a> Runner<'a> { if diag.span.id().is_some_and(|id| id != self.test.source.id()) { return; } + // TODO: remove this once HTML export is stable + if diag.message == "html export is under active development and incomplete" { + return; + } let message = diag.message.replace("\\", "/"); let range = self.world.range(diag.span); @@ -349,6 +323,153 @@ impl<'a> Runner<'a> { } } +/// An output type we can test. +trait OutputType: Document { + /// The type that represents live output. + type Live; + + /// The path at which the live output is stored. + fn live_path(name: &str) -> PathBuf; + + /// The path at which the reference output is stored. + fn ref_path(name: &str) -> PathBuf; + + /// Whether the test output is trivial and needs no reference output. + fn is_skippable(&self) -> Result<bool, ()> { + Ok(false) + } + + /// Produces the live output. + fn make_live(&self) -> Self::Live; + + /// Saves the live output. + fn save_live(&self, name: &str, live: &Self::Live); + + /// Produces the reference output from the live output. + fn make_ref(live: Self::Live) -> Vec<u8>; + + /// Checks whether the live and reference output match. + fn matches(live: &Self::Live, ref_data: &[u8]) -> bool; + + /// Runs additional checks. + #[expect(unused_variables)] + fn check_custom(runner: &mut Runner, doc: Option<&Self>) {} +} + +impl OutputType for PagedDocument { + type Live = tiny_skia::Pixmap; + + fn live_path(name: &str) -> PathBuf { + format!("{}/render/{}.png", crate::STORE_PATH, name).into() + } + + fn ref_path(name: &str) -> PathBuf { + format!("{}/{}.png", crate::REF_PATH, name).into() + } + + fn is_skippable(&self) -> Result<bool, ()> { + /// Whether rendering of a frame can be skipped. + fn skippable_frame(frame: &Frame) -> bool { + frame.items().all(|(_, item)| match item { + FrameItem::Group(group) => skippable_frame(&group.frame), + FrameItem::Tag(_) => true, + _ => false, + }) + } + + match self.pages.as_slice() { + [] => Err(()), + [page] => Ok(page.frame.width().approx_eq(Abs::pt(120.0)) + && page.frame.height().approx_eq(Abs::pt(20.0)) + && page.fill.is_auto() + && skippable_frame(&page.frame)), + _ => Ok(false), + } + } + + fn make_live(&self) -> Self::Live { + render(self, 1.0) + } + + fn save_live(&self, name: &str, live: &Self::Live) { + // Save live version, possibly rerendering if different scale is + // requested. + let mut pixmap_live = live; + let slot; + let scale = crate::ARGS.scale; + if scale != 1.0 { + slot = render(self, scale); + pixmap_live = &slot; + } + let data: Vec<u8> = pixmap_live.encode_png().unwrap(); + std::fs::write(Self::live_path(name), data).unwrap(); + + // Write PDF if requested. + if crate::ARGS.pdf() { + let pdf_path = format!("{}/pdf/{}.pdf", crate::STORE_PATH, name); + let pdf = typst_pdf::pdf(self, &PdfOptions::default()).unwrap(); + std::fs::write(pdf_path, pdf).unwrap(); + } + + // Write SVG if requested. + if crate::ARGS.svg() { + let svg_path = format!("{}/svg/{}.svg", crate::STORE_PATH, name); + let svg = typst_svg::svg_merged(self, Abs::pt(5.0)); + std::fs::write(svg_path, svg).unwrap(); + } + } + + fn make_ref(live: Self::Live) -> Vec<u8> { + let opts = oxipng::Options::max_compression(); + let data = live.encode_png().unwrap(); + oxipng::optimize_from_memory(&data, &opts).unwrap() + } + + fn matches(live: &Self::Live, ref_data: &[u8]) -> bool { + let ref_pixmap = sk::Pixmap::decode_png(ref_data).unwrap(); + approx_equal(live, &ref_pixmap) + } + + fn check_custom(runner: &mut Runner, doc: Option<&Self>) { + let errors = crate::custom::check(runner.test, &runner.world, doc); + if !errors.is_empty() { + log!(runner, "custom check failed"); + for line in errors.lines() { + log!(runner, " {line}"); + } + } + } +} + +impl OutputType for HtmlDocument { + type Live = String; + + fn live_path(name: &str) -> PathBuf { + format!("{}/html/{}.html", crate::STORE_PATH, name).into() + } + + fn ref_path(name: &str) -> PathBuf { + format!("{}/html/{}.html", crate::REF_PATH, name).into() + } + + fn make_live(&self) -> Self::Live { + // TODO: Do this earlier to be able to process export errors. + typst_html::html(self).unwrap() + } + + fn save_live(&self, name: &str, live: &Self::Live) { + std::fs::write(Self::live_path(name), live).unwrap(); + } + + fn make_ref(live: Self::Live) -> Vec<u8> { + live.into_bytes() + } + + fn matches(live: &Self::Live, ref_data: &[u8]) -> bool { + live.as_bytes() == ref_data + } +} + /// Draw all frames into one image with padding in between. fn render(document: &PagedDocument, pixel_per_pt: f32) -> sk::Pixmap { for page in &document.pages { @@ -397,23 +518,6 @@ fn render_links(canvas: &mut sk::Pixmap, ts: sk::Transform, frame: &Frame) { } } -/// Whether rendering of a frame can be skipped. -fn skippable(page: &Page) -> bool { - page.frame.width().approx_eq(Abs::pt(120.0)) - && page.frame.height().approx_eq(Abs::pt(20.0)) - && page.fill.is_auto() - && skippable_frame(&page.frame) -} - -/// Whether rendering of a frame can be skipped. -fn skippable_frame(frame: &Frame) -> bool { - frame.items().all(|(_, item)| match item { - FrameItem::Group(group) => skippable_frame(&group.frame), - FrameItem::Tag(_) => true, - _ => false, - }) -} - /// Whether two pixel images are approximately equal. fn approx_equal(a: &sk::Pixmap, b: &sk::Pixmap) -> bool { a.width() == b.width() diff --git a/tests/src/tests.rs b/tests/src/tests.rs index 2b09b29c..26eb63be 100644 --- a/tests/src/tests.rs +++ b/tests/src/tests.rs @@ -37,13 +37,13 @@ const STORE_PATH: &str = "tests/store"; /// The directory where syntax trees are stored. const SYNTAX_PATH: &str = "tests/store/syntax"; -/// The directory where the reference images are stored. +/// The directory where the reference output is stored. const REF_PATH: &str = "tests/ref"; /// The file where the skipped tests are stored. const SKIP_PATH: &str = "tests/skip.txt"; -/// The maximum size of reference images that aren't marked as `// LARGE`. +/// The maximum size of reference output that isn't marked as `large`. const REF_LIMIT: usize = 20 * 1024; fn main() { @@ -62,7 +62,7 @@ fn setup() { std::env::set_current_dir("..").unwrap(); // Create the storage. - for ext in ["render", "pdf", "svg"] { + for ext in ["render", "html", "pdf", "svg"] { std::fs::create_dir_all(Path::new(STORE_PATH).join(ext)).unwrap(); } @@ -156,10 +156,10 @@ fn clean() { fn undangle() { match crate::collect::collect() { - Ok(_) => eprintln!("no danging reference images"), + Ok(_) => eprintln!("no danging reference output"), Err(errors) => { for error in errors { - if error.message == "dangling reference image" { + if error.message == "dangling reference output" { std::fs::remove_file(&error.pos.path).unwrap(); eprintln!("✅ deleted {}", error.pos.path.display()); } @@ -188,7 +188,7 @@ fn run_parser_test( let mut result = TestResult { errors: String::new(), infos: String::new(), - mismatched_image: false, + mismatched_output: false, }; let syntax_file = live_path.join(format!("{}.syntax", test.name)); diff --git a/tests/src/world.rs b/tests/src/world.rs index f5e49ba6..a08f1efa 100644 --- a/tests/src/world.rs +++ b/tests/src/world.rs @@ -19,7 +19,7 @@ use typst::syntax::{FileId, Source, Span}; use typst::text::{Font, FontBook, TextElem, TextSize}; use typst::utils::{singleton, LazyHash}; use typst::visualize::Color; -use typst::{Library, World}; +use typst::{Feature, Library, World}; /// A world that provides access to the tests environment. #[derive(Clone)] @@ -180,7 +180,9 @@ fn library() -> Library { // Set page width to 120pt with 10pt margins, so that the inner page is // exactly 100pt wide. Page height is unbounded and font size is 10pt so // that it multiplies to nice round numbers. - let mut lib = Library::default(); + let mut lib = Library::builder() + .with_features([Feature::Html].into_iter().collect()) + .build(); // Hook up helpers into the global scope. lib.global.scope_mut().define_func::<test>(); |