From f364395e1d774456500ea61bb7b931f48a62ddfa Mon Sep 17 00:00:00 2001 From: Laurenz Date: Sat, 7 Dec 2019 14:42:25 +0100 Subject: =?UTF-8?q?Create=20parsing=20test=20harness=20=E2=9A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.toml | 10 ++- build.rs | 34 ++++++++++ src/syntax/tokens.rs | 2 +- tests/layout.rs | 157 ++++++++++++++++++++++++++++++++++++++++++++++ tests/layouting.rs | 154 --------------------------------------------- tests/layouting/align.typ | 23 +++++++ tests/layouting/coma.typ | 26 ++++++++ tests/layouting/lines.typ | 14 +++++ tests/layouts/align.typ | 23 ------- tests/layouts/coma.typ | 26 -------- tests/layouts/lines.typ | 14 ----- tests/parse.rs | 59 +++++++++++++++++ tests/parsing/base.rs | 78 +++++++++++++++++++++++ 13 files changed, 400 insertions(+), 220 deletions(-) create mode 100644 build.rs create mode 100644 tests/layout.rs delete mode 100644 tests/layouting.rs create mode 100644 tests/layouting/align.typ create mode 100644 tests/layouting/coma.typ create mode 100644 tests/layouting/lines.typ delete mode 100644 tests/layouts/align.typ delete mode 100644 tests/layouts/coma.typ delete mode 100644 tests/layouts/lines.typ create mode 100644 tests/parse.rs create mode 100644 tests/parsing/base.rs diff --git a/Cargo.toml b/Cargo.toml index 0ce0d396..76db3d18 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ name = "typstc" version = "0.1.0" authors = ["Laurenz Mädje "] edition = "2018" +build = "build.rs" [dependencies] tide = { path = "../tide" } @@ -16,6 +17,11 @@ name = "typst-bin" path = "src/bin/main.rs" [[test]] -name = "layouting" -path = "tests/layouting.rs" +name = "layout" +path = "tests/layout.rs" +harness = false + +[[test]] +name = "parse" +path = "tests/parse.rs" harness = false diff --git a/build.rs b/build.rs new file mode 100644 index 00000000..c91db8e0 --- /dev/null +++ b/build.rs @@ -0,0 +1,34 @@ +use std::fs; +use std::ffi::OsStr; + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed=tests/parsing"); + + fs::create_dir_all("tests/cache").unwrap(); + + let paths = fs::read_dir("tests/parsing").unwrap() + .map(|entry| entry.unwrap().path()) + .filter(|path| path.extension() == Some(OsStr::new("rs"))); + + let mut code = "vec![".to_string(); + for path in paths { + let name = path.file_stem().unwrap().to_str().unwrap(); + let file = fs::read_to_string(&path).unwrap(); + + println!("cargo:rerun-if-changed=tests/parsing/{}.rs", name); + + code.push_str(&format!("(\"{}\", tokens!{{", name)); + + for (index, line) in file.lines().enumerate() { + let mut line = line.replace("=>", &format!("=>({})=>", index + 1)); + line.push('\n'); + code.push_str(&line); + } + + code.push_str("}),"); + } + code.push(']'); + + fs::write("tests/cache/parsing.rs", code).unwrap(); +} diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs index 4fdee371..95b2ea3e 100644 --- a/src/syntax/tokens.rs +++ b/src/syntax/tokens.rs @@ -204,7 +204,7 @@ impl<'s> Iterator for Tokens<'s> { '\\' => { if let Some((index, c)) = self.chars.peek() { let escapable = match c { - '[' | ']' | '\\' | '*' | '_' | '`' | ':' | '=' | '/' => true, + '[' | ']' | '\\' | '*' | '_' | '`' | ':' | '=' | ',' | '/' => true, _ => false, }; diff --git a/tests/layout.rs b/tests/layout.rs new file mode 100644 index 00000000..46835a2a --- /dev/null +++ b/tests/layout.rs @@ -0,0 +1,157 @@ +use std::fs::{self, File}; +use std::io::{BufWriter, Read, Write}; +use std::process::Command; + +use typstc::export::pdf::PdfExporter; +use typstc::layout::{LayoutAction, Serialize}; +use typstc::size::{Size, Size2D, SizeBox}; +use typstc::style::PageStyle; +use typstc::toddle::query::FileSystemFontProvider; +use typstc::Typesetter; + +const CACHE_DIR: &str = "tests/cache"; + +fn main() { + let mut perfect_match = false; + let mut filter = Vec::new(); + + for arg in std::env::args().skip(1) { + if arg.as_str() == "--nocapture" { + continue; + } else if arg.as_str() == "=" { + perfect_match = true; + } else { + filter.push(arg); + } + } + + fs::create_dir_all(format!("{}/serialized", CACHE_DIR)).unwrap(); + fs::create_dir_all(format!("{}/rendered", CACHE_DIR)).unwrap(); + fs::create_dir_all(format!("{}/pdf", CACHE_DIR)).unwrap(); + + let mut failed = 0; + + for entry in fs::read_dir("tests/layouting/").unwrap() { + let path = entry.unwrap().path(); + + if path.extension() != Some(std::ffi::OsStr::new("typ")) { + continue; + } + + let name = path.file_stem().unwrap().to_str().unwrap(); + + let matches = if perfect_match { + filter.iter().any(|pattern| name == pattern) + } else { + filter.is_empty() || filter.iter().any(|pattern| name.contains(pattern)) + }; + + if matches { + let mut file = File::open(&path).unwrap(); + let mut src = String::new(); + file.read_to_string(&mut src).unwrap(); + + if std::panic::catch_unwind(|| test(name, &src)).is_err() { + failed += 1; + println!(); + } + } + } + + if failed > 0 { + println!("{} tests failed.", failed); + println!(); + std::process::exit(-1); + } + + println!(); +} + +/// Create a _PDF_ with a name from the source code. +fn test(name: &str, src: &str) { + println!("Testing: {}.", name); + + let mut typesetter = Typesetter::new(); + + typesetter.set_page_style(PageStyle { + dimensions: Size2D::with_all(Size::pt(250.0)), + margins: SizeBox::with_all(Size::pt(10.0)), + }); + + let provider = FileSystemFontProvider::from_listing("fonts/fonts.toml").unwrap(); + typesetter.add_font_provider(provider.clone()); + + #[cfg(not(debug_assertions))] { + use std::time::Instant; + + // Warmup. + let warmup_start = Instant::now(); + let is_ok = typesetter.typeset(&src).is_ok(); + let warmup_end = Instant::now(); + + if is_ok { + let start = Instant::now(); + let tree = typesetter.parse(&src).unwrap(); + let mid = Instant::now(); + typesetter.layout(&tree).unwrap(); + let end = Instant::now(); + + println!(" - cold start: {:?}", warmup_end - warmup_start); + println!(" - warmed up: {:?}", end - start); + println!(" - parsing: {:?}", mid - start); + println!(" - layouting: {:?}", end - mid); + println!(); + } + }; + + let layouts = match typesetter.typeset(&src) { + Ok(layouts) => layouts, + Err(err) => { + println!(" - compilation failed: {}", err); + #[cfg(not(debug_assertions))] + println!(); + return; + } + }; + + // Write the serialed layout file. + let path = format!("{}/serialized/{}.tld", CACHE_DIR, name); + let mut file = File::create(path).unwrap(); + + // Find all used fonts and their filenames. + let mut map = Vec::new(); + let mut loader = typesetter.loader().borrow_mut(); + for layout in &layouts { + for action in &layout.actions { + if let LayoutAction::SetFont(index, _) = action { + if map.iter().find(|(i, _)| i == index).is_none() { + let (_, provider_index) = loader.get_provider_and_index(*index); + let filename = provider.get_path(provider_index).to_str().unwrap(); + map.push((*index, filename)); + } + } + } + } + drop(loader); + + // Write the font mapping into the serialization file. + writeln!(file, "{}", map.len()).unwrap(); + for (index, path) in map { + writeln!(file, "{} {}", index, path).unwrap(); + } + + layouts.serialize(&mut file).unwrap(); + + // Render the layout into a PNG. + Command::new("python") + .arg("tests/render.py") + .arg(name) + .spawn() + .expect("failed to run python-based renderer"); + + // Write the PDF file. + let path = format!("{}/pdf/{}.pdf", CACHE_DIR, name); + let file = BufWriter::new(File::create(path).unwrap()); + let exporter = PdfExporter::new(); + exporter.export(&layouts, typesetter.loader(), file).unwrap(); +} diff --git a/tests/layouting.rs b/tests/layouting.rs deleted file mode 100644 index 001ff45a..00000000 --- a/tests/layouting.rs +++ /dev/null @@ -1,154 +0,0 @@ -use std::fs::{self, File}; -use std::io::{BufWriter, Read, Write}; -use std::process::Command; - -use typstc::export::pdf::PdfExporter; -use typstc::layout::{LayoutAction, Serialize}; -use typstc::size::{Size, Size2D, SizeBox}; -use typstc::style::PageStyle; -use typstc::toddle::query::FileSystemFontProvider; -use typstc::Typesetter; - -const CACHE_DIR: &str = "tests/cache"; - -fn main() { - let mut perfect_match = false; - let mut filter = Vec::new(); - - for arg in std::env::args().skip(1) { - if arg.as_str() == "--nocapture" { - continue; - } else if arg.as_str() == "=" { - perfect_match = true; - } else { - filter.push(arg); - } - } - - fs::create_dir_all(format!("{}/serialized", CACHE_DIR)).unwrap(); - fs::create_dir_all(format!("{}/rendered", CACHE_DIR)).unwrap(); - fs::create_dir_all(format!("{}/pdf", CACHE_DIR)).unwrap(); - - let mut failed = 0; - - for entry in fs::read_dir("tests/layouts/").unwrap() { - let path = entry.unwrap().path(); - - if path.extension() != Some(std::ffi::OsStr::new("typ")) { - continue; - } - - let name = path.file_stem().unwrap().to_str().unwrap(); - - let matches = if perfect_match { - filter.iter().any(|pattern| name == pattern) - } else { - filter.is_empty() || filter.iter().any(|pattern| name.contains(pattern)) - }; - - if matches { - let mut file = File::open(&path).unwrap(); - let mut src = String::new(); - file.read_to_string(&mut src).unwrap(); - - if std::panic::catch_unwind(|| test(name, &src)).is_err() { - failed += 1; - println!(); - } - } - } - - if failed > 0 { - println!("{} tests failed.", failed); - std::process::exit(-1); - } -} - -/// Create a _PDF_ with a name from the source code. -fn test(name: &str, src: &str) { - println!("Testing: {}.", name); - - let mut typesetter = Typesetter::new(); - - typesetter.set_page_style(PageStyle { - dimensions: Size2D::with_all(Size::pt(250.0)), - margins: SizeBox::with_all(Size::pt(10.0)), - }); - - let provider = FileSystemFontProvider::from_listing("fonts/fonts.toml").unwrap(); - typesetter.add_font_provider(provider.clone()); - - #[cfg(not(debug_assertions))] { - use std::time::Instant; - - // Warmup. - let warmup_start = Instant::now(); - let is_ok = typesetter.typeset(&src).is_ok(); - let warmup_end = Instant::now(); - - if is_ok { - let start = Instant::now(); - let tree = typesetter.parse(&src).unwrap(); - let mid = Instant::now(); - typesetter.layout(&tree).unwrap(); - let end = Instant::now(); - - println!(" - cold start: {:?}", warmup_end - warmup_start); - println!(" - warmed up: {:?}", end - start); - println!(" - parsing: {:?}", mid - start); - println!(" - layouting: {:?}", end - mid); - println!(); - } - }; - - let layouts = match typesetter.typeset(&src) { - Ok(layouts) => layouts, - Err(err) => { - println!(" - compilation failed: {}", err); - #[cfg(not(debug_assertions))] - println!(); - return; - } - }; - - // Write the serialed layout file. - let path = format!("{}/serialized/{}.tld", CACHE_DIR, name); - let mut file = File::create(path).unwrap(); - - // Find all used fonts and their filenames. - let mut map = Vec::new(); - let mut loader = typesetter.loader().borrow_mut(); - for layout in &layouts { - for action in &layout.actions { - if let LayoutAction::SetFont(index, _) = action { - if map.iter().find(|(i, _)| i == index).is_none() { - let (_, provider_index) = loader.get_provider_and_index(*index); - let filename = provider.get_path(provider_index).to_str().unwrap(); - map.push((*index, filename)); - } - } - } - } - drop(loader); - - // Write the font mapping into the serialization file. - writeln!(file, "{}", map.len()).unwrap(); - for (index, path) in map { - writeln!(file, "{} {}", index, path).unwrap(); - } - - layouts.serialize(&mut file).unwrap(); - - // Render the layout into a PNG. - Command::new("python") - .arg("tests/render.py") - .arg(name) - .spawn() - .expect("failed to run python-based renderer"); - - // Write the PDF file. - let path = format!("{}/pdf/{}.pdf", CACHE_DIR, name); - let file = BufWriter::new(File::create(path).unwrap()); - let exporter = PdfExporter::new(); - exporter.export(&layouts, typesetter.loader(), file).unwrap(); -} diff --git a/tests/layouting/align.typ b/tests/layouting/align.typ new file mode 100644 index 00000000..a7101932 --- /dev/null +++ b/tests/layouting/align.typ @@ -0,0 +1,23 @@ +[box][ + A short sentence. [align: right][words.] + + A short sentence. [n] [align: right][words.] + + A short sentence. [par.break] [align: right][words.] + + [align: bottom] + A longer sentence with a few more words. +] + +[page.break] + +[box][ + [align: vertical=top] + Top + + [align: vertical=center] + Center + + [align: vertical=bottom] + Bottom [n] Bottom [n] Bottom +] diff --git a/tests/layouting/coma.typ b/tests/layouting/coma.typ new file mode 100644 index 00000000..d0a02b74 --- /dev/null +++ b/tests/layouting/coma.typ @@ -0,0 +1,26 @@ +[page.size: width=450pt, height=300pt] +[page.margins: 1cm] + +[box][ + *Technical University Berlin* [n] + *Faculty II, Institute for Mathematics* [n] + Secretary Example [n] + Prof. Dr. Example [n] + Assistant #1, Assistant #2, Assistant #3 +] +[align: right][*WiSe 2019/2020* [n] Week 1] + +[v: 6mm] + +[align: center][ + *3. Ubungsblatt Computerorientierte Mathematik II* [v: 0.3mm] + *Abgabe: 03.05.2019* (bis 10:10 Uhr in MA 001) [v: 0.3mm] + *Alle Antworten sind zu beweisen.* +] + +*1. Aufgabe* [align: right][(1 + 1 + 2 Punkte)] + +Ein _Binärbaum_ ist ein Wurzelbaum, in dem jeder Knoten ≤ 2 Kinder hat. +Die Tiefe eines Knotens _v_ ist die Länge des eindeutigen Weges von der Wurzel +zu _v_, und die Höhe von _v_ ist die Länge eines längsten (absteigenden) Weges +von _v_ zu einem Blatt. Die Höhe des Baumes ist die Höhe der Wurzel. diff --git a/tests/layouting/lines.typ b/tests/layouting/lines.typ new file mode 100644 index 00000000..705de638 --- /dev/null +++ b/tests/layouting/lines.typ @@ -0,0 +1,14 @@ +[page.size: height=5cm] + +Line. + +[box][ + Lines with [box][two] [box][boxes]. + + Lines with two boxes. +] +[box][ + Lines without two boxes. + + Lines without two boxes. +] diff --git a/tests/layouts/align.typ b/tests/layouts/align.typ deleted file mode 100644 index a7101932..00000000 --- a/tests/layouts/align.typ +++ /dev/null @@ -1,23 +0,0 @@ -[box][ - A short sentence. [align: right][words.] - - A short sentence. [n] [align: right][words.] - - A short sentence. [par.break] [align: right][words.] - - [align: bottom] - A longer sentence with a few more words. -] - -[page.break] - -[box][ - [align: vertical=top] - Top - - [align: vertical=center] - Center - - [align: vertical=bottom] - Bottom [n] Bottom [n] Bottom -] diff --git a/tests/layouts/coma.typ b/tests/layouts/coma.typ deleted file mode 100644 index d0a02b74..00000000 --- a/tests/layouts/coma.typ +++ /dev/null @@ -1,26 +0,0 @@ -[page.size: width=450pt, height=300pt] -[page.margins: 1cm] - -[box][ - *Technical University Berlin* [n] - *Faculty II, Institute for Mathematics* [n] - Secretary Example [n] - Prof. Dr. Example [n] - Assistant #1, Assistant #2, Assistant #3 -] -[align: right][*WiSe 2019/2020* [n] Week 1] - -[v: 6mm] - -[align: center][ - *3. Ubungsblatt Computerorientierte Mathematik II* [v: 0.3mm] - *Abgabe: 03.05.2019* (bis 10:10 Uhr in MA 001) [v: 0.3mm] - *Alle Antworten sind zu beweisen.* -] - -*1. Aufgabe* [align: right][(1 + 1 + 2 Punkte)] - -Ein _Binärbaum_ ist ein Wurzelbaum, in dem jeder Knoten ≤ 2 Kinder hat. -Die Tiefe eines Knotens _v_ ist die Länge des eindeutigen Weges von der Wurzel -zu _v_, und die Höhe von _v_ ist die Länge eines längsten (absteigenden) Weges -von _v_ zu einem Blatt. Die Höhe des Baumes ist die Höhe der Wurzel. diff --git a/tests/layouts/lines.typ b/tests/layouts/lines.typ deleted file mode 100644 index 705de638..00000000 --- a/tests/layouts/lines.typ +++ /dev/null @@ -1,14 +0,0 @@ -[page.size: height=5cm] - -Line. - -[box][ - Lines with [box][two] [box][boxes]. - - Lines with two boxes. -] -[box][ - Lines without two boxes. - - Lines without two boxes. -] diff --git a/tests/parse.rs b/tests/parse.rs new file mode 100644 index 00000000..a56059d7 --- /dev/null +++ b/tests/parse.rs @@ -0,0 +1,59 @@ +use typstc::syntax::*; + +use Token::{ + Space as S, Newline as N, LeftBracket as LB, + RightBracket as RB, Text as T, * +}; + +macro_rules! tokens { + ($($src:expr =>($line:expr)=> $tokens:expr)*) => ({ + #[allow(unused_mut)] + let mut cases = Vec::new(); + $(cases.push(($line, $src, $tokens.to_vec()));)* + cases + }); +} + +fn main() { + let tests = include!("cache/parsing.rs"); + + let mut errors = false; + for (file, cases) in tests.into_iter() { + print!("Testing: {}. ", file); + + let mut okay = 0; + let mut failed = 0; + + for (line, src, expected) in cases.into_iter() { + let found: Vec<_> = tokenize(src).map(Spanned::value).collect(); + + if found == expected { + okay += 1; + } else { + if failed == 0 { + println!(); + } + + println!(" - Case failed in file {}.rs in line {}.", file, line); + println!(" - Source: {:?}", src); + println!(" - Expected: {:?}", expected); + println!(" - Found: {:?}", found); + + failed += 1; + errors = true; + } + } + + print!("{} okay, {} failed.", okay, failed); + if failed == 0 { + print!(" ✔") + } + println!(); + } + + println!(); + + if errors { + std::process::exit(-1); + } +} diff --git a/tests/parsing/base.rs b/tests/parsing/base.rs new file mode 100644 index 00000000..ad7d87c0 --- /dev/null +++ b/tests/parsing/base.rs @@ -0,0 +1,78 @@ +// Spaces, Newlines, Brackets. +"" => [] +" " => [S] +" " => [S] +"\t" => [S] +" \t" => [S] +"\n" => [N] +"\n " => [N, S] +" \n" => [S, N] +" \n " => [S, N, S] +"[" => [LB] +"]" => [RB] + +// Header only tokens. +"[:]" => [LB, Colon, RB] +"[=]" => [LB, Equals, RB] +"[,]" => [LB, Comma, RB] +":" => [T(":")] +"=" => [T("=")] +"," => [T(",")] +r#"["hi"]"# => [LB, Quoted("hi"), RB] +r#""hi""# => [T(r#""hi""#)] + +// Body only tokens. +"_" => [Underscore] +"*" => [Star] +"`" => [Backtick] +"[_]" => [LB, T("_"), RB] +"[*]" => [LB, T("*"), RB] +"[`]" => [LB, T("`"), RB] + +// Comments. +"//line" => [LineComment("line")] +"/*block*/" => [BlockComment("block")] +"*/" => [StarSlash] + +// Plain text. +"A" => [T("A")] +"Hello" => [T("Hello")] +"Hello-World" => [T("Hello-World")] +r#"A"B"# => [T(r#"A"B"#)] +"🌍" => [T("🌍")] + +// Escapes. +r"\[" => [T("[")] +r"\]" => [T("]")] +r"\\" => [T(r"\")] +r"[\[]" => [LB, T("["), RB] +r"[\]]" => [LB, T("]"), RB] +r"[\\]" => [LB, T(r"\"), RB] +r"\:" => [T(":")] +r"\=" => [T("=")] +r"\/" => [T("/")] +r"[\:]" => [LB, T(":"), RB] +r"[\=]" => [LB, T("="), RB] +r"[\,]" => [LB, T(","), RB] +r"\*" => [T("*")] +r"\_" => [T("_")] +r"\`" => [T("`")] +r"[\*]" => [LB, T("*"), RB] +r"[\_]" => [LB, T("_"), RB] +r"[\`]" => [LB, T("`"), RB] + +// Whitespace. +"Hello World" => [T("Hello"), S, T("World")] +"Hello World" => [T("Hello"), S, T("World")] +"Hello \t World" => [T("Hello"), S, T("World")] + +// Newline. +"First\n" => [T("First"), N] +"First \n" => [T("First"), S, N] +"First\n " => [T("First"), N, S] +"First \n " => [T("First"), S, N, S] +"First\nSecond" => [T("First"), N, T("Second")] +"First\r\nSecond" => [T("First"), N, T("Second")] +"First \nSecond" => [T("First"), S, N, T("Second")] +"First\n Second" => [T("First"), N, S, T("Second")] +"First \n Second" => [T("First"), S, N, S, T("Second")] -- cgit v1.2.3