From e16d3f5a67a31154797b4d56cdc6ed142ee2a7cf Mon Sep 17 00:00:00 2001 From: Laurenz Date: Wed, 28 Feb 2024 11:06:54 +0100 Subject: Externalize assets (#3515) --- crates/typst-cli/Cargo.toml | 1 + crates/typst-cli/src/fonts.rs | 26 +------- crates/typst-docs/Cargo.toml | 2 + crates/typst-docs/src/html.rs | 17 +++--- crates/typst-docs/src/lib.rs | 9 ++- crates/typst-pdf/Cargo.toml | 1 + crates/typst-pdf/src/color.rs | 6 +- crates/typst-pdf/src/icc/sGrey-v4.icc | Bin 360 -> 0 bytes crates/typst-pdf/src/icc/sRGB-v4.icc | Bin 480 -> 0 bytes crates/typst-pdf/src/oklab.ps | 78 ++++++++++++++++++++++++ crates/typst-pdf/src/postscript/oklab.ps | 78 ------------------------ crates/typst/Cargo.toml | 1 + crates/typst/assets/CGATS001Compat-v2-micro.icc | Bin 8464 -> 0 bytes crates/typst/assets/cj_linebreak_data.postcard | Bin 18848 -> 0 bytes crates/typst/assets/icudata.postcard | Bin 352005 -> 0 bytes crates/typst/src/layout/inline/linebreak.rs | 41 +++---------- crates/typst/src/loading/csv.rs | 2 +- crates/typst/src/loading/read.rs | 2 +- crates/typst/src/visualize/color.rs | 8 +-- 19 files changed, 115 insertions(+), 157 deletions(-) delete mode 100644 crates/typst-pdf/src/icc/sGrey-v4.icc delete mode 100644 crates/typst-pdf/src/icc/sRGB-v4.icc create mode 100644 crates/typst-pdf/src/oklab.ps delete mode 100644 crates/typst-pdf/src/postscript/oklab.ps delete mode 100644 crates/typst/assets/CGATS001Compat-v2-micro.icc delete mode 100644 crates/typst/assets/cj_linebreak_data.postcard delete mode 100644 crates/typst/assets/icudata.postcard (limited to 'crates') diff --git a/crates/typst-cli/Cargo.toml b/crates/typst-cli/Cargo.toml index 9f0f280e..ab9ed6f2 100644 --- a/crates/typst-cli/Cargo.toml +++ b/crates/typst-cli/Cargo.toml @@ -21,6 +21,7 @@ doc = false [dependencies] typst = { workspace = true } +typst-assets = { workspace = true, features = ["fonts"] } typst-macros = { workspace = true } typst-pdf = { workspace = true } typst-render = { workspace = true } diff --git a/crates/typst-cli/src/fonts.rs b/crates/typst-cli/src/fonts.rs index 4ea9f446..a5e454ed 100644 --- a/crates/typst-cli/src/fonts.rs +++ b/crates/typst-cli/src/fonts.rs @@ -107,8 +107,8 @@ impl FontSearcher { /// Add fonts that are embedded in the binary. #[cfg(feature = "embed-fonts")] fn add_embedded(&mut self) { - let mut process = |bytes: &'static [u8]| { - let buffer = typst::foundations::Bytes::from_static(bytes); + for data in typst_assets::fonts() { + let buffer = typst::foundations::Bytes::from_static(data); for (i, font) in Font::iter(buffer).enumerate() { self.book.push(font.info().clone()); self.fonts.push(FontSlot { @@ -117,28 +117,6 @@ impl FontSearcher { font: OnceLock::from(Some(font)), }); } - }; - - macro_rules! add { - ($filename:literal) => { - process(include_bytes!(concat!("../../../assets/fonts/", $filename))); - }; } - - // Embed default fonts. - add!("LinLibertine_R.ttf"); - add!("LinLibertine_RB.ttf"); - add!("LinLibertine_RBI.ttf"); - add!("LinLibertine_RI.ttf"); - add!("NewCMMath-Book.otf"); - add!("NewCMMath-Regular.otf"); - add!("NewCM10-Regular.otf"); - add!("NewCM10-Bold.otf"); - add!("NewCM10-Italic.otf"); - add!("NewCM10-BoldItalic.otf"); - add!("DejaVuSansMono.ttf"); - add!("DejaVuSansMono-Bold.ttf"); - add!("DejaVuSansMono-Oblique.ttf"); - add!("DejaVuSansMono-BoldOblique.ttf"); } } diff --git a/crates/typst-docs/Cargo.toml b/crates/typst-docs/Cargo.toml index bb32aaf2..af661bfb 100644 --- a/crates/typst-docs/Cargo.toml +++ b/crates/typst-docs/Cargo.toml @@ -19,6 +19,8 @@ cli = ["clap", "typst-render", "serde_json"] [dependencies] typst = { workspace = true } +typst-assets = { workspace = true, features = ["fonts"] } +typst-dev-assets = { workspace = true } comemo = { workspace = true } ecow = { workspace = true } heck = { workspace = true } diff --git a/crates/typst-docs/src/html.rs b/crates/typst-docs/src/html.rs index 0ec0ddf0..6c47b8df 100644 --- a/crates/typst-docs/src/html.rs +++ b/crates/typst-docs/src/html.rs @@ -17,7 +17,7 @@ use typst::{Library, World}; use unscanny::Scanner; use yaml_front_matter::YamlFrontMatter; -use crate::{contributors, OutlineItem, Resolver, FILE_DIR, FONTS, LIBRARY}; +use crate::{contributors, OutlineItem, Resolver, FONTS, LIBRARY}; /// HTML documentation. #[derive(Serialize)] @@ -250,8 +250,8 @@ impl<'a> Handler<'a> { } fn handle_image(&self, link: &str) -> String { - if let Some(file) = FILE_DIR.get_file(link) { - self.resolver.image(link, file.contents()) + if let Some(data) = typst_dev_assets::get(link) { + self.resolver.image(link, data) } else if let Some(url) = self.resolver.link(link) { url } else { @@ -451,11 +451,12 @@ impl World for DocWorld { fn file(&self, id: FileId) -> FileResult { assert!(id.package().is_none()); - Ok(FILE_DIR - .get_file(id.vpath().as_rootless_path()) - .unwrap_or_else(|| panic!("failed to load {:?}", id.vpath())) - .contents() - .into()) + Ok(Bytes::from_static( + typst_dev_assets::get_by_name( + &id.vpath().as_rootless_path().to_string_lossy(), + ) + .unwrap_or_else(|| panic!("failed to load {:?}", id.vpath())), + )) } fn font(&self, index: usize) -> Option { diff --git a/crates/typst-docs/src/lib.rs b/crates/typst-docs/src/lib.rs index 0a69523e..5439ad1b 100644 --- a/crates/typst-docs/src/lib.rs +++ b/crates/typst-docs/src/lib.rs @@ -20,6 +20,7 @@ use serde::de::DeserializeOwned; use serde::Deserialize; use serde_yaml as yaml; use typst::diag::{bail, StrResult}; +use typst::foundations::Bytes; use typst::foundations::{ CastInfo, Category, Func, Module, ParamInfo, Repr, Scope, Smart, Type, Value, FOUNDATIONS, @@ -36,8 +37,6 @@ use typst::visualize::VISUALIZE; use typst::Library; static DOCS_DIR: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/../../docs"); -static FILE_DIR: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/../../assets/files"); -static FONT_DIR: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/../../assets/fonts"); static GROUPS: Lazy> = Lazy::new(|| { let mut groups: Vec = yaml("reference/groups.yml"); @@ -67,9 +66,9 @@ static LIBRARY: Lazy> = Lazy::new(|| { }); static FONTS: Lazy<(Prehashed, Vec)> = Lazy::new(|| { - let fonts: Vec<_> = FONT_DIR - .files() - .flat_map(|file| Font::iter(file.contents().into())) + let fonts: Vec<_> = typst_assets::fonts() + .chain(typst_dev_assets::fonts()) + .flat_map(|data| Font::iter(Bytes::from_static(data))) .collect(); let book = FontBook::from_fonts(&fonts); (Prehashed::new(book), fonts) diff --git a/crates/typst-pdf/Cargo.toml b/crates/typst-pdf/Cargo.toml index 3dcddbb0..dd71b61e 100644 --- a/crates/typst-pdf/Cargo.toml +++ b/crates/typst-pdf/Cargo.toml @@ -17,6 +17,7 @@ bench = false [dependencies] typst = { workspace = true } +typst-assets = { workspace = true } typst-macros = { workspace = true } typst-timing = { workspace = true } base64 = { workspace = true } diff --git a/crates/typst-pdf/src/color.rs b/crates/typst-pdf/src/color.rs index f0d483cf..4c35d9a2 100644 --- a/crates/typst-pdf/src/color.rs +++ b/crates/typst-pdf/src/color.rs @@ -19,13 +19,13 @@ const OKLAB_B: Name<'static> = Name(b"B"); // The ICC profiles. static SRGB_ICC_DEFLATED: Lazy> = - Lazy::new(|| deflate(include_bytes!("icc/sRGB-v4.icc"))); + Lazy::new(|| deflate(typst_assets::icc::S_RGB_V4)); static GRAY_ICC_DEFLATED: Lazy> = - Lazy::new(|| deflate(include_bytes!("icc/sGrey-v4.icc"))); + Lazy::new(|| deflate(typst_assets::icc::S_GREY_V4)); // The PostScript functions for color spaces. static OKLAB_DEFLATED: Lazy> = - Lazy::new(|| deflate(minify(include_str!("postscript/oklab.ps")).as_bytes())); + Lazy::new(|| deflate(minify(include_str!("oklab.ps")).as_bytes())); /// The color spaces present in the PDF document #[derive(Default)] diff --git a/crates/typst-pdf/src/icc/sGrey-v4.icc b/crates/typst-pdf/src/icc/sGrey-v4.icc deleted file mode 100644 index 2187b678..00000000 Binary files a/crates/typst-pdf/src/icc/sGrey-v4.icc and /dev/null differ diff --git a/crates/typst-pdf/src/icc/sRGB-v4.icc b/crates/typst-pdf/src/icc/sRGB-v4.icc deleted file mode 100644 index d9f3c055..00000000 Binary files a/crates/typst-pdf/src/icc/sRGB-v4.icc and /dev/null differ diff --git a/crates/typst-pdf/src/oklab.ps b/crates/typst-pdf/src/oklab.ps new file mode 100644 index 00000000..e766bbd8 --- /dev/null +++ b/crates/typst-pdf/src/oklab.ps @@ -0,0 +1,78 @@ +{ + % Starting stack: L, A, B + % /!\ WARNING: The A and B components **MUST** be encoded + % in the range [0, 1] before calling this function. + % This is because the function assumes that the + % A and B components are offset by a factor of 0.5 + % in order to meet the range requirements of the + % PDF specification. + + exch 0.5 sub + exch 0.5 sub + + % Load L a and b into the stack + 2 index + 2 index + 2 index + + % Compute f1 = ((0.3963377774 * a) + (0.2158037573 * b) + L)^3 + 0.2158037573 mul exch + 0.3963377774 mul add add + dup dup mul mul + + % Load L, a, and b into the stack + 3 index + 3 index + 3 index + + % Compute f2 = ((-0.1055613458 * a) + (-0.0638541728 * b) + L)^3 + -0.0638541728 mul exch + -0.1055613458 mul add add + dup dup mul mul + + % Load L, a, and b into the stack + 4 index + 4 index + 4 index + + % Compute f3 = ((-0.0894841775 * a) + (-1.2914855480 * b) + L)^3 + -1.2914855480 mul exch + -0.0894841775 mul add add + dup dup mul mul + + % Discard L, a, and b by rolling the stack and popping + 6 3 roll pop pop pop + + % Load f1, f2, and f3 into the stack + 2 index + 2 index + 2 index + + % Compute R = f1 * 4.0767416621 + f2 * -3.3077115913 + f3 * 0.2309699292 + 0.2309699292 mul exch + -3.3077115913 mul add exch + 4.0767416621 mul add + + % Load f1, f2, and f3 into the stack + 3 index + 3 index + 3 index + + % Compute G = f1 * -1.2684380046 + f2 * 2.6097574011 + f3 * -0.3413193965 + -0.3413193965 mul exch + 2.6097574011 mul add exch + -1.2684380046 mul add + + % Load f1, f2, and f3 into the stack + 4 index + 4 index + 4 index + + % Compute B = f1 * -0.0041960863 + f2 * -0.7034186147 + f3 * 1.7076147010 + 1.7076147010 mul exch + -0.7034186147 mul add exch + -0.0041960863 mul add + + % Discard f1, f2, and f3 by rolling the stack and popping + 6 3 roll pop pop pop +} diff --git a/crates/typst-pdf/src/postscript/oklab.ps b/crates/typst-pdf/src/postscript/oklab.ps deleted file mode 100644 index e766bbd8..00000000 --- a/crates/typst-pdf/src/postscript/oklab.ps +++ /dev/null @@ -1,78 +0,0 @@ -{ - % Starting stack: L, A, B - % /!\ WARNING: The A and B components **MUST** be encoded - % in the range [0, 1] before calling this function. - % This is because the function assumes that the - % A and B components are offset by a factor of 0.5 - % in order to meet the range requirements of the - % PDF specification. - - exch 0.5 sub - exch 0.5 sub - - % Load L a and b into the stack - 2 index - 2 index - 2 index - - % Compute f1 = ((0.3963377774 * a) + (0.2158037573 * b) + L)^3 - 0.2158037573 mul exch - 0.3963377774 mul add add - dup dup mul mul - - % Load L, a, and b into the stack - 3 index - 3 index - 3 index - - % Compute f2 = ((-0.1055613458 * a) + (-0.0638541728 * b) + L)^3 - -0.0638541728 mul exch - -0.1055613458 mul add add - dup dup mul mul - - % Load L, a, and b into the stack - 4 index - 4 index - 4 index - - % Compute f3 = ((-0.0894841775 * a) + (-1.2914855480 * b) + L)^3 - -1.2914855480 mul exch - -0.0894841775 mul add add - dup dup mul mul - - % Discard L, a, and b by rolling the stack and popping - 6 3 roll pop pop pop - - % Load f1, f2, and f3 into the stack - 2 index - 2 index - 2 index - - % Compute R = f1 * 4.0767416621 + f2 * -3.3077115913 + f3 * 0.2309699292 - 0.2309699292 mul exch - -3.3077115913 mul add exch - 4.0767416621 mul add - - % Load f1, f2, and f3 into the stack - 3 index - 3 index - 3 index - - % Compute G = f1 * -1.2684380046 + f2 * 2.6097574011 + f3 * -0.3413193965 - -0.3413193965 mul exch - 2.6097574011 mul add exch - -1.2684380046 mul add - - % Load f1, f2, and f3 into the stack - 4 index - 4 index - 4 index - - % Compute B = f1 * -0.0041960863 + f2 * -0.7034186147 + f3 * 1.7076147010 - 1.7076147010 mul exch - -0.7034186147 mul add exch - -0.0041960863 mul add - - % Discard f1, f2, and f3 by rolling the stack and popping - 6 3 roll pop pop pop -} diff --git a/crates/typst/Cargo.toml b/crates/typst/Cargo.toml index 01f6e7c6..77809ed1 100644 --- a/crates/typst/Cargo.toml +++ b/crates/typst/Cargo.toml @@ -16,6 +16,7 @@ doctest = false bench = false [dependencies] +typst-assets = { workspace = true } typst-macros = { workspace = true } typst-syntax = { workspace = true } typst-timing = { workspace = true } diff --git a/crates/typst/assets/CGATS001Compat-v2-micro.icc b/crates/typst/assets/CGATS001Compat-v2-micro.icc deleted file mode 100644 index b5a73495..00000000 Binary files a/crates/typst/assets/CGATS001Compat-v2-micro.icc and /dev/null differ diff --git a/crates/typst/assets/cj_linebreak_data.postcard b/crates/typst/assets/cj_linebreak_data.postcard deleted file mode 100644 index 910dd167..00000000 Binary files a/crates/typst/assets/cj_linebreak_data.postcard and /dev/null differ diff --git a/crates/typst/assets/icudata.postcard b/crates/typst/assets/icudata.postcard deleted file mode 100644 index a1fdbd48..00000000 Binary files a/crates/typst/assets/icudata.postcard and /dev/null differ diff --git a/crates/typst/src/layout/inline/linebreak.rs b/crates/typst/src/layout/inline/linebreak.rs index 1f00dc20..3c3416ed 100644 --- a/crates/typst/src/layout/inline/linebreak.rs +++ b/crates/typst/src/layout/inline/linebreak.rs @@ -10,51 +10,28 @@ use super::Preparation; use crate::syntax::link_prefix; use crate::text::{Lang, TextElem}; -/// Generated by the following command: -/// -/// ```sh -/// icu4x-datagen --locales full \ -/// --format blob \ -/// --keys-for-bin target/debug/typst \ -/// --out crates/typst-library/assets/icudata.postcard \ -/// --overwrite -/// ``` -/// -/// Install icu_datagen with `cargo install icu_datagen`. -static ICU_DATA: &[u8] = include_bytes!("../../../assets/icudata.postcard"); - -/// Generated by the following command: -/// -/// ```sh -/// icu4x-datagen --locales zh ja \ -/// --format blob \ -/// --keys segmenter/line@1 \ -/// --out crates/typst-library/assets/cj_linebreak_data.postcard \ -/// --overwrite -/// ``` -/// -/// The used icu_datagen should be patched by -/// https://github.com/peng1999/icu4x/commit/b9beb6cbf633d61fc3d7983e5baf7f4449fbfae5 -static CJ_LINEBREAK_DATA: &[u8] = - include_bytes!("../../../assets/cj_linebreak_data.postcard"); - /// The general line break segmenter. static SEGMENTER: Lazy = Lazy::new(|| { - let provider = BlobDataProvider::try_new_from_static_blob(ICU_DATA).unwrap(); + let provider = + BlobDataProvider::try_new_from_static_blob(typst_assets::icu::ICU).unwrap(); LineSegmenter::try_new_lstm_with_buffer_provider(&provider).unwrap() }); /// The line break segmenter for Chinese/Japanese text. static CJ_SEGMENTER: Lazy = Lazy::new(|| { - let provider = BlobDataProvider::try_new_from_static_blob(ICU_DATA).unwrap(); - let cj_blob = BlobDataProvider::try_new_from_static_blob(CJ_LINEBREAK_DATA).unwrap(); + let provider = + BlobDataProvider::try_new_from_static_blob(typst_assets::icu::ICU).unwrap(); + let cj_blob = + BlobDataProvider::try_new_from_static_blob(typst_assets::icu::ICU_CJ_SEGMENT) + .unwrap(); let cj_provider = ForkByKeyProvider::new(cj_blob, provider); LineSegmenter::try_new_lstm_with_buffer_provider(&cj_provider).unwrap() }); /// The Unicode line break properties for each code point. static LINEBREAK_DATA: Lazy> = Lazy::new(|| { - let provider = BlobDataProvider::try_new_from_static_blob(ICU_DATA).unwrap(); + let provider = + BlobDataProvider::try_new_from_static_blob(typst_assets::icu::ICU).unwrap(); let deser_provider = provider.as_deserializing(); icu_properties::maps::load_line_break(&deser_provider).unwrap() }); diff --git a/crates/typst/src/loading/csv.rs b/crates/typst/src/loading/csv.rs index 101b3812..4548bbd7 100644 --- a/crates/typst/src/loading/csv.rs +++ b/crates/typst/src/loading/csv.rs @@ -16,7 +16,7 @@ use crate::World; /// /// # Example /// ```example -/// #let results = csv("data.csv") +/// #let results = csv("example.csv") /// /// #table( /// columns: 2, diff --git a/crates/typst/src/loading/read.rs b/crates/typst/src/loading/read.rs index 50772bee..7ffdbd3a 100644 --- a/crates/typst/src/loading/read.rs +++ b/crates/typst/src/loading/read.rs @@ -16,7 +16,7 @@ use crate::World; /// # Example /// ```example /// An example for a HTML file: \ -/// #let text = read("data.html") +/// #let text = read("example.html") /// #raw(text, lang: "html") /// /// Raw bytes: diff --git a/crates/typst/src/visualize/color.rs b/crates/typst/src/visualize/color.rs index 29bd3fca..f51875bc 100644 --- a/crates/typst/src/visualize/color.rs +++ b/crates/typst/src/visualize/color.rs @@ -36,10 +36,8 @@ const ANGLE_EPSILON: f32 = 1e-5; /// to convert from CMYK to RGB. It is based on the CGATS TR 001-1995 /// specification. See /// https://github.com/saucecontrol/Compact-ICC-Profiles#cmyk. -static CGATS001_COMPACT_PROFILE: Lazy> = Lazy::new(|| { - let bytes = include_bytes!("../../assets/CGATS001Compat-v2-micro.icc"); - Profile::new_from_slice(bytes, false).unwrap() -}); +static CMYK_TO_XYZ: Lazy> = + Lazy::new(|| Profile::new_from_slice(typst_assets::icc::CMYK_TO_XYZ, false).unwrap()); /// The target sRGB profile. static SRGB_PROFILE: Lazy> = Lazy::new(|| { @@ -50,7 +48,7 @@ static SRGB_PROFILE: Lazy> = Lazy::new(|| { static TO_SRGB: Lazy = Lazy::new(|| { qcms::Transform::new_to( - &CGATS001_COMPACT_PROFILE, + &CMYK_TO_XYZ, &SRGB_PROFILE, qcms::DataType::CMYK, qcms::DataType::RGB8, -- cgit v1.2.3