diff options
Diffstat (limited to 'crates/typst-kit')
| -rw-r--r-- | crates/typst-kit/Cargo.toml | 57 | ||||
| -rw-r--r-- | crates/typst-kit/src/download.rs | 259 | ||||
| -rw-r--r-- | crates/typst-kit/src/fonts.rs | 216 | ||||
| -rw-r--r-- | crates/typst-kit/src/lib.rs | 27 | ||||
| -rw-r--r-- | crates/typst-kit/src/package.rs | 179 |
5 files changed, 738 insertions, 0 deletions
diff --git a/crates/typst-kit/Cargo.toml b/crates/typst-kit/Cargo.toml new file mode 100644 index 00000000..9c13ceed --- /dev/null +++ b/crates/typst-kit/Cargo.toml @@ -0,0 +1,57 @@ +[package] +name = "typst-kit" +description = "Common utilities for Typst tooling." +version = { workspace = true } +rust-version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +repository = { workspace = true } +license = { workspace = true } +readme = { workspace = true } + +[dependencies] +typst = { workspace = true } +typst-assets = { workspace = true, optional = true } +typst-timing = { workspace = true } +typst-utils = { workspace = true } +ecow = { workspace = true } +env_proxy = { workspace = true, optional = true } +dirs = { workspace = true, optional = true } +flate2 = { workspace = true, optional = true } +fontdb = { workspace = true, optional = true } +native-tls = { workspace = true, optional = true } +once_cell = { workspace = true } +tar = { workspace = true, optional = true } +ureq = { workspace = true, optional = true } + +# Explicitly depend on OpenSSL if applicable, so that we can add the +# `openssl/vendored` feature to it if `vendor-openssl` is enabled. +[target.'cfg(not(any(target_os = "windows", target_os = "macos", target_os = "ios", target_os = "watchos", target_os = "tvos")))'.dependencies] +openssl = { workspace = true } + +[features] +default = ["fonts", "packages"] + +# Add font loading utilitites +fonts = ["dep:fontdb", "fontdb/memmap", "fontdb/fontconfig"] + +# Add generic downloading utilities +downloads = ["dep:env_proxy", "dep:native-tls", "dep:ureq"] + +# Add package downloading utilities, implies `downloads` +packages = ["downloads", "dep:dirs", "dep:flate2", "dep:tar"] + +# Embeds some fonts into the binary: +# - For text: Linux Libertine, New Computer Modern +# - For math: New Computer Modern Math +# - For code: Deja Vu Sans Mono +# +# Implies `fonts` +embed-fonts = ["fonts", "dep:typst-assets", "typst-assets/fonts"] + +# Whether to vendor OpenSSL. Not applicable to Windows and macOS builds. +vendor-openssl = ["openssl/vendored"] + +[lints] +workspace = true diff --git a/crates/typst-kit/src/download.rs b/crates/typst-kit/src/download.rs new file mode 100644 index 00000000..9aff4dc0 --- /dev/null +++ b/crates/typst-kit/src/download.rs @@ -0,0 +1,259 @@ +// Acknowledgement: +// Closely modelled after rustup's `DownloadTracker`. +// https://github.com/rust-lang/rustup/blob/master/src/cli/download_tracker.rs + +//! Helpers for making various web requests with status reporting. These are +//! primarily used for communicating with package registries. + +use std::collections::VecDeque; +use std::fmt::Debug; +use std::io::{self, ErrorKind, Read}; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use ecow::EcoString; +use native_tls::{Certificate, TlsConnector}; +use once_cell::sync::OnceCell; +use ureq::Response; + +/// Manages progress reporting for downloads. +pub trait Progress { + /// Invoked when a download is started. + fn print_start(&mut self); + + /// Invoked repeatedly while a download is ongoing. + fn print_progress(&mut self, state: &DownloadState); + + /// Invoked when a download is finished. + fn print_finish(&mut self, state: &DownloadState); +} + +/// An implementation of [`Progress`] whth no-op reporting, i.e. reporting +/// events are swallowed. +pub struct ProgressSink; + +impl Progress for ProgressSink { + fn print_start(&mut self) {} + fn print_progress(&mut self, _: &DownloadState) {} + fn print_finish(&mut self, _: &DownloadState) {} +} + +/// The current state of an in progress or finished download. +#[derive(Debug)] +pub struct DownloadState { + /// The expected amount of bytes to download, `None` if the response header + /// was not set. + pub content_len: Option<usize>, + /// The total amount of downloaded bytes until now. + pub total_downloaded: usize, + /// A backlog of the amount of downloaded bytes each second. + pub bytes_per_second: VecDeque<usize>, + /// The download starting instant. + pub start_time: Instant, +} + +/// A minimal https client for downloading various resources. +pub struct Downloader { + user_agent: EcoString, + cert_path: Option<PathBuf>, + cert: OnceCell<Certificate>, +} + +impl Downloader { + /// Crates a new downloader with the given user agent and no certificate. + pub fn new(user_agent: impl Into<EcoString>) -> Self { + Self { + user_agent: user_agent.into(), + cert_path: None, + cert: OnceCell::new(), + } + } + + /// Crates a new downloader with the given user agent and certificate path. + /// + /// If the certificate cannot be read it is set to `None`. + pub fn with_path(user_agent: impl Into<EcoString>, cert_path: PathBuf) -> Self { + Self { + user_agent: user_agent.into(), + cert_path: Some(cert_path), + cert: OnceCell::new(), + } + } + + /// Crates a new downloader with the given user agent and certificate. + pub fn with_cert(user_agent: impl Into<EcoString>, cert: Certificate) -> Self { + Self { + user_agent: user_agent.into(), + cert_path: None, + cert: OnceCell::with_value(cert), + } + } + + /// Returns the certificate this client is using, if a custom certificate + /// is used it is loaded on first access. + /// + /// - Returns `None` if `--cert` and `TYPST_CERT` are not set. + /// - Returns `Some(Ok(cert))` if the certificate was loaded successfully. + /// - Returns `Some(Err(err))` if an error occurred while loading the certificate. + pub fn cert(&self) -> Option<io::Result<&Certificate>> { + self.cert_path.as_ref().map(|path| { + self.cert.get_or_try_init(|| { + let pem = std::fs::read(path)?; + Certificate::from_pem(&pem).map_err(io::Error::other) + }) + }) + } + + /// Download binary data from the given url. + #[allow(clippy::result_large_err)] + pub fn download(&self, url: &str) -> Result<ureq::Response, ureq::Error> { + let mut builder = ureq::AgentBuilder::new(); + let mut tls = TlsConnector::builder(); + + // Set user agent. + builder = builder.user_agent(&self.user_agent); + + // Get the network proxy config from the environment and apply it. + if let Some(proxy) = env_proxy::for_url_str(url) + .to_url() + .and_then(|url| ureq::Proxy::new(url).ok()) + { + builder = builder.proxy(proxy); + } + + // Apply a custom CA certificate if present. + if let Some(cert) = self.cert() { + tls.add_root_certificate(cert?.clone()); + } + + // Configure native TLS. + let connector = + tls.build().map_err(|err| io::Error::new(io::ErrorKind::Other, err))?; + builder = builder.tls_connector(Arc::new(connector)); + + builder.build().get(url).call() + } + + /// Download binary data from the given url and report its progress. + #[allow(clippy::result_large_err)] + pub fn download_with_progress( + &self, + url: &str, + progress: &mut dyn Progress, + ) -> Result<Vec<u8>, ureq::Error> { + progress.print_start(); + let response = self.download(url)?; + Ok(RemoteReader::from_response(response, progress).download()?) + } +} + +impl Debug for Downloader { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Downloader") + .field("user_agent", &self.user_agent) + .field("cert_path", &self.cert_path) + .field( + "cert", + &self + .cert + .get() + .map(|_| typst_utils::debug(|f| write!(f, "Certificate(..)"))), + ) + .finish() + } +} + +/// Keep track of this many download speed samples. +const SAMPLES: usize = 5; + +/// A wrapper around [`ureq::Response`] that reads the response body in chunks +/// over a websocket and reports its progress. +struct RemoteReader<'p> { + /// The reader returned by the ureq::Response. + reader: Box<dyn Read + Send + Sync + 'static>, + /// The download state, holding download metadata for progress reporting. + state: DownloadState, + /// The instant at which progress was last reported. + last_progress: Option<Instant>, + /// A trait object used to report download progress. + progress: &'p mut dyn Progress, +} + +impl<'p> RemoteReader<'p> { + /// Wraps a [`ureq::Response`] and prepares it for downloading. + /// + /// The 'Content-Length' header is used as a size hint for read + /// optimization, if present. + fn from_response(response: Response, progress: &'p mut dyn Progress) -> Self { + let content_len: Option<usize> = response + .header("Content-Length") + .and_then(|header| header.parse().ok()); + + Self { + reader: response.into_reader(), + last_progress: None, + state: DownloadState { + content_len, + total_downloaded: 0, + bytes_per_second: VecDeque::with_capacity(SAMPLES), + start_time: Instant::now(), + }, + progress, + } + } + + /// Download the body's content as raw bytes while reporting download + /// progress. + fn download(mut self) -> io::Result<Vec<u8>> { + let mut buffer = vec![0; 8192]; + let mut data = match self.state.content_len { + Some(content_len) => Vec::with_capacity(content_len), + None => Vec::with_capacity(8192), + }; + + let mut downloaded_this_sec = 0; + loop { + let read = match self.reader.read(&mut buffer) { + Ok(0) => break, + Ok(n) => n, + // If the data is not yet ready but will be available eventually + // keep trying until we either get an actual error, receive data + // or an Ok(0). + Err(ref e) if e.kind() == ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + }; + + data.extend(&buffer[..read]); + + let last_printed = match self.last_progress { + Some(prev) => prev, + None => { + let current_time = Instant::now(); + self.last_progress = Some(current_time); + current_time + } + }; + let elapsed = Instant::now().saturating_duration_since(last_printed); + + downloaded_this_sec += read; + self.state.total_downloaded += read; + + if elapsed >= Duration::from_secs(1) { + if self.state.bytes_per_second.len() == SAMPLES { + self.state.bytes_per_second.pop_back(); + } + + self.state.bytes_per_second.push_front(downloaded_this_sec); + downloaded_this_sec = 0; + + self.progress.print_progress(&self.state); + self.last_progress = Some(Instant::now()); + } + } + + self.progress.print_finish(&self.state); + + Ok(data) + } +} diff --git a/crates/typst-kit/src/fonts.rs b/crates/typst-kit/src/fonts.rs new file mode 100644 index 00000000..07a41f4b --- /dev/null +++ b/crates/typst-kit/src/fonts.rs @@ -0,0 +1,216 @@ +//! Default implementation for searching local and system installed fonts as +//! well as loading embedded default fonts. +//! +//! # Embedded fonts +//! The following fonts are available as embedded fonts via the `embed-fonts` +//! feature flag: +//! - For text: Linux Libertine, New Computer Modern +//! - For math: New Computer Modern Math +//! - For code: Deja Vu Sans Mono + +use std::path::PathBuf; +use std::sync::OnceLock; +use std::{fs, path::Path}; + +use fontdb::{Database, Source}; +use typst::text::{Font, FontBook, FontInfo}; +use typst_timing::TimingScope; + +/// Holds details about the location of a font and lazily the font itself. +#[derive(Debug)] +pub struct FontSlot { + /// The path at which the font can be found on the system. + path: Option<PathBuf>, + /// The index of the font in its collection. Zero if the path does not point + /// to a collection. + index: u32, + /// The lazily loaded font. + font: OnceLock<Option<Font>>, +} + +impl FontSlot { + /// Returns the path at which the font can be found on the system, or `None` + /// if the font was embedded. + pub fn path(&self) -> Option<&Path> { + self.path.as_deref() + } + + /// Returns the index of the font in its collection. Zero if the path does + /// not point to a collection. + pub fn index(&self) -> u32 { + self.index + } + + /// Get the font for this slot. This loads the font into memory on first + /// access. + pub fn get(&self) -> Option<Font> { + self.font + .get_or_init(|| { + let _scope = TimingScope::new("load font", None); + let data = fs::read( + self.path + .as_ref() + .expect("`path` is not `None` if `font` is uninitialized"), + ) + .ok()? + .into(); + Font::new(data, self.index) + }) + .clone() + } +} + +/// The result of a font search, created by calling [`FontSearcher::search`]. +#[derive(Debug)] +pub struct Fonts { + /// Metadata about all discovered fonts. + pub book: FontBook, + /// Slots that the fonts are loaded into. + pub fonts: Vec<FontSlot>, +} + +impl Fonts { + /// Creates a new font searcer with the default settings. + pub fn searcher() -> FontSearcher { + FontSearcher::new() + } +} + +/// Searches for fonts. +/// +/// Fonts are added in the following order (descending priority): +/// 1. Font directories +/// 2. System fonts (if included & enabled) +/// 3. Embedded fonts (if enabled) +#[derive(Debug)] +pub struct FontSearcher { + db: Database, + include_system_fonts: bool, + #[cfg(feature = "embed-fonts")] + include_embedded_fonts: bool, + book: FontBook, + fonts: Vec<FontSlot>, +} + +impl FontSearcher { + /// Create a new, empty system searcher. The searcher is created with the + /// default configuration, it will include embedded fonts and system fonts. + pub fn new() -> Self { + Self { + db: Database::new(), + include_system_fonts: true, + #[cfg(feature = "embed-fonts")] + include_embedded_fonts: true, + book: FontBook::new(), + fonts: vec![], + } + } + + /// Whether to search for and load system fonts, defaults to `true`. + pub fn include_system_fonts(&mut self, value: bool) -> &mut Self { + self.include_system_fonts = value; + self + } + + /// Whether to load embedded fonts, defaults to `true`. + #[cfg(feature = "embed-fonts")] + pub fn include_embedded_fonts(&mut self, value: bool) -> &mut Self { + self.include_embedded_fonts = value; + self + } + + /// Start searching for and loading fonts. To additionally load fonts + /// from specific directories, use [`search_with`][Self::search_with]. + /// + /// # Examples + /// ```no_run + /// # use typst_kit::fonts::FontSearcher; + /// let fonts = FontSearcher::new() + /// .include_system_fonts(true) + /// .search(); + /// ``` + pub fn search(&mut self) -> Fonts { + self.search_with::<_, &str>([]) + } + + /// Start searching for and loading fonts, with additional directories. + /// + /// # Examples + /// ```no_run + /// # use typst_kit::fonts::FontSearcher; + /// let fonts = FontSearcher::new() + /// .include_system_fonts(true) + /// .search_with(["./assets/fonts/"]); + /// ``` + pub fn search_with<I, P>(&mut self, font_dirs: I) -> Fonts + where + I: IntoIterator<Item = P>, + P: AsRef<Path>, + { + // Font paths have highest priority. + for path in font_dirs { + self.db.load_fonts_dir(path); + } + + if self.include_system_fonts { + // System fonts have second priority. + self.db.load_system_fonts(); + } + + for face in self.db.faces() { + let path = match &face.source { + Source::File(path) | Source::SharedFile(path, _) => path, + // We never add binary sources to the database, so there + // shouln't be any. + Source::Binary(_) => continue, + }; + + let info = self + .db + .with_face_data(face.id, FontInfo::new) + .expect("database must contain this font"); + + if let Some(info) = info { + self.book.push(info); + self.fonts.push(FontSlot { + path: Some(path.clone()), + index: face.index, + font: OnceLock::new(), + }); + } + } + + // Embedded fonts have lowest priority. + #[cfg(feature = "embed-fonts")] + if self.include_embedded_fonts { + self.add_embedded(); + } + + Fonts { + book: std::mem::take(&mut self.book), + fonts: std::mem::take(&mut self.fonts), + } + } + + /// Add fonts that are embedded in the binary. + #[cfg(feature = "embed-fonts")] + fn add_embedded(&mut self) { + for data in typst_assets::fonts() { + let buffer = typst::foundations::Bytes::from_static(data); + for (i, font) in Font::iter(buffer).enumerate() { + self.book.push(font.info().clone()); + self.fonts.push(FontSlot { + path: None, + index: i as u32, + font: OnceLock::from(Some(font)), + }); + } + } + } +} + +impl Default for FontSearcher { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/typst-kit/src/lib.rs b/crates/typst-kit/src/lib.rs new file mode 100644 index 00000000..4301727f --- /dev/null +++ b/crates/typst-kit/src/lib.rs @@ -0,0 +1,27 @@ +//! Typst-kit contains various default implementations of functionality used in +//! typst-cli. It is intended as a single source of truth for things like font +//! searching, package downloads and more. Each component of typst-kit is +//! optional, but enabled by default. +//! +//! # Components +//! - [fonts] contains a default implementation for searching local and system +//! installed fonts. It is enabled by the `fonts` feature flag, additionally +//! the `embed-fonts` feature can be used to embed the Typst default fonts. +//! - For text: Linux Libertine, New Computer Modern +//! - For math: New Computer Modern Math +//! - For code: Deja Vu Sans Mono +//! - [download] contains functionality for making simple web requests with +//! status reporting, useful for downloading packages from package registires. +//! It is enabled by the `downloads` feature flag, additionally the +//! `vendor-openssl` can be used on operating systems other than macOS and +//! Windows to vendor OpenSSL when building. +//! - [package] contains package storage and downloading functionality based on +//! [download]. It is enabled by the `packages` feature flag and implies the +//! `downloads` feature flag. + +#[cfg(feature = "downloads")] +pub mod download; +#[cfg(feature = "fonts")] +pub mod fonts; +#[cfg(feature = "packages")] +pub mod package; diff --git a/crates/typst-kit/src/package.rs b/crates/typst-kit/src/package.rs new file mode 100644 index 00000000..ad69df01 --- /dev/null +++ b/crates/typst-kit/src/package.rs @@ -0,0 +1,179 @@ +//! Download and unpack packages and package indices. + +use std::fs; +use std::path::{Path, PathBuf}; + +use ecow::eco_format; +use once_cell::sync::OnceCell; +use typst::diag::{bail, PackageError, PackageResult, StrResult}; +use typst::syntax::package::{ + PackageInfo, PackageSpec, PackageVersion, VersionlessPackageSpec, +}; + +use crate::download::{Downloader, Progress}; + +/// The default Typst registry. +pub const DEFAULT_REGISTRY: &str = "https://packages.typst.org"; + +/// The default packages sub directory within the package and package cache paths. +pub const DEFAULT_PACKAGES_SUBDIR: &str = "typst/packages"; + +/// Holds information about where packages should be stored and downloads them +/// on demand, if possible. +#[derive(Debug)] +pub struct PackageStorage { + /// The path at which non-local packages should be stored when downloaded. + package_cache_path: Option<PathBuf>, + /// The path at which local packages are stored. + package_path: Option<PathBuf>, + /// The downloader used for fetching the index and packages. + downloader: Downloader, + /// The cached index of the preview namespace. + index: OnceCell<Vec<PackageInfo>>, +} + +impl PackageStorage { + /// Creates a new package storage for the given package paths. Falls back to + /// the recommended XDG directories if they are `None`. + pub fn new( + package_cache_path: Option<PathBuf>, + package_path: Option<PathBuf>, + downloader: Downloader, + ) -> Self { + Self { + package_cache_path: package_cache_path.or_else(|| { + dirs::cache_dir().map(|cache_dir| cache_dir.join(DEFAULT_PACKAGES_SUBDIR)) + }), + package_path: package_path.or_else(|| { + dirs::data_dir().map(|data_dir| data_dir.join(DEFAULT_PACKAGES_SUBDIR)) + }), + downloader, + index: OnceCell::new(), + } + } + + /// Returns a the path at which non-local packages should be stored when + /// downloaded. + pub fn package_cache_path(&self) -> Option<&Path> { + self.package_cache_path.as_deref() + } + + /// Returns a the path at which local packages are stored. + pub fn package_path(&self) -> Option<&Path> { + self.package_path.as_deref() + } + + /// Make a package available in the on-disk. + pub fn prepare_package( + &self, + spec: &PackageSpec, + progress: &mut dyn Progress, + ) -> PackageResult<PathBuf> { + let subdir = format!("{}/{}/{}", spec.namespace, spec.name, spec.version); + + if let Some(packages_dir) = &self.package_path { + let dir = packages_dir.join(&subdir); + if dir.exists() { + return Ok(dir); + } + } + + if let Some(cache_dir) = &self.package_cache_path { + let dir = cache_dir.join(&subdir); + if dir.exists() { + return Ok(dir); + } + + // Download from network if it doesn't exist yet. + self.download_package(spec, &dir, progress)?; + if dir.exists() { + return Ok(dir); + } + } + + Err(PackageError::NotFound(spec.clone())) + } + + /// Try to determine the latest version of a package. + pub fn determine_latest_version( + &self, + spec: &VersionlessPackageSpec, + ) -> StrResult<PackageVersion> { + if spec.namespace == "preview" { + // For `@preview`, download the package index and find the latest + // version. + self.download_index()? + .iter() + .filter(|package| package.name == spec.name) + .map(|package| package.version) + .max() + .ok_or_else(|| eco_format!("failed to find package {spec}")) + } else { + // For other namespaces, search locally. We only search in the data + // directory and not the cache directory, because the latter is not + // intended for storage of local packages. + let subdir = format!("{}/{}", spec.namespace, spec.name); + self.package_path + .iter() + .flat_map(|dir| std::fs::read_dir(dir.join(&subdir)).ok()) + .flatten() + .filter_map(|entry| entry.ok()) + .map(|entry| entry.path()) + .filter_map(|path| path.file_name()?.to_string_lossy().parse().ok()) + .max() + .ok_or_else(|| eco_format!("please specify the desired version")) + } + } + + /// Download the package index. The result of this is cached for efficiency. + pub fn download_index(&self) -> StrResult<&Vec<PackageInfo>> { + self.index.get_or_try_init(|| { + let url = format!("{DEFAULT_REGISTRY}/preview/index.json"); + match self.downloader.download(&url) { + Ok(response) => response + .into_json() + .map_err(|err| eco_format!("failed to parse package index: {err}")), + Err(ureq::Error::Status(404, _)) => { + bail!("failed to fetch package index (not found)") + } + Err(err) => bail!("failed to fetch package index ({err})"), + } + }) + } + + /// Download a package over the network. + /// + /// # Panics + /// Panics if the package spec namespace isn't `preview`. + pub fn download_package( + &self, + spec: &PackageSpec, + package_dir: &Path, + progress: &mut dyn Progress, + ) -> PackageResult<()> { + assert_eq!(spec.namespace, "preview"); + + let url = + format!("{DEFAULT_REGISTRY}/preview/{}-{}.tar.gz", spec.name, spec.version); + + let data = match self.downloader.download_with_progress(&url, progress) { + Ok(data) => data, + Err(ureq::Error::Status(404, _)) => { + if let Ok(version) = self.determine_latest_version(&spec.versionless()) { + return Err(PackageError::VersionNotFound(spec.clone(), version)); + } else { + return Err(PackageError::NotFound(spec.clone())); + } + } + Err(err) => { + return Err(PackageError::NetworkFailed(Some(eco_format!("{err}")))) + } + }; + + let decompressed = flate2::read::GzDecoder::new(data.as_slice()); + tar::Archive::new(decompressed).unpack(package_dir).map_err(|err| { + fs::remove_dir_all(package_dir).ok(); + PackageError::MalformedArchive(Some(eco_format!("{err}"))) + }) + } +} |
