summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authortingerrr <me@tinger.dev>2024-08-05 20:49:02 +0200
committerGitHub <noreply@github.com>2024-08-05 18:49:02 +0000
commit672f6e5f97c2bdcd3e94754c9486869bf7b8de56 (patch)
tree14dc74c6f841f69e7e820240af84b41f7c32b52b
parent810491c9d31b614a435020f888fbd380e8e039a1 (diff)
Add typst-kit crate (#4540)
-rw-r--r--Cargo.lock25
-rw-r--r--Cargo.toml1
-rw-r--r--crates/typst-cli/Cargo.toml18
-rw-r--r--crates/typst-cli/src/download.rs263
-rw-r--r--crates/typst-cli/src/fonts.rs112
-rw-r--r--crates/typst-cli/src/init.rs8
-rw-r--r--crates/typst-cli/src/package.rs176
-rw-r--r--crates/typst-cli/src/update.rs28
-rw-r--r--crates/typst-cli/src/world.rs22
-rw-r--r--crates/typst-kit/Cargo.toml57
-rw-r--r--crates/typst-kit/src/download.rs259
-rw-r--r--crates/typst-kit/src/fonts.rs216
-rw-r--r--crates/typst-kit/src/lib.rs27
-rw-r--r--crates/typst-kit/src/package.rs179
14 files changed, 890 insertions, 501 deletions
diff --git a/Cargo.lock b/Cargo.lock
index 9d5e9283..d43899f7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2670,15 +2670,11 @@ dependencies = [
"comemo",
"dirs",
"ecow",
- "env_proxy",
- "flate2",
- "fontdb",
"fs_extra",
"native-tls",
"notify",
"once_cell",
"open",
- "openssl",
"parking_lot",
"pathdiff",
"rayon",
@@ -2694,6 +2690,7 @@ dependencies = [
"toml",
"typst",
"typst-assets",
+ "typst-kit",
"typst-macros",
"typst-pdf",
"typst-render",
@@ -2761,6 +2758,26 @@ dependencies = [
]
[[package]]
+name = "typst-kit"
+version = "0.11.0"
+dependencies = [
+ "dirs",
+ "ecow",
+ "env_proxy",
+ "flate2",
+ "fontdb",
+ "native-tls",
+ "once_cell",
+ "openssl",
+ "tar",
+ "typst",
+ "typst-assets",
+ "typst-timing",
+ "typst-utils",
+ "ureq",
+]
+
+[[package]]
name = "typst-macros"
version = "0.11.0"
dependencies = [
diff --git a/Cargo.toml b/Cargo.toml
index bfd3b140..18b670f0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -19,6 +19,7 @@ readme = "README.md"
typst = { path = "crates/typst", version = "0.11.0" }
typst-cli = { path = "crates/typst-cli", version = "0.11.0" }
typst-ide = { path = "crates/typst-ide", version = "0.11.0" }
+typst-kit = { path = "crates/typst-kit", version = "0.11.0" }
typst-macros = { path = "crates/typst-macros", version = "0.11.0" }
typst-pdf = { path = "crates/typst-pdf", version = "0.11.0" }
typst-render = { path = "crates/typst-render", version = "0.11.0" }
diff --git a/crates/typst-cli/Cargo.toml b/crates/typst-cli/Cargo.toml
index 7ada123c..31f19f39 100644
--- a/crates/typst-cli/Cargo.toml
+++ b/crates/typst-cli/Cargo.toml
@@ -20,6 +20,7 @@ doc = false
[dependencies]
typst = { workspace = true }
typst-assets = { workspace = true, features = ["fonts"] }
+typst-kit = { workspace = true }
typst-macros = { workspace = true }
typst-pdf = { workspace = true }
typst-render = { workspace = true }
@@ -31,9 +32,6 @@ codespan-reporting = { workspace = true }
comemo = { workspace = true }
dirs = { workspace = true }
ecow = { workspace = true }
-env_proxy = { workspace = true }
-flate2 = { workspace = true }
-fontdb = { workspace = true, features = ["memmap", "fontconfig"] }
fs_extra = { workspace = true }
native-tls = { workspace = true }
notify = { workspace = true }
@@ -56,11 +54,6 @@ ureq = { workspace = true }
xz2 = { workspace = true, optional = true }
zip = { workspace = true, optional = true }
-# Explicitly depend on OpenSSL if applicable, so that we can add the
-# `openssl/vendored` feature to it if `vendor-openssl` is enabled.
-[target.'cfg(not(any(target_os = "windows", target_os = "macos", target_os = "ios", target_os = "watchos", target_os = "tvos")))'.dependencies]
-openssl = { workspace = true }
-
[build-dependencies]
chrono = { workspace = true }
clap = { workspace = true, features = ["string"] }
@@ -71,17 +64,14 @@ semver = { workspace = true }
[features]
default = ["embed-fonts"]
-# Embeds some fonts into the binary:
-# - For text: Linux Libertine, New Computer Modern
-# - For math: New Computer Modern Math
-# - For code: Deja Vu Sans Mono
-embed-fonts = []
+# Embeds some fonts into the binary, see typst-kit
+embed-fonts = ["typst-kit/embed-fonts"]
# Permits the CLI to update itself without a package manager.
self-update = ["dep:self-replace", "dep:xz2", "dep:zip"]
# Whether to vendor OpenSSL. Not applicable to Windows and macOS builds.
-vendor-openssl = ["openssl/vendored"]
+vendor-openssl = ["typst-kit/vendor-openssl"]
[lints]
workspace = true
diff --git a/crates/typst-cli/src/download.rs b/crates/typst-cli/src/download.rs
index 63a2e416..8082fa52 100644
--- a/crates/typst-cli/src/download.rs
+++ b/crates/typst-cli/src/download.rs
@@ -1,207 +1,92 @@
-// Acknowledgement:
-// Closely modelled after rustup's `DownloadTracker`.
-// https://github.com/rust-lang/rustup/blob/master/src/cli/download_tracker.rs
-
-use std::collections::VecDeque;
-use std::io::{self, ErrorKind, Read, Write};
-use std::sync::Arc;
+use std::fmt::Display;
+use std::io;
+use std::io::Write;
use std::time::{Duration, Instant};
-use native_tls::{Certificate, TlsConnector};
-use once_cell::sync::OnceCell;
-use ureq::Response;
-
-use crate::terminal;
-
-/// Keep track of this many download speed samples.
-const SPEED_SAMPLES: usize = 5;
-
-/// Load a certificate from the file system if the `--cert` argument or
-/// `TYPST_CERT` environment variable is present. The certificate is cached for
-/// efficiency.
-///
-/// - Returns `None` if `--cert` and `TYPST_CERT` are not set.
-/// - Returns `Some(Ok(cert))` if the certificate was loaded successfully.
-/// - Returns `Some(Err(err))` if an error occurred while loading the certificate.
-fn cert() -> Option<Result<&'static Certificate, io::Error>> {
- static CERT: OnceCell<Certificate> = OnceCell::new();
- crate::ARGS.cert.as_ref().map(|path| {
- CERT.get_or_try_init(|| {
- let pem = std::fs::read(path)?;
- Certificate::from_pem(&pem).map_err(io::Error::other)
- })
- })
-}
+use codespan_reporting::term;
+use codespan_reporting::term::termcolor::WriteColor;
+use typst_kit::download::{DownloadState, Downloader, Progress};
-/// Download binary data and display its progress.
-#[allow(clippy::result_large_err)]
-pub fn download_with_progress(url: &str) -> Result<Vec<u8>, ureq::Error> {
- let response = download(url)?;
- Ok(RemoteReader::from_response(response).download()?)
-}
+use crate::terminal::{self, TermOut};
+use crate::ARGS;
-/// Download from a URL.
-#[allow(clippy::result_large_err)]
-pub fn download(url: &str) -> Result<ureq::Response, ureq::Error> {
- let mut builder = ureq::AgentBuilder::new();
- let mut tls = TlsConnector::builder();
-
- // Set user agent.
- builder = builder.user_agent(concat!("typst/", env!("CARGO_PKG_VERSION")));
-
- // Get the network proxy config from the environment and apply it.
- if let Some(proxy) = env_proxy::for_url_str(url)
- .to_url()
- .and_then(|url| ureq::Proxy::new(url).ok())
- {
- builder = builder.proxy(proxy);
- }
+/// Prints download progress by writing `downloading {0}` followed by repeatedly
+/// updating the last terminal line.
+pub struct PrintDownload<T>(pub T);
- // Apply a custom CA certificate if present.
- if let Some(cert) = cert() {
- tls.add_root_certificate(cert?.clone());
- }
+impl<T: Display> Progress for PrintDownload<T> {
+ fn print_start(&mut self) {
+ // Print that a package downloading is happening.
+ let styles = term::Styles::default();
- // Configure native TLS.
- let connector =
- tls.build().map_err(|err| io::Error::new(io::ErrorKind::Other, err))?;
- builder = builder.tls_connector(Arc::new(connector));
-
- builder.build().get(url).call()
-}
-
-/// A wrapper around [`ureq::Response`] that reads the response body in chunks
-/// over a websocket and displays statistics about its progress.
-///
-/// Downloads will _never_ fail due to statistics failing to print, print errors
-/// are silently ignored.
-struct RemoteReader {
- reader: Box<dyn Read + Send + Sync + 'static>,
- content_len: Option<usize>,
- total_downloaded: usize,
- downloaded_this_sec: usize,
- downloaded_last_few_secs: VecDeque<usize>,
- start_time: Instant,
- last_print: Option<Instant>,
-}
+ let mut out = terminal::out();
+ let _ = out.set_color(&styles.header_help);
+ let _ = write!(out, "downloading");
-impl RemoteReader {
- /// Wraps a [`ureq::Response`] and prepares it for downloading.
- ///
- /// The 'Content-Length' header is used as a size hint for read
- /// optimization, if present.
- pub fn from_response(response: Response) -> Self {
- let content_len: Option<usize> = response
- .header("Content-Length")
- .and_then(|header| header.parse().ok());
-
- Self {
- reader: response.into_reader(),
- content_len,
- total_downloaded: 0,
- downloaded_this_sec: 0,
- downloaded_last_few_secs: VecDeque::with_capacity(SPEED_SAMPLES),
- start_time: Instant::now(),
- last_print: None,
- }
+ let _ = out.reset();
+ let _ = writeln!(out, " {}", self.0);
}
- /// Download the bodies content as raw bytes while attempting to print
- /// download statistics to standard error. Download progress gets displayed
- /// and updated every second.
- ///
- /// These statistics will never prevent a download from completing, errors
- /// are silently ignored.
- pub fn download(mut self) -> io::Result<Vec<u8>> {
- let mut buffer = vec![0; 8192];
- let mut data = match self.content_len {
- Some(content_len) => Vec::with_capacity(content_len),
- None => Vec::with_capacity(8192),
- };
-
- loop {
- let read = match self.reader.read(&mut buffer) {
- Ok(0) => break,
- Ok(n) => n,
- // If the data is not yet ready but will be available eventually
- // keep trying until we either get an actual error, receive data
- // or an Ok(0).
- Err(ref e) if e.kind() == ErrorKind::Interrupted => continue,
- Err(e) => return Err(e),
- };
-
- data.extend(&buffer[..read]);
-
- let last_printed = match self.last_print {
- Some(prev) => prev,
- None => {
- let current_time = Instant::now();
- self.last_print = Some(current_time);
- current_time
- }
- };
- let elapsed = Instant::now().saturating_duration_since(last_printed);
-
- self.total_downloaded += read;
- self.downloaded_this_sec += read;
-
- if elapsed >= Duration::from_secs(1) {
- if self.downloaded_last_few_secs.len() == SPEED_SAMPLES {
- self.downloaded_last_few_secs.pop_back();
- }
-
- self.downloaded_last_few_secs.push_front(self.downloaded_this_sec);
- self.downloaded_this_sec = 0;
-
- terminal::out().clear_last_line()?;
- self.display()?;
- self.last_print = Some(Instant::now());
- }
- }
-
- self.display()?;
- writeln!(&mut terminal::out())?;
+ fn print_progress(&mut self, state: &DownloadState) {
+ let mut out = terminal::out();
+ let _ = out.clear_last_line();
+ let _ = display_download_progress(&mut out, state);
+ }
- Ok(data)
+ fn print_finish(&mut self, state: &DownloadState) {
+ let mut out = terminal::out();
+ let _ = display_download_progress(&mut out, state);
+ let _ = writeln!(out);
}
+}
- /// Compile and format several download statistics and make an attempt at
- /// displaying them on standard error.
- fn display(&mut self) -> io::Result<()> {
- let sum: usize = self.downloaded_last_few_secs.iter().sum();
- let len = self.downloaded_last_few_secs.len();
- let speed = if len > 0 { sum / len } else { self.content_len.unwrap_or(0) };
-
- let total_downloaded = as_bytes_unit(self.total_downloaded);
- let speed_h = as_throughput_unit(speed);
- let elapsed =
- time_suffix(Instant::now().saturating_duration_since(self.start_time));
-
- match self.content_len {
- Some(content_len) => {
- let percent = (self.total_downloaded as f64 / content_len as f64) * 100.;
- let remaining = content_len - self.total_downloaded;
-
- let download_size = as_bytes_unit(content_len);
- let eta = time_suffix(Duration::from_secs(if speed == 0 {
- 0
- } else {
- (remaining / speed) as u64
- }));
- writeln!(
- terminal::out(),
- "{total_downloaded} / {download_size} ({percent:3.0} %) {speed_h} in {elapsed} ETA: {eta}",
- )?;
- }
- None => writeln!(
- terminal::out(),
- "Total downloaded: {total_downloaded} Speed: {speed_h} Elapsed: {elapsed}",
- )?,
- };
- Ok(())
+/// Returns a new downloader.
+pub fn downloader() -> Downloader {
+ let user_agent = concat!("typst/", env!("CARGO_PKG_VERSION"));
+ match ARGS.cert.clone() {
+ Some(cert) => Downloader::with_path(user_agent, cert),
+ None => Downloader::new(user_agent),
}
}
+/// Compile and format several download statistics and make and attempt at
+/// displaying them on standard error.
+pub fn display_download_progress(
+ out: &mut TermOut,
+ state: &DownloadState,
+) -> io::Result<()> {
+ let sum: usize = state.bytes_per_second.iter().sum();
+ let len = state.bytes_per_second.len();
+ let speed = if len > 0 { sum / len } else { state.content_len.unwrap_or(0) };
+
+ let total_downloaded = as_bytes_unit(state.total_downloaded);
+ let speed_h = as_throughput_unit(speed);
+ let elapsed = time_suffix(Instant::now().saturating_duration_since(state.start_time));
+
+ match state.content_len {
+ Some(content_len) => {
+ let percent = (state.total_downloaded as f64 / content_len as f64) * 100.;
+ let remaining = content_len - state.total_downloaded;
+
+ let download_size = as_bytes_unit(content_len);
+ let eta = time_suffix(Duration::from_secs(if speed == 0 {
+ 0
+ } else {
+ (remaining / speed) as u64
+ }));
+ writeln!(
+ out,
+ "{total_downloaded} / {download_size} ({percent:3.0} %) {speed_h} in {elapsed} ETA: {eta}",
+ )?;
+ }
+ None => writeln!(
+ out,
+ "Total downloaded: {total_downloaded} Speed: {speed_h} Elapsed: {elapsed}",
+ )?,
+ };
+ Ok(())
+}
+
/// Append a unit-of-time suffix.
fn time_suffix(duration: Duration) -> String {
let secs = duration.as_secs();
diff --git a/crates/typst-cli/src/fonts.rs b/crates/typst-cli/src/fonts.rs
index de9d1fc1..f5aa9826 100644
--- a/crates/typst-cli/src/fonts.rs
+++ b/crates/typst-cli/src/fonts.rs
@@ -1,20 +1,16 @@
-use std::fs;
-use std::path::PathBuf;
-use std::sync::OnceLock;
-
-use fontdb::{Database, Source};
use typst::diag::StrResult;
-use typst::text::{Font, FontBook, FontInfo, FontVariant};
-use typst_timing::TimingScope;
+use typst::text::FontVariant;
+use typst_kit::fonts::Fonts;
use crate::args::FontsCommand;
/// Execute a font listing command.
pub fn fonts(command: &FontsCommand) -> StrResult<()> {
- let mut searcher = FontSearcher::new();
- searcher.search(&command.font_args.font_paths, command.font_args.ignore_system_fonts);
+ let fonts = Fonts::searcher()
+ .include_system_fonts(!command.font_args.ignore_system_fonts)
+ .search_with(&command.font_args.font_paths);
- for (name, infos) in searcher.book.families() {
+ for (name, infos) in fonts.book.families() {
println!("{name}");
if command.variants {
for info in infos {
@@ -26,99 +22,3 @@ pub fn fonts(command: &FontsCommand) -> StrResult<()> {
Ok(())
}
-
-/// Searches for fonts.
-pub struct FontSearcher {
- /// Metadata about all discovered fonts.
- pub book: FontBook,
- /// Slots that the fonts are loaded into.
- pub fonts: Vec<FontSlot>,
-}
-
-/// Holds details about the location of a font and lazily the font itself.
-pub struct FontSlot {
- /// The path at which the font can be found on the system.
- path: PathBuf,
- /// The index of the font in its collection. Zero if the path does not point
- /// to a collection.
- index: u32,
- /// The lazily loaded font.
- font: OnceLock<Option<Font>>,
-}
-
-impl FontSlot {
- /// Get the font for this slot.
- pub fn get(&self) -> Option<Font> {
- self.font
- .get_or_init(|| {
- let _scope = TimingScope::new("load font", None);
- let data = fs::read(&self.path).ok()?.into();
- Font::new(data, self.index)
- })
- .clone()
- }
-}
-
-impl FontSearcher {
- /// Create a new, empty system searcher.
- pub fn new() -> Self {
- Self { book: FontBook::new(), fonts: vec![] }
- }
-
- /// Search everything that is available.
- pub fn search(&mut self, font_paths: &[PathBuf], ignore_system_fonts: bool) {
- let mut db = Database::new();
-
- // Font paths have highest priority.
- for path in font_paths {
- db.load_fonts_dir(path);
- }
-
- if !ignore_system_fonts {
- // System fonts have second priority.
- db.load_system_fonts();
- }
-
- for face in db.faces() {
- let path = match &face.source {
- Source::File(path) | Source::SharedFile(path, _) => path,
- // We never add binary sources to the database, so there
- // shouln't be any.
- Source::Binary(_) => continue,
- };
-
- let info = db
- .with_face_data(face.id, FontInfo::new)
- .expect("database must contain this font");
-
- if let Some(info) = info {
- self.book.push(info);
- self.fonts.push(FontSlot {
- path: path.clone(),
- index: face.index,
- font: OnceLock::new(),
- });
- }
- }
-
- // Embedded fonts have lowest priority.
- #[cfg(feature = "embed-fonts")]
- self.add_embedded();
- }
-
- /// Add fonts that are embedded in the binary.
- #[cfg(feature = "embed-fonts")]
- fn add_embedded(&mut self) {
- for data in typst_assets::fonts() {
- let buffer = typst::foundations::Bytes::from_static(data);
- for (i, font) in Font::iter(buffer).enumerate() {
- self.book.push(font.info().clone());
- self.fonts.push(FontSlot {
- path: PathBuf::new(),
- index: i as u32,
- font: OnceLock::from(Some(font)),
- });
- }
- }
- }
-}
diff --git a/crates/typst-cli/src/init.rs b/crates/typst-cli/src/init.rs
index cb6b6627..842419fc 100644
--- a/crates/typst-cli/src/init.rs
+++ b/crates/typst-cli/src/init.rs
@@ -10,11 +10,12 @@ use typst::syntax::package::{
};
use crate::args::InitCommand;
-use crate::package::PackageStorage;
+use crate::download::PrintDownload;
+use crate::package;
/// Execute an initialization command.
pub fn init(command: &InitCommand) -> StrResult<()> {
- let package_storage = PackageStorage::from_args(&command.package_storage_args);
+ let package_storage = package::storage(&command.package_storage_args);
// Parse the package specification. If the user didn't specify the version,
// we try to figure it out automatically by downloading the package index
@@ -28,7 +29,8 @@ pub fn init(command: &InitCommand) -> StrResult<()> {
})?;
// Find or download the package.
- let package_path = package_storage.prepare_package(&spec)?;
+ let package_path =
+ package_storage.prepare_package(&spec, &mut PrintDownload(&spec))?;
// Parse the manifest.
let manifest = parse_manifest(&package_path)?;
diff --git a/crates/typst-cli/src/package.rs b/crates/typst-cli/src/package.rs
index bd5dd549..b4965f89 100644
--- a/crates/typst-cli/src/package.rs
+++ b/crates/typst-cli/src/package.rs
@@ -1,169 +1,13 @@
-use std::fs;
-use std::io::{self, Write};
-use std::path::{Path, PathBuf};
+use typst_kit::package::PackageStorage;
use crate::args::PackageStorageArgs;
-use codespan_reporting::term::{self, termcolor};
-use ecow::eco_format;
-use once_cell::sync::OnceCell;
-use termcolor::WriteColor;
-use typst::diag::{bail, PackageError, PackageResult, StrResult};
-use typst::syntax::package::{
- PackageInfo, PackageSpec, PackageVersion, VersionlessPackageSpec,
-};
-
-use crate::download::{download, download_with_progress};
-use crate::terminal;
-
-const HOST: &str = "https://packages.typst.org";
-const DEFAULT_PACKAGES_SUBDIR: &str = "typst/packages";
-
-/// Holds information about where packages should be stored.
-pub struct PackageStorage {
- pub package_cache_path: Option<PathBuf>,
- pub package_path: Option<PathBuf>,
- index: OnceCell<Vec<PackageInfo>>,
-}
-
-impl PackageStorage {
- pub fn from_args(args: &PackageStorageArgs) -> Self {
- let package_cache_path = args.package_cache_path.clone().or_else(|| {
- dirs::cache_dir().map(|cache_dir| cache_dir.join(DEFAULT_PACKAGES_SUBDIR))
- });
- let package_path = args.package_path.clone().or_else(|| {
- dirs::data_dir().map(|data_dir| data_dir.join(DEFAULT_PACKAGES_SUBDIR))
- });
- Self {
- package_cache_path,
- package_path,
- index: OnceCell::new(),
- }
- }
-
- /// Make a package available in the on-disk cache.
- pub fn prepare_package(&self, spec: &PackageSpec) -> PackageResult<PathBuf> {
- let subdir = format!("{}/{}/{}", spec.namespace, spec.name, spec.version);
-
- if let Some(packages_dir) = &self.package_path {
- let dir = packages_dir.join(&subdir);
- if dir.exists() {
- return Ok(dir);
- }
- }
-
- if let Some(cache_dir) = &self.package_cache_path {
- let dir = cache_dir.join(&subdir);
- if dir.exists() {
- return Ok(dir);
- }
-
- // Download from network if it doesn't exist yet.
- if spec.namespace == "preview" {
- self.download_package(spec, &dir)?;
- if dir.exists() {
- return Ok(dir);
- }
- }
- }
-
- Err(PackageError::NotFound(spec.clone()))
- }
-
- /// Try to determine the latest version of a package.
- pub fn determine_latest_version(
- &self,
- spec: &VersionlessPackageSpec,
- ) -> StrResult<PackageVersion> {
- if spec.namespace == "preview" {
- // For `@preview`, download the package index and find the latest
- // version.
- self.download_index()?
- .iter()
- .filter(|package| package.name == spec.name)
- .map(|package| package.version)
- .max()
- .ok_or_else(|| eco_format!("failed to find package {spec}"))
- } else {
- // For other namespaces, search locally. We only search in the data
- // directory and not the cache directory, because the latter is not
- // intended for storage of local packages.
- let subdir = format!("{}/{}", spec.namespace, spec.name);
- self.package_path
- .iter()
- .flat_map(|dir| std::fs::read_dir(dir.join(&subdir)).ok())
- .flatten()
- .filter_map(|entry| entry.ok())
- .map(|entry| entry.path())
- .filter_map(|path| path.file_name()?.to_string_lossy().parse().ok())
- .max()
- .ok_or_else(|| eco_format!("please specify the desired version"))
- }
- }
-}
-
-impl PackageStorage {
- /// Download a package over the network.
- fn download_package(
- &self,
- spec: &PackageSpec,
- package_dir: &Path,
- ) -> PackageResult<()> {
- // The `@preview` namespace is the only namespace that supports on-demand
- // fetching.
- assert_eq!(spec.namespace, "preview");
-
- let url = format!("{HOST}/preview/{}-{}.tar.gz", spec.name, spec.version);
-
- print_downloading(spec).unwrap();
-
- let data = match download_with_progress(&url) {
- Ok(data) => data,
- Err(ureq::Error::Status(404, _)) => {
- if let Ok(version) = self.determine_latest_version(&spec.versionless()) {
- return Err(PackageError::VersionNotFound(spec.clone(), version));
- } else {
- return Err(PackageError::NotFound(spec.clone()));
- }
- }
- Err(err) => {
- return Err(PackageError::NetworkFailed(Some(eco_format!("{err}"))))
- }
- };
-
- let decompressed = flate2::read::GzDecoder::new(data.as_slice());
- tar::Archive::new(decompressed).unpack(package_dir).map_err(|err| {
- fs::remove_dir_all(package_dir).ok();
- PackageError::MalformedArchive(Some(eco_format!("{err}")))
- })
- }
-
- /// Download the `@preview` package index.
- ///
- /// To avoid downloading the index multiple times, the result is cached.
- fn download_index(&self) -> StrResult<&Vec<PackageInfo>> {
- self.index.get_or_try_init(|| {
- let url = format!("{HOST}/preview/index.json");
- match download(&url) {
- Ok(response) => response
- .into_json()
- .map_err(|err| eco_format!("failed to parse package index: {err}")),
- Err(ureq::Error::Status(404, _)) => {
- bail!("failed to fetch package index (not found)")
- }
- Err(err) => bail!("failed to fetch package index ({err})"),
- }
- })
- }
-}
-
-/// Print that a package downloading is happening.
-fn print_downloading(spec: &PackageSpec) -> io::Result<()> {
- let styles = term::Styles::default();
-
- let mut out = terminal::out();
- out.set_color(&styles.header_help)?;
- write!(out, "downloading")?;
-
- out.reset()?;
- writeln!(out, " {spec}")
+use crate::download;
+
+/// Returns a new package storage for the given args.
+pub fn storage(args: &PackageStorageArgs) -> PackageStorage {
+ PackageStorage::new(
+ args.package_cache_path.clone(),
+ args.package_path.clone(),
+ download::downloader(),
+ )
}
diff --git a/crates/typst-cli/src/update.rs b/crates/typst-cli/src/update.rs
index fa7c3a34..adec4a2c 100644
--- a/crates/typst-cli/src/update.rs
+++ b/crates/typst-cli/src/update.rs
@@ -7,11 +7,12 @@ use semver::Version;
use serde::Deserialize;
use tempfile::NamedTempFile;
use typst::diag::{bail, StrResult};
+use typst_kit::download::Downloader;
use xz2::bufread::XzDecoder;
use zip::ZipArchive;
use crate::args::UpdateCommand;
-use crate::download::{download, download_with_progress};
+use crate::download::{self, PrintDownload};
const TYPST_GITHUB_ORG: &str = "typst";
const TYPST_REPO: &str = "typst";
@@ -68,13 +69,15 @@ pub fn update(command: &UpdateCommand) -> StrResult<()> {
fs::copy(current_exe, &backup_path)
.map_err(|err| eco_format!("failed to create backup ({err})"))?;
- let release = Release::from_tag(command.version.as_ref())?;
+ let downloader = download::downloader();
+
+ let release = Release::from_tag(command.version.as_ref(), &downloader)?;
if !update_needed(&release)? && !command.force {
eprintln!("Already up-to-date.");
return Ok(());
}
- let binary_data = release.download_binary(needed_asset()?)?;
+ let binary_data = release.download_binary(needed_asset()?, &downloader)?;
let mut temp_exe = NamedTempFile::new()
.map_err(|err| eco_format!("failed to create temporary file ({err})"))?;
temp_exe
@@ -106,7 +109,10 @@ struct Release {
impl Release {
/// Download the target release, or latest if version is `None`, from the
/// Typst repository.
- pub fn from_tag(tag: Option<&Version>) -> StrResult<Release> {
+ pub fn from_tag(
+ tag: Option<&Version>,
+ downloader: &Downloader,
+ ) -> StrResult<Release> {
let url = match tag {
Some(tag) => format!(
"https://api.github.com/repos/{TYPST_GITHUB_ORG}/{TYPST_REPO}/releases/tags/v{tag}"
@@ -116,7 +122,7 @@ impl Release {
),
};
- match download(&url) {
+ match downloader.download(&url) {
Ok(response) => response.into_json().map_err(|err| {
eco_format!("failed to parse release information ({err})")
}),
@@ -130,15 +136,21 @@ impl Release {
/// Download the binary from a given [`Release`] and select the
/// corresponding asset for this target platform, returning the raw binary
/// data.
- pub fn download_binary(&self, asset_name: &str) -> StrResult<Vec<u8>> {
+ pub fn download_binary(
+ &self,
+ asset_name: &str,
+ downloader: &Downloader,
+ ) -> StrResult<Vec<u8>> {
let asset = self
.assets
.iter()
.find(|a| a.name.starts_with(asset_name))
.ok_or("could not find release for your target platform")?;
- eprintln!("Downloading release ...");
- let data = match download_with_progress(&asset.browser_download_url) {
+ let data = match downloader.download_with_progress(
+ &asset.browser_download_url,
+ &mut PrintDownload("release"),
+ ) {
Ok(data) => data,
Err(ureq::Error::Status(404, _)) => {
bail!("asset not found (searched for {})", asset.name);
diff --git a/crates/typst-cli/src/world.rs b/crates/typst-cli/src/world.rs
index 5a0814a8..70c63355 100644
--- a/crates/typst-cli/src/world.rs
+++ b/crates/typst-cli/src/world.rs
@@ -14,12 +14,14 @@ use typst::syntax::{FileId, Source, VirtualPath};
use typst::text::{Font, FontBook};
use typst::utils::LazyHash;
use typst::{Library, World};
+use typst_kit::fonts::{FontSlot, Fonts};
+use typst_kit::package::PackageStorage;
use typst_timing::{timed, TimingScope};
use crate::args::{Input, SharedArgs};
use crate::compile::ExportCache;
-use crate::fonts::{FontSearcher, FontSlot};
-use crate::package::PackageStorage;
+use crate::download::PrintDownload;
+use crate::package;
/// Static `FileId` allocated for stdin.
/// This is to ensure that a file is read in the correct way.
@@ -110,26 +112,24 @@ impl SystemWorld {
Library::builder().with_inputs(inputs).build()
};
- let mut searcher = FontSearcher::new();
- searcher
- .search(&command.font_args.font_paths, command.font_args.ignore_system_fonts);
+ let fonts = Fonts::searcher()
+ .include_system_fonts(command.font_args.ignore_system_fonts)
+ .search_with(&command.font_args.font_paths);
let now = match command.creation_timestamp {
Some(time) => Now::Fixed(time),
None => Now::System(OnceLock::new()),
};
- let package_storage = PackageStorage::from_args(&command.package_storage_args);
-
Ok(Self {
workdir: std::env::current_dir().ok(),
root,
main,
library: LazyHash::new(library),
- book: LazyHash::new(searcher.book),
- fonts: searcher.fonts,
+ book: LazyHash::new(fonts.book),
+ fonts: fonts.fonts,
slots: Mutex::new(HashMap::new()),
- package_storage,
+ package_storage: package::storage(&command.package_storage_args),
now,
export_cache: ExportCache::new(),
})
@@ -378,7 +378,7 @@ fn system_path(
let buf;
let mut root = project_root;
if let Some(spec) = id.package() {
- buf = package_storage.prepare_package(spec)?;
+ buf = package_storage.prepare_package(spec, &mut PrintDownload(&spec))?;
root = &buf;
}
diff --git a/crates/typst-kit/Cargo.toml b/crates/typst-kit/Cargo.toml
new file mode 100644
index 00000000..9c13ceed
--- /dev/null
+++ b/crates/typst-kit/Cargo.toml
@@ -0,0 +1,57 @@
+[package]
+name = "typst-kit"
+description = "Common utilities for Typst tooling."
+version = { workspace = true }
+rust-version = { workspace = true }
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+readme = { workspace = true }
+
+[dependencies]
+typst = { workspace = true }
+typst-assets = { workspace = true, optional = true }
+typst-timing = { workspace = true }
+typst-utils = { workspace = true }
+ecow = { workspace = true }
+env_proxy = { workspace = true, optional = true }
+dirs = { workspace = true, optional = true }
+flate2 = { workspace = true, optional = true }
+fontdb = { workspace = true, optional = true }
+native-tls = { workspace = true, optional = true }
+once_cell = { workspace = true }
+tar = { workspace = true, optional = true }
+ureq = { workspace = true, optional = true }
+
+# Explicitly depend on OpenSSL if applicable, so that we can add the
+# `openssl/vendored` feature to it if `vendor-openssl` is enabled.
+[target.'cfg(not(any(target_os = "windows", target_os = "macos", target_os = "ios", target_os = "watchos", target_os = "tvos")))'.dependencies]
+openssl = { workspace = true }
+
+[features]
+default = ["fonts", "packages"]
+
+# Add font loading utilitites
+fonts = ["dep:fontdb", "fontdb/memmap", "fontdb/fontconfig"]
+
+# Add generic downloading utilities
+downloads = ["dep:env_proxy", "dep:native-tls", "dep:ureq"]
+
+# Add package downloading utilities, implies `downloads`
+packages = ["downloads", "dep:dirs", "dep:flate2", "dep:tar"]
+
+# Embeds some fonts into the binary:
+# - For text: Linux Libertine, New Computer Modern
+# - For math: New Computer Modern Math
+# - For code: Deja Vu Sans Mono
+#
+# Implies `fonts`
+embed-fonts = ["fonts", "dep:typst-assets", "typst-assets/fonts"]
+
+# Whether to vendor OpenSSL. Not applicable to Windows and macOS builds.
+vendor-openssl = ["openssl/vendored"]
+
+[lints]
+workspace = true
diff --git a/crates/typst-kit/src/download.rs b/crates/typst-kit/src/download.rs
new file mode 100644
index 00000000..9aff4dc0
--- /dev/null
+++ b/crates/typst-kit/src/download.rs
@@ -0,0 +1,259 @@
+// Acknowledgement:
+// Closely modelled after rustup's `DownloadTracker`.
+// https://github.com/rust-lang/rustup/blob/master/src/cli/download_tracker.rs
+
+//! Helpers for making various web requests with status reporting. These are
+//! primarily used for communicating with package registries.
+
+use std::collections::VecDeque;
+use std::fmt::Debug;
+use std::io::{self, ErrorKind, Read};
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use ecow::EcoString;
+use native_tls::{Certificate, TlsConnector};
+use once_cell::sync::OnceCell;
+use ureq::Response;
+
+/// Manages progress reporting for downloads.
+pub trait Progress {
+ /// Invoked when a download is started.
+ fn print_start(&mut self);
+
+ /// Invoked repeatedly while a download is ongoing.
+ fn print_progress(&mut self, state: &DownloadState);
+
+ /// Invoked when a download is finished.
+ fn print_finish(&mut self, state: &DownloadState);
+}
+
+/// An implementation of [`Progress`] whth no-op reporting, i.e. reporting
+/// events are swallowed.
+pub struct ProgressSink;
+
+impl Progress for ProgressSink {
+ fn print_start(&mut self) {}
+ fn print_progress(&mut self, _: &DownloadState) {}
+ fn print_finish(&mut self, _: &DownloadState) {}
+}
+
+/// The current state of an in progress or finished download.
+#[derive(Debug)]
+pub struct DownloadState {
+ /// The expected amount of bytes to download, `None` if the response header
+ /// was not set.
+ pub content_len: Option<usize>,
+ /// The total amount of downloaded bytes until now.
+ pub total_downloaded: usize,
+ /// A backlog of the amount of downloaded bytes each second.
+ pub bytes_per_second: VecDeque<usize>,
+ /// The download starting instant.
+ pub start_time: Instant,
+}
+
+/// A minimal https client for downloading various resources.
+pub struct Downloader {
+ user_agent: EcoString,
+ cert_path: Option<PathBuf>,
+ cert: OnceCell<Certificate>,
+}
+
+impl Downloader {
+ /// Crates a new downloader with the given user agent and no certificate.
+ pub fn new(user_agent: impl Into<EcoString>) -> Self {
+ Self {
+ user_agent: user_agent.into(),
+ cert_path: None,
+ cert: OnceCell::new(),
+ }
+ }
+
+ /// Crates a new downloader with the given user agent and certificate path.
+ ///
+ /// If the certificate cannot be read it is set to `None`.
+ pub fn with_path(user_agent: impl Into<EcoString>, cert_path: PathBuf) -> Self {
+ Self {
+ user_agent: user_agent.into(),
+ cert_path: Some(cert_path),
+ cert: OnceCell::new(),
+ }
+ }
+
+ /// Crates a new downloader with the given user agent and certificate.
+ pub fn with_cert(user_agent: impl Into<EcoString>, cert: Certificate) -> Self {
+ Self {
+ user_agent: user_agent.into(),
+ cert_path: None,
+ cert: OnceCell::with_value(cert),
+ }
+ }
+
+ /// Returns the certificate this client is using, if a custom certificate
+ /// is used it is loaded on first access.
+ ///
+ /// - Returns `None` if `--cert` and `TYPST_CERT` are not set.
+ /// - Returns `Some(Ok(cert))` if the certificate was loaded successfully.
+ /// - Returns `Some(Err(err))` if an error occurred while loading the certificate.
+ pub fn cert(&self) -> Option<io::Result<&Certificate>> {
+ self.cert_path.as_ref().map(|path| {
+ self.cert.get_or_try_init(|| {
+ let pem = std::fs::read(path)?;
+ Certificate::from_pem(&pem).map_err(io::Error::other)
+ })
+ })
+ }
+
+ /// Download binary data from the given url.
+ #[allow(clippy::result_large_err)]
+ pub fn download(&self, url: &str) -> Result<ureq::Response, ureq::Error> {
+ let mut builder = ureq::AgentBuilder::new();
+ let mut tls = TlsConnector::builder();
+
+ // Set user agent.
+ builder = builder.user_agent(&self.user_agent);
+
+ // Get the network proxy config from the environment and apply it.
+ if let Some(proxy) = env_proxy::for_url_str(url)
+ .to_url()
+ .and_then(|url| ureq::Proxy::new(url).ok())
+ {
+ builder = builder.proxy(proxy);
+ }
+
+ // Apply a custom CA certificate if present.
+ if let Some(cert) = self.cert() {
+ tls.add_root_certificate(cert?.clone());
+ }
+
+ // Configure native TLS.
+ let connector =
+ tls.build().map_err(|err| io::Error::new(io::ErrorKind::Other, err))?;
+ builder = builder.tls_connector(Arc::new(connector));
+
+ builder.build().get(url).call()
+ }
+
+ /// Download binary data from the given url and report its progress.
+ #[allow(clippy::result_large_err)]
+ pub fn download_with_progress(
+ &self,
+ url: &str,
+ progress: &mut dyn Progress,
+ ) -> Result<Vec<u8>, ureq::Error> {
+ progress.print_start();
+ let response = self.download(url)?;
+ Ok(RemoteReader::from_response(response, progress).download()?)
+ }
+}
+
+impl Debug for Downloader {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("Downloader")
+ .field("user_agent", &self.user_agent)
+ .field("cert_path", &self.cert_path)
+ .field(
+ "cert",
+ &self
+ .cert
+ .get()
+ .map(|_| typst_utils::debug(|f| write!(f, "Certificate(..)"))),
+ )
+ .finish()
+ }
+}
+
+/// Keep track of this many download speed samples.
+const SAMPLES: usize = 5;
+
+/// A wrapper around [`ureq::Response`] that reads the response body in chunks
+/// over a websocket and reports its progress.
+struct RemoteReader<'p> {
+ /// The reader returned by the ureq::Response.
+ reader: Box<dyn Read + Send + Sync + 'static>,
+ /// The download state, holding download metadata for progress reporting.
+ state: DownloadState,
+ /// The instant at which progress was last reported.
+ last_progress: Option<Instant>,
+ /// A trait object used to report download progress.
+ progress: &'p mut dyn Progress,
+}
+
+impl<'p> RemoteReader<'p> {
+ /// Wraps a [`ureq::Response`] and prepares it for downloading.
+ ///
+ /// The 'Content-Length' header is used as a size hint for read
+ /// optimization, if present.
+ fn from_response(response: Response, progress: &'p mut dyn Progress) -> Self {
+ let content_len: Option<usize> = response
+ .header("Content-Length")
+ .and_then(|header| header.parse().ok());
+
+ Self {
+ reader: response.into_reader(),
+ last_progress: None,
+ state: DownloadState {
+ content_len,
+ total_downloaded: 0,
+ bytes_per_second: VecDeque::with_capacity(SAMPLES),
+ start_time: Instant::now(),
+ },
+ progress,
+ }
+ }
+
+ /// Download the body's content as raw bytes while reporting download
+ /// progress.
+ fn download(mut self) -> io::Result<Vec<u8>> {
+ let mut buffer = vec![0; 8192];
+ let mut data = match self.state.content_len {
+ Some(content_len) => Vec::with_capacity(content_len),
+ None => Vec::with_capacity(8192),
+ };
+
+ let mut downloaded_this_sec = 0;
+ loop {
+ let read = match self.reader.read(&mut buffer) {
+ Ok(0) => break,
+ Ok(n) => n,
+ // If the data is not yet ready but will be available eventually
+ // keep trying until we either get an actual error, receive data
+ // or an Ok(0).
+ Err(ref e) if e.kind() == ErrorKind::Interrupted => continue,
+ Err(e) => return Err(e),
+ };
+
+ data.extend(&buffer[..read]);
+
+ let last_printed = match self.last_progress {
+ Some(prev) => prev,
+ None => {
+ let current_time = Instant::now();
+ self.last_progress = Some(current_time);
+ current_time
+ }
+ };
+ let elapsed = Instant::now().saturating_duration_since(last_printed);
+
+ downloaded_this_sec += read;
+ self.state.total_downloaded += read;
+
+ if elapsed >= Duration::from_secs(1) {
+ if self.state.bytes_per_second.len() == SAMPLES {
+ self.state.bytes_per_second.pop_back();
+ }
+
+ self.state.bytes_per_second.push_front(downloaded_this_sec);
+ downloaded_this_sec = 0;
+
+ self.progress.print_progress(&self.state);
+ self.last_progress = Some(Instant::now());
+ }
+ }
+
+ self.progress.print_finish(&self.state);
+
+ Ok(data)
+ }
+}
diff --git a/crates/typst-kit/src/fonts.rs b/crates/typst-kit/src/fonts.rs
new file mode 100644
index 00000000..07a41f4b
--- /dev/null
+++ b/crates/typst-kit/src/fonts.rs
@@ -0,0 +1,216 @@
+//! Default implementation for searching local and system installed fonts as
+//! well as loading embedded default fonts.
+//!
+//! # Embedded fonts
+//! The following fonts are available as embedded fonts via the `embed-fonts`
+//! feature flag:
+//! - For text: Linux Libertine, New Computer Modern
+//! - For math: New Computer Modern Math
+//! - For code: Deja Vu Sans Mono
+
+use std::path::PathBuf;
+use std::sync::OnceLock;
+use std::{fs, path::Path};
+
+use fontdb::{Database, Source};
+use typst::text::{Font, FontBook, FontInfo};
+use typst_timing::TimingScope;
+
+/// Holds details about the location of a font and lazily the font itself.
+#[derive(Debug)]
+pub struct FontSlot {
+ /// The path at which the font can be found on the system.
+ path: Option<PathBuf>,
+ /// The index of the font in its collection. Zero if the path does not point
+ /// to a collection.
+ index: u32,
+ /// The lazily loaded font.
+ font: OnceLock<Option<Font>>,
+}
+
+impl FontSlot {
+ /// Returns the path at which the font can be found on the system, or `None`
+ /// if the font was embedded.
+ pub fn path(&self) -> Option<&Path> {
+ self.path.as_deref()
+ }
+
+ /// Returns the index of the font in its collection. Zero if the path does
+ /// not point to a collection.
+ pub fn index(&self) -> u32 {
+ self.index
+ }
+
+ /// Get the font for this slot. This loads the font into memory on first
+ /// access.
+ pub fn get(&self) -> Option<Font> {
+ self.font
+ .get_or_init(|| {
+ let _scope = TimingScope::new("load font", None);
+ let data = fs::read(
+ self.path
+ .as_ref()
+ .expect("`path` is not `None` if `font` is uninitialized"),
+ )
+ .ok()?
+ .into();
+ Font::new(data, self.index)
+ })
+ .clone()
+ }
+}
+
+/// The result of a font search, created by calling [`FontSearcher::search`].
+#[derive(Debug)]
+pub struct Fonts {
+ /// Metadata about all discovered fonts.
+ pub book: FontBook,
+ /// Slots that the fonts are loaded into.
+ pub fonts: Vec<FontSlot>,
+}
+
+impl Fonts {
+ /// Creates a new font searcer with the default settings.
+ pub fn searcher() -> FontSearcher {
+ FontSearcher::new()
+ }
+}
+
+/// Searches for fonts.
+///
+/// Fonts are added in the following order (descending priority):
+/// 1. Font directories
+/// 2. System fonts (if included & enabled)
+/// 3. Embedded fonts (if enabled)
+#[derive(Debug)]
+pub struct FontSearcher {
+ db: Database,
+ include_system_fonts: bool,
+ #[cfg(feature = "embed-fonts")]
+ include_embedded_fonts: bool,
+ book: FontBook,
+ fonts: Vec<FontSlot>,
+}
+
+impl FontSearcher {
+ /// Create a new, empty system searcher. The searcher is created with the
+ /// default configuration, it will include embedded fonts and system fonts.
+ pub fn new() -> Self {
+ Self {
+ db: Database::new(),
+ include_system_fonts: true,
+ #[cfg(feature = "embed-fonts")]
+ include_embedded_fonts: true,
+ book: FontBook::new(),
+ fonts: vec![],
+ }
+ }
+
+ /// Whether to search for and load system fonts, defaults to `true`.
+ pub fn include_system_fonts(&mut self, value: bool) -> &mut Self {
+ self.include_system_fonts = value;
+ self
+ }
+
+ /// Whether to load embedded fonts, defaults to `true`.
+ #[cfg(feature = "embed-fonts")]
+ pub fn include_embedded_fonts(&mut self, value: bool) -> &mut Self {
+ self.include_embedded_fonts = value;
+ self
+ }
+
+ /// Start searching for and loading fonts. To additionally load fonts
+ /// from specific directories, use [`search_with`][Self::search_with].
+ ///
+ /// # Examples
+ /// ```no_run
+ /// # use typst_kit::fonts::FontSearcher;
+ /// let fonts = FontSearcher::new()
+ /// .include_system_fonts(true)
+ /// .search();
+ /// ```
+ pub fn search(&mut self) -> Fonts {
+ self.search_with::<_, &str>([])
+ }
+
+ /// Start searching for and loading fonts, with additional directories.
+ ///
+ /// # Examples
+ /// ```no_run
+ /// # use typst_kit::fonts::FontSearcher;
+ /// let fonts = FontSearcher::new()
+ /// .include_system_fonts(true)
+ /// .search_with(["./assets/fonts/"]);
+ /// ```
+ pub fn search_with<I, P>(&mut self, font_dirs: I) -> Fonts
+ where
+ I: IntoIterator<Item = P>,
+ P: AsRef<Path>,
+ {
+ // Font paths have highest priority.
+ for path in font_dirs {
+ self.db.load_fonts_dir(path);
+ }
+
+ if self.include_system_fonts {
+ // System fonts have second priority.
+ self.db.load_system_fonts();
+ }
+
+ for face in self.db.faces() {
+ let path = match &face.source {
+ Source::File(path) | Source::SharedFile(path, _) => path,
+ // We never add binary sources to the database, so there
+ // shouln't be any.
+ Source::Binary(_) => continue,
+ };
+
+ let info = self
+ .db
+ .with_face_data(face.id, FontInfo::new)
+ .expect("database must contain this font");
+
+ if let Some(info) = info {
+ self.book.push(info);
+ self.fonts.push(FontSlot {
+ path: Some(path.clone()),
+ index: face.index,
+ font: OnceLock::new(),
+ });
+ }
+ }
+
+ // Embedded fonts have lowest priority.
+ #[cfg(feature = "embed-fonts")]
+ if self.include_embedded_fonts {
+ self.add_embedded();
+ }
+
+ Fonts {
+ book: std::mem::take(&mut self.book),
+ fonts: std::mem::take(&mut self.fonts),
+ }
+ }
+
+ /// Add fonts that are embedded in the binary.
+ #[cfg(feature = "embed-fonts")]
+ fn add_embedded(&mut self) {
+ for data in typst_assets::fonts() {
+ let buffer = typst::foundations::Bytes::from_static(data);
+ for (i, font) in Font::iter(buffer).enumerate() {
+ self.book.push(font.info().clone());
+ self.fonts.push(FontSlot {
+ path: None,
+ index: i as u32,
+ font: OnceLock::from(Some(font)),
+ });
+ }
+ }
+ }
+}
+
+impl Default for FontSearcher {
+ fn default() -> Self {
+ Self::new()
+ }
+}
diff --git a/crates/typst-kit/src/lib.rs b/crates/typst-kit/src/lib.rs
new file mode 100644
index 00000000..4301727f
--- /dev/null
+++ b/crates/typst-kit/src/lib.rs
@@ -0,0 +1,27 @@
+//! Typst-kit contains various default implementations of functionality used in
+//! typst-cli. It is intended as a single source of truth for things like font
+//! searching, package downloads and more. Each component of typst-kit is
+//! optional, but enabled by default.
+//!
+//! # Components
+//! - [fonts] contains a default implementation for searching local and system
+//! installed fonts. It is enabled by the `fonts` feature flag, additionally
+//! the `embed-fonts` feature can be used to embed the Typst default fonts.
+//! - For text: Linux Libertine, New Computer Modern
+//! - For math: New Computer Modern Math
+//! - For code: Deja Vu Sans Mono
+//! - [download] contains functionality for making simple web requests with
+//! status reporting, useful for downloading packages from package registires.
+//! It is enabled by the `downloads` feature flag, additionally the
+//! `vendor-openssl` can be used on operating systems other than macOS and
+//! Windows to vendor OpenSSL when building.
+//! - [package] contains package storage and downloading functionality based on
+//! [download]. It is enabled by the `packages` feature flag and implies the
+//! `downloads` feature flag.
+
+#[cfg(feature = "downloads")]
+pub mod download;
+#[cfg(feature = "fonts")]
+pub mod fonts;
+#[cfg(feature = "packages")]
+pub mod package;
diff --git a/crates/typst-kit/src/package.rs b/crates/typst-kit/src/package.rs
new file mode 100644
index 00000000..ad69df01
--- /dev/null
+++ b/crates/typst-kit/src/package.rs
@@ -0,0 +1,179 @@
+//! Download and unpack packages and package indices.
+
+use std::fs;
+use std::path::{Path, PathBuf};
+
+use ecow::eco_format;
+use once_cell::sync::OnceCell;
+use typst::diag::{bail, PackageError, PackageResult, StrResult};
+use typst::syntax::package::{
+ PackageInfo, PackageSpec, PackageVersion, VersionlessPackageSpec,
+};
+
+use crate::download::{Downloader, Progress};
+
+/// The default Typst registry.
+pub const DEFAULT_REGISTRY: &str = "https://packages.typst.org";
+
+/// The default packages sub directory within the package and package cache paths.
+pub const DEFAULT_PACKAGES_SUBDIR: &str = "typst/packages";
+
+/// Holds information about where packages should be stored and downloads them
+/// on demand, if possible.
+#[derive(Debug)]
+pub struct PackageStorage {
+ /// The path at which non-local packages should be stored when downloaded.
+ package_cache_path: Option<PathBuf>,
+ /// The path at which local packages are stored.
+ package_path: Option<PathBuf>,
+ /// The downloader used for fetching the index and packages.
+ downloader: Downloader,
+ /// The cached index of the preview namespace.
+ index: OnceCell<Vec<PackageInfo>>,
+}
+
+impl PackageStorage {
+ /// Creates a new package storage for the given package paths. Falls back to
+ /// the recommended XDG directories if they are `None`.
+ pub fn new(
+ package_cache_path: Option<PathBuf>,
+ package_path: Option<PathBuf>,
+ downloader: Downloader,
+ ) -> Self {
+ Self {
+ package_cache_path: package_cache_path.or_else(|| {
+ dirs::cache_dir().map(|cache_dir| cache_dir.join(DEFAULT_PACKAGES_SUBDIR))
+ }),
+ package_path: package_path.or_else(|| {
+ dirs::data_dir().map(|data_dir| data_dir.join(DEFAULT_PACKAGES_SUBDIR))
+ }),
+ downloader,
+ index: OnceCell::new(),
+ }
+ }
+
+ /// Returns a the path at which non-local packages should be stored when
+ /// downloaded.
+ pub fn package_cache_path(&self) -> Option<&Path> {
+ self.package_cache_path.as_deref()
+ }
+
+ /// Returns a the path at which local packages are stored.
+ pub fn package_path(&self) -> Option<&Path> {
+ self.package_path.as_deref()
+ }
+
+ /// Make a package available in the on-disk.
+ pub fn prepare_package(
+ &self,
+ spec: &PackageSpec,
+ progress: &mut dyn Progress,
+ ) -> PackageResult<PathBuf> {
+ let subdir = format!("{}/{}/{}", spec.namespace, spec.name, spec.version);
+
+ if let Some(packages_dir) = &self.package_path {
+ let dir = packages_dir.join(&subdir);
+ if dir.exists() {
+ return Ok(dir);
+ }
+ }
+
+ if let Some(cache_dir) = &self.package_cache_path {
+ let dir = cache_dir.join(&subdir);
+ if dir.exists() {
+ return Ok(dir);
+ }
+
+ // Download from network if it doesn't exist yet.
+ self.download_package(spec, &dir, progress)?;
+ if dir.exists() {
+ return Ok(dir);
+ }
+ }
+
+ Err(PackageError::NotFound(spec.clone()))
+ }
+
+ /// Try to determine the latest version of a package.
+ pub fn determine_latest_version(
+ &self,
+ spec: &VersionlessPackageSpec,
+ ) -> StrResult<PackageVersion> {
+ if spec.namespace == "preview" {
+ // For `@preview`, download the package index and find the latest
+ // version.
+ self.download_index()?
+ .iter()
+ .filter(|package| package.name == spec.name)
+ .map(|package| package.version)
+ .max()
+ .ok_or_else(|| eco_format!("failed to find package {spec}"))
+ } else {
+ // For other namespaces, search locally. We only search in the data
+ // directory and not the cache directory, because the latter is not
+ // intended for storage of local packages.
+ let subdir = format!("{}/{}", spec.namespace, spec.name);
+ self.package_path
+ .iter()
+ .flat_map(|dir| std::fs::read_dir(dir.join(&subdir)).ok())
+ .flatten()
+ .filter_map(|entry| entry.ok())
+ .map(|entry| entry.path())
+ .filter_map(|path| path.file_name()?.to_string_lossy().parse().ok())
+ .max()
+ .ok_or_else(|| eco_format!("please specify the desired version"))
+ }
+ }
+
+ /// Download the package index. The result of this is cached for efficiency.
+ pub fn download_index(&self) -> StrResult<&Vec<PackageInfo>> {
+ self.index.get_or_try_init(|| {
+ let url = format!("{DEFAULT_REGISTRY}/preview/index.json");
+ match self.downloader.download(&url) {
+ Ok(response) => response
+ .into_json()
+ .map_err(|err| eco_format!("failed to parse package index: {err}")),
+ Err(ureq::Error::Status(404, _)) => {
+ bail!("failed to fetch package index (not found)")
+ }
+ Err(err) => bail!("failed to fetch package index ({err})"),
+ }
+ })
+ }
+
+ /// Download a package over the network.
+ ///
+ /// # Panics
+ /// Panics if the package spec namespace isn't `preview`.
+ pub fn download_package(
+ &self,
+ spec: &PackageSpec,
+ package_dir: &Path,
+ progress: &mut dyn Progress,
+ ) -> PackageResult<()> {
+ assert_eq!(spec.namespace, "preview");
+
+ let url =
+ format!("{DEFAULT_REGISTRY}/preview/{}-{}.tar.gz", spec.name, spec.version);
+
+ let data = match self.downloader.download_with_progress(&url, progress) {
+ Ok(data) => data,
+ Err(ureq::Error::Status(404, _)) => {
+ if let Ok(version) = self.determine_latest_version(&spec.versionless()) {
+ return Err(PackageError::VersionNotFound(spec.clone(), version));
+ } else {
+ return Err(PackageError::NotFound(spec.clone()));
+ }
+ }
+ Err(err) => {
+ return Err(PackageError::NetworkFailed(Some(eco_format!("{err}"))))
+ }
+ };
+
+ let decompressed = flate2::read::GzDecoder::new(data.as_slice());
+ tar::Archive::new(decompressed).unpack(package_dir).map_err(|err| {
+ fs::remove_dir_all(package_dir).ok();
+ PackageError::MalformedArchive(Some(eco_format!("{err}")))
+ })
+ }
+}