Revamp data loading and deprecate `decode` functions (#5671)

6 files changed, 169 insertions, 18 deletions

diff --git a/crates/typst-library/src/foundations/array.rs b/crates/typst-library/src/foundations/array.rs
index 4667ee76..e79a4e93 100644
--- a/crates/typst-library/src/foundations/array.rs
+++ b/crates/typst-library/src/foundations/array.rs
@@ -1124,6 +1124,53 @@ impl<T: FromValue, const N: usize> FromValue for SmallVec<[T; N]> {
     }
 }
 
+/// One element, or multiple provided as an array.
+#[derive(Debug, Clone, PartialEq, Hash)]
+pub struct OneOrMultiple<T>(pub Vec<T>);
+
+impl<T: Reflect> Reflect for OneOrMultiple<T> {
+    fn input() -> CastInfo {
+        T::input() + Array::input()
+    }
+
+    fn output() -> CastInfo {
+        T::output() + Array::output()
+    }
+
+    fn castable(value: &Value) -> bool {
+        Array::castable(value) || T::castable(value)
+    }
+}
+
+impl<T: IntoValue + Clone> IntoValue for OneOrMultiple<T> {
+    fn into_value(self) -> Value {
+        self.0.into_value()
+    }
+}
+
+impl<T: FromValue> FromValue for OneOrMultiple<T> {
+    fn from_value(value: Value) -> HintedStrResult<Self> {
+        if T::castable(&value) {
+            return Ok(Self(vec![T::from_value(value)?]));
+        }
+        if Array::castable(&value) {
+            return Ok(Self(
+                Array::from_value(value)?
+                    .into_iter()
+                    .map(|value| T::from_value(value))
+                    .collect::<HintedStrResult<_>>()?,
+            ));
+        }
+        Err(Self::error(&value))
+    }
+}
+
+impl<T> Default for OneOrMultiple<T> {
+    fn default() -> Self {
+        Self(vec![])
+    }
+}
+
 /// The error message when the array is empty.
 #[cold]
 fn array_is_empty() -> EcoString {
diff --git a/crates/typst-library/src/foundations/bytes.rs b/crates/typst-library/src/foundations/bytes.rs
index 20034d07..d633c99a 100644
--- a/crates/typst-library/src/foundations/bytes.rs
+++ b/crates/typst-library/src/foundations/bytes.rs
@@ -2,6 +2,7 @@ use std::any::Any;
 use std::fmt::{self, Debug, Formatter};
 use std::hash::{Hash, Hasher};
 use std::ops::{Add, AddAssign, Deref};
+use std::str::Utf8Error;
 use std::sync::Arc;
 
 use ecow::{eco_format, EcoString};
@@ -80,16 +81,37 @@ impl Bytes {
         self.as_slice().is_empty()
     }
 
-    /// Return a view into the buffer.
+    /// Return a view into the bytes.
     pub fn as_slice(&self) -> &[u8] {
         self
     }
 
-    /// Return a copy of the buffer as a vector.
+    /// Try to view the bytes as an UTF-8 string.
+    ///
+    /// If these bytes were created via `Bytes::from_string`, UTF-8 validation
+    /// is skipped.
+    pub fn as_str(&self) -> Result<&str, Utf8Error> {
+        self.inner().as_str()
+    }
+
+    /// Return a copy of the bytes as a vector.
     pub fn to_vec(&self) -> Vec<u8> {
         self.as_slice().to_vec()
     }
 
+    /// Try to turn the bytes into a `Str`.
+    ///
+    /// - If these bytes were created via `Bytes::from_string::<Str>`, the
+    ///   string is cloned directly.
+    /// - If these bytes were created via `Bytes::from_string`, but from a
+    ///   different type of string, UTF-8 validation is still skipped.
+    pub fn to_str(&self) -> Result<Str, Utf8Error> {
+        match self.inner().as_any().downcast_ref::<Str>() {
+            Some(string) => Ok(string.clone()),
+            None => self.as_str().map(Into::into),
+        }
+    }
+
     /// Resolve an index or throw an out of bounds error.
     fn locate(&self, index: i64) -> StrResult<usize> {
         self.locate_opt(index).ok_or_else(|| out_of_bounds(index, self.len()))
@@ -104,6 +126,11 @@ impl Bytes {
             if index >= 0 { Some(index) } else { (len as i64).checked_add(index) };
         wrapped.and_then(|v| usize::try_from(v).ok()).filter(|&v| v <= len)
     }
+
+    /// Access the inner `dyn Bytelike`.
+    fn inner(&self) -> &dyn Bytelike {
+        &**self.0
+    }
 }
 
 #[scope]
@@ -203,7 +230,7 @@ impl Deref for Bytes {
     type Target = [u8];
 
     fn deref(&self) -> &Self::Target {
-        self.0.as_bytes()
+        self.inner().as_bytes()
     }
 }
 
@@ -262,6 +289,8 @@ impl Serialize for Bytes {
 /// Any type that can back a byte buffer.
 trait Bytelike: Send + Sync {
     fn as_bytes(&self) -> &[u8];
+    fn as_str(&self) -> Result<&str, Utf8Error>;
+    fn as_any(&self) -> &dyn Any;
     fn as_any_mut(&mut self) -> &mut dyn Any;
 }
 
@@ -273,6 +302,14 @@ where
         self.as_ref()
     }
 
+    fn as_str(&self) -> Result<&str, Utf8Error> {
+        std::str::from_utf8(self.as_ref())
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
     fn as_any_mut(&mut self) -> &mut dyn Any {
         self
     }
@@ -295,6 +332,14 @@ where
         self.0.as_ref().as_bytes()
     }
 
+    fn as_str(&self) -> Result<&str, Utf8Error> {
+        Ok(self.0.as_ref())
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
     fn as_any_mut(&mut self) -> &mut dyn Any {
         self
     }
diff --git a/crates/typst-library/src/foundations/cast.rs b/crates/typst-library/src/foundations/cast.rs
index 84f38f36..38f409c6 100644
--- a/crates/typst-library/src/foundations/cast.rs
+++ b/crates/typst-library/src/foundations/cast.rs
@@ -13,7 +13,9 @@ use typst_syntax::{Span, Spanned};
 use unicode_math_class::MathClass;
 
 use crate::diag::{At, HintedStrResult, HintedString, SourceResult, StrResult};
-use crate::foundations::{array, repr, NativeElement, Packed, Repr, Str, Type, Value};
+use crate::foundations::{
+    array, repr, Fold, NativeElement, Packed, Repr, Str, Type, Value,
+};
 
 /// Determine details of a type.
 ///
@@ -497,3 +499,58 @@ cast! {
     /// An operator that can be both unary or binary like `+`.
     "vary" => MathClass::Vary,
 }
+
+/// A type that contains a user-visible source portion and something that is
+/// derived from it, but not user-visible.
+///
+/// An example usage would be `source` being a `DataSource` and `derived` a
+/// TextMate theme parsed from it. With `Derived`, we can store both parts in
+/// the `RawElem::theme` field and get automatic nice `Reflect` and `IntoValue`
+/// impls.
+#[derive(Debug, Default, Copy, Clone, Eq, PartialEq, Hash)]
+pub struct Derived<S, D> {
+    /// The source portion.
+    pub source: S,
+    /// The derived portion.
+    pub derived: D,
+}
+
+impl<S, D> Derived<S, D> {
+    /// Create a new instance from the `source` and the `derived` data.
+    pub fn new(source: S, derived: D) -> Self {
+        Self { source, derived }
+    }
+}
+
+impl<S: Reflect, D> Reflect for Derived<S, D> {
+    fn input() -> CastInfo {
+        S::input()
+    }
+
+    fn output() -> CastInfo {
+        S::output()
+    }
+
+    fn castable(value: &Value) -> bool {
+        S::castable(value)
+    }
+
+    fn error(found: &Value) -> HintedString {
+        S::error(found)
+    }
+}
+
+impl<S: IntoValue, D> IntoValue for Derived<S, D> {
+    fn into_value(self) -> Value {
+        self.source.into_value()
+    }
+}
+
+impl<S: Fold, D: Fold> Fold for Derived<S, D> {
+    fn fold(self, outer: Self) -> Self {
+        Self {
+            source: self.source.fold(outer.source),
+            derived: self.derived.fold(outer.derived),
+        }
+    }
+}
diff --git a/crates/typst-library/src/foundations/plugin.rs b/crates/typst-library/src/foundations/plugin.rs
index a7c341d8..adf23a47 100644
--- a/crates/typst-library/src/foundations/plugin.rs
+++ b/crates/typst-library/src/foundations/plugin.rs
@@ -9,7 +9,7 @@ use wasmi::{AsContext, AsContextMut};
 use crate::diag::{bail, At, SourceResult, StrResult};
 use crate::engine::Engine;
 use crate::foundations::{func, repr, scope, ty, Bytes};
-use crate::World;
+use crate::loading::{DataSource, Load};
 
 /// A WebAssembly plugin.
 ///
@@ -154,15 +154,13 @@ impl Plugin {
     pub fn construct(
         /// The engine.
         engine: &mut Engine,
-        /// Path to a WebAssembly file.
+        /// A path to a WebAssembly file or raw WebAssembly bytes.
         ///
-        /// For more details, see the [Paths section]($syntax/#paths).
-        path: Spanned<EcoString>,
+        /// For more details about paths, see the [Paths section]($syntax/#paths).
+        source: Spanned<DataSource>,
     ) -> SourceResult<Plugin> {
-        let Spanned { v: path, span } = path;
-        let id = span.resolve_path(&path).at(span)?;
-        let data = engine.world.file(id).at(span)?;
-        Plugin::new(data).at(span)
+        let data = source.load(engine.world)?;
+        Plugin::new(data).at(source.span)
     }
 }
 
diff --git a/crates/typst-library/src/foundations/str.rs b/crates/typst-library/src/foundations/str.rs
index 4025d1ab..2e90b307 100644
--- a/crates/typst-library/src/foundations/str.rs
+++ b/crates/typst-library/src/foundations/str.rs
@@ -784,11 +784,7 @@ cast! {
     v: f64 => Self::Str(repr::display_float(v).into()),
     v: Decimal => Self::Str(format_str!("{}", v)),
     v: Version => Self::Str(format_str!("{}", v)),
-    v: Bytes => Self::Str(
-        std::str::from_utf8(&v)
-            .map_err(|_| "bytes are not valid utf-8")?
-            .into()
-    ),
+    v: Bytes => Self::Str(v.to_str().map_err(|_| "bytes are not valid utf-8")?),
     v: Label => Self::Str(v.resolve().as_str().into()),
     v: Type => Self::Str(v.long_name().into()),
     v: Str => Self::Str(v),
diff --git a/crates/typst-library/src/foundations/styles.rs b/crates/typst-library/src/foundations/styles.rs
index 7354719e..37094dcd 100644
--- a/crates/typst-library/src/foundations/styles.rs
+++ b/crates/typst-library/src/foundations/styles.rs
@@ -12,7 +12,8 @@ use typst_utils::LazyHash;
 use crate::diag::{SourceResult, Trace, Tracepoint};
 use crate::engine::Engine;
 use crate::foundations::{
-    cast, ty, Content, Context, Element, Func, NativeElement, Repr, Selector,
+    cast, ty, Content, Context, Element, Func, NativeElement, OneOrMultiple, Repr,
+    Selector,
 };
 use crate::text::{FontFamily, FontList, TextElem};
 
@@ -939,6 +940,13 @@ impl<T, const N: usize> Fold for SmallVec<[T; N]> {
     }
 }
 
+impl<T> Fold for OneOrMultiple<T> {
+    fn fold(self, mut outer: Self) -> Self {
+        outer.0.extend(self.0);
+        outer
+    }
+}
+
 /// A variant of fold for foldable optional (`Option<T>`) values where an inner
 /// `None` value isn't respected (contrary to `Option`'s usual `Fold`
 /// implementation, with which folding with an inner `None` always returns