Semantic paragraphs (#5746)

8 files changed, 151 insertions, 86 deletions

diff --git a/crates/typst-layout/src/inline/box.rs b/crates/typst-layout/src/inline/box.rs
index 6dfbc969..e21928d3 100644
--- a/crates/typst-layout/src/inline/box.rs
+++ b/crates/typst-layout/src/inline/box.rs
@@ -11,7 +11,7 @@ use typst_utils::Numeric;
 use crate::flow::unbreakable_pod;
 use crate::shapes::{clip_rect, fill_and_stroke};
 
-/// Lay out a box as part of a paragraph.
+/// Lay out a box as part of inline layout.
 #[typst_macros::time(name = "box", span = elem.span())]
 pub fn layout_box(
     elem: &Packed<BoxElem>,
diff --git a/crates/typst-layout/src/inline/collect.rs b/crates/typst-layout/src/inline/collect.rs
index 6023f5c6..cbc490ba 100644
--- a/crates/typst-layout/src/inline/collect.rs
+++ b/crates/typst-layout/src/inline/collect.rs
@@ -1,10 +1,11 @@
-use typst_library::diag::bail;
+use typst_library::diag::warning;
 use typst_library::foundations::{Packed, Resolve};
 use typst_library::introspection::{SplitLocator, Tag, TagElem};
 use typst_library::layout::{
     Abs, AlignElem, BoxElem, Dir, Fr, Frame, HElem, InlineElem, InlineItem, Sizing,
     Spacing,
 };
+use typst_library::routines::Pair;
 use typst_library::text::{
     is_default_ignorable, LinebreakElem, SmartQuoteElem, SmartQuoter, SmartQuotes,
     SpaceElem, TextElem,
@@ -16,7 +17,7 @@ use super::*;
 use crate::modifiers::{layout_and_modify, FrameModifiers, FrameModify};
 
 // The characters by which spacing, inline content and pins are replaced in the
-// paragraph's full text.
+// full text.
 const SPACING_REPLACE: &str = " "; // Space
 const OBJ_REPLACE: &str = "\u{FFFC}"; // Object Replacement Character
 
@@ -27,7 +28,7 @@ const POP_EMBEDDING: &str = "\u{202C}";
 const LTR_ISOLATE: &str = "\u{2066}";
 const POP_ISOLATE: &str = "\u{2069}";
 
-/// A prepared item in a paragraph layout.
+/// A prepared item in a inline layout.
 #[derive(Debug)]
 pub enum Item<'a> {
     /// A shaped text run with consistent style and direction.
@@ -113,38 +114,44 @@ impl Segment<'_> {
     }
 }
 
-/// Collects all text of the paragraph into one string and a collection of
-/// segments that correspond to pieces of that string. This also performs
-/// string-level preprocessing like case transformations.
+/// Collects all text into one string and a collection of segments that
+/// correspond to pieces of that string. This also performs string-level
+/// preprocessing like case transformations.
 #[typst_macros::time]
 pub fn collect<'a>(
-    children: &'a StyleVec,
+    children: &[Pair<'a>],
     engine: &mut Engine<'_>,
     locator: &mut SplitLocator<'a>,
-    styles: &'a StyleChain<'a>,
+    styles: StyleChain<'a>,
     region: Size,
     consecutive: bool,
+    paragraph: bool,
 ) -> SourceResult<(String, Vec<Segment<'a>>, SpanMapper)> {
     let mut collector = Collector::new(2 + children.len());
     let mut quoter = SmartQuoter::new();
 
-    let outer_dir = TextElem::dir_in(*styles);
-    let first_line_indent = ParElem::first_line_indent_in(*styles);
-    if !first_line_indent.is_zero()
-        && consecutive
-        && AlignElem::alignment_in(*styles).resolve(*styles).x == outer_dir.start().into()
-    {
-        collector.push_item(Item::Absolute(first_line_indent.resolve(*styles), false));
-        collector.spans.push(1, Span::detached());
+    let outer_dir = TextElem::dir_in(styles);
+
+    if paragraph && consecutive {
+        let first_line_indent = ParElem::first_line_indent_in(styles);
+        if !first_line_indent.is_zero()
+            && AlignElem::alignment_in(styles).resolve(styles).x
+                == outer_dir.start().into()
+        {
+            collector.push_item(Item::Absolute(first_line_indent.resolve(styles), false));
+            collector.spans.push(1, Span::detached());
+        }
     }
 
-    let hang = ParElem::hanging_indent_in(*styles);
-    if !hang.is_zero() {
-        collector.push_item(Item::Absolute(-hang, false));
-        collector.spans.push(1, Span::detached());
+    if paragraph {
+        let hang = ParElem::hanging_indent_in(styles);
+        if !hang.is_zero() {
+            collector.push_item(Item::Absolute(-hang, false));
+            collector.spans.push(1, Span::detached());
+        }
     }
 
-    for (child, styles) in children.iter(styles) {
+    for &(child, styles) in children {
         let prev_len = collector.full.len();
 
         if child.is::<SpaceElem>() {
@@ -234,7 +241,13 @@ pub fn collect<'a>(
         } else if let Some(elem) = child.to_packed::<TagElem>() {
             collector.push_item(Item::Tag(&elem.tag));
         } else {
-            bail!(child.span(), "unexpected paragraph child");
+            // Non-paragraph inline layout should never trigger this since it
+            // only won't be triggered if we see any non-inline content.
+            engine.sink.warn(warning!(
+                child.span(),
+                "{} may not occur inside of a paragraph and was ignored",
+                child.func().name()
+            ));
         };
 
         let len = collector.full.len() - prev_len;
diff --git a/crates/typst-layout/src/inline/finalize.rs b/crates/typst-layout/src/inline/finalize.rs
index 57044f0e..7ad287c4 100644
--- a/crates/typst-layout/src/inline/finalize.rs
+++ b/crates/typst-layout/src/inline/finalize.rs
@@ -14,7 +14,7 @@ pub fn finalize(
     expand: bool,
     locator: &mut SplitLocator<'_>,
 ) -> SourceResult<Fragment> {
-    // Determine the paragraph's width: Full width of the region if we should
+    // Determine the resulting width: Full width of the region if we should
     // expand or there's fractional spacing, fit-to-width otherwise.
     let width = if !region.x.is_finite()
         || (!expand && lines.iter().all(|line| line.fr().is_zero()))
diff --git a/crates/typst-layout/src/inline/line.rs b/crates/typst-layout/src/inline/line.rs
index fba4bef8..9f697380 100644
--- a/crates/typst-layout/src/inline/line.rs
+++ b/crates/typst-layout/src/inline/line.rs
@@ -18,12 +18,12 @@ const EN_DASH: char = '–';
 const EM_DASH: char = '—';
 const LINE_SEPARATOR: char = '\u{2028}'; // We use LS to distinguish justified breaks.
 
-/// A layouted line, consisting of a sequence of layouted paragraph items that
-/// are mostly borrowed from the preparation phase. This type enables you to
-/// measure the size of a line in a range before committing to building the
-/// line's frame.
+/// A layouted line, consisting of a sequence of layouted inline items that are
+/// mostly borrowed from the preparation phase. This type enables you to measure
+/// the size of a line in a range before committing to building the line's
+/// frame.
 ///
-/// At most two paragraph items must be created individually for this line: The
+/// At most two inline items must be created individually for this line: The
 /// first and last one since they may be broken apart by the start or end of the
 /// line, respectively. But even those can partially reuse previous results when
 /// the break index is safe-to-break per rustybuzz.
@@ -430,7 +430,7 @@ pub fn commit(
     let mut offset = Abs::zero();
 
     // We always build the line from left to right. In an LTR paragraph, we must
-    // thus add the hanging indent to the offset. When the paragraph is RTL, the
+    // thus add the hanging indent to the offset. In an RTL paragraph, the
     // hanging indent arises naturally due to the line width.
     if p.dir == Dir::LTR {
         offset += p.hang;
@@ -631,7 +631,7 @@ fn overhang(c: char) -> f64 {
     }
 }
 
-/// A collection of owned or borrowed paragraph items.
+/// A collection of owned or borrowed inline items.
 pub struct Items<'a>(Vec<ItemEntry<'a>>);
 
 impl<'a> Items<'a> {
diff --git a/crates/typst-layout/src/inline/linebreak.rs b/crates/typst-layout/src/inline/linebreak.rs
index 7b66fcdb..87113c68 100644
--- a/crates/typst-layout/src/inline/linebreak.rs
+++ b/crates/typst-layout/src/inline/linebreak.rs
@@ -17,7 +17,7 @@ use unicode_segmentation::UnicodeSegmentation;
 
 use super::*;
 
-/// The cost of a line or paragraph layout.
+/// The cost of a line or inline layout.
 type Cost = f64;
 
 // Cost parameters.
@@ -104,7 +104,7 @@ impl Breakpoint {
     }
 }
 
-/// Breaks the paragraph into lines.
+/// Breaks the text into lines.
 pub fn linebreak<'a>(
     engine: &Engine,
     p: &'a Preparation<'a>,
@@ -181,13 +181,12 @@ fn linebreak_simple<'a>(
 /// lines with hyphens even more.
 ///
 /// To find the layout with the minimal total cost the algorithm uses dynamic
-/// programming: For each possible breakpoint it determines the optimal
-/// paragraph layout _up to that point_. It walks over all possible start points
-/// for a line ending at that point and finds the one for which the cost of the
-/// line plus the cost of the optimal paragraph up to the start point (already
-/// computed and stored in dynamic programming table) is minimal. The final
-/// result is simply the layout determined for the last breakpoint at the end of
-/// text.
+/// programming: For each possible breakpoint, it determines the optimal layout
+/// _up to that point_. It walks over all possible start points for a line
+/// ending at that point and finds the one for which the cost of the line plus
+/// the cost of the optimal layout up to the start point (already computed and
+/// stored in dynamic programming table) is minimal. The final result is simply
+/// the layout determined for the last breakpoint at the end of text.
 #[typst_macros::time]
 fn linebreak_optimized<'a>(
     engine: &Engine,
@@ -215,7 +214,7 @@ fn linebreak_optimized_bounded<'a>(
     metrics: &CostMetrics,
     upper_bound: Cost,
 ) -> Vec<Line<'a>> {
-    /// An entry in the dynamic programming table for paragraph optimization.
+    /// An entry in the dynamic programming table for inline layout optimization.
     struct Entry<'a> {
         pred: usize,
         total: Cost,
@@ -321,7 +320,7 @@ fn linebreak_optimized_bounded<'a>(
     // This should only happen if our bound was faulty. Which shouldn't happen!
     if table[idx].end != p.text.len() {
         #[cfg(debug_assertions)]
-        panic!("bounded paragraph layout is incomplete");
+        panic!("bounded inline layout is incomplete");
 
         #[cfg(not(debug_assertions))]
         return linebreak_optimized_bounded(engine, p, width, metrics, Cost::INFINITY);
@@ -342,7 +341,7 @@ fn linebreak_optimized_bounded<'a>(
 /// (which is costly) to determine costs, it determines approximate costs using
 /// cumulative arrays.
 ///
-/// This results in a likely good paragraph layouts, for which we then compute
+/// This results in a likely good inline layouts, for which we then compute
 /// the exact cost. This cost is an upper bound for proper optimized
 /// linebreaking. We can use it to heavily prune the search space.
 #[typst_macros::time]
@@ -355,7 +354,7 @@ fn linebreak_optimized_approximate(
     // Determine the cumulative estimation metrics.
     let estimates = Estimates::compute(p);
 
-    /// An entry in the dynamic programming table for paragraph optimization.
+    /// An entry in the dynamic programming table for inline layout optimization.
     struct Entry {
         pred: usize,
         total: Cost,
@@ -862,7 +861,7 @@ struct CostMetrics {
 }
 
 impl CostMetrics {
-    /// Compute shared metrics for paragraph optimization.
+    /// Compute shared metrics for inline layout optimization.
     fn compute(p: &Preparation) -> Self {
         Self {
             // When justifying, we may stretch spaces below their natural width.
diff --git a/crates/typst-layout/src/inline/mod.rs b/crates/typst-layout/src/inline/mod.rs
index bedc54d6..83ca82bf 100644
--- a/crates/typst-layout/src/inline/mod.rs
+++ b/crates/typst-layout/src/inline/mod.rs
@@ -13,11 +13,11 @@ pub use self::box_::layout_box;
 use comemo::{Track, Tracked, TrackedMut};
 use typst_library::diag::SourceResult;
 use typst_library::engine::{Engine, Route, Sink, Traced};
-use typst_library::foundations::{StyleChain, StyleVec};
-use typst_library::introspection::{Introspector, Locator, LocatorLink};
+use typst_library::foundations::{Packed, StyleChain};
+use typst_library::introspection::{Introspector, Locator, LocatorLink, SplitLocator};
 use typst_library::layout::{Fragment, Size};
 use typst_library::model::ParElem;
-use typst_library::routines::Routines;
+use typst_library::routines::{Arenas, Pair, RealizationKind, Routines};
 use typst_library::World;
 
 use self::collect::{collect, Item, Segment, SpanMapper};
@@ -34,18 +34,18 @@ use self::shaping::{
 /// Range of a substring of text.
 type Range = std::ops::Range<usize>;
 
-/// Layouts content inline.
-pub fn layout_inline(
+/// Layouts the paragraph.
+pub fn layout_par(
+    elem: &Packed<ParElem>,
     engine: &mut Engine,
-    children: &StyleVec,
     locator: Locator,
     styles: StyleChain,
-    consecutive: bool,
     region: Size,
     expand: bool,
+    consecutive: bool,
 ) -> SourceResult<Fragment> {
-    layout_inline_impl(
-        children,
+    layout_par_impl(
+        elem,
         engine.routines,
         engine.world,
         engine.introspector,
@@ -54,17 +54,17 @@ pub fn layout_inline(
         engine.route.track(),
         locator.track(),
         styles,
-        consecutive,
         region,
         expand,
+        consecutive,
     )
 }
 
-/// The internal, memoized implementation of `layout_inline`.
+/// The internal, memoized implementation of `layout_par`.
 #[comemo::memoize]
 #[allow(clippy::too_many_arguments)]
-fn layout_inline_impl(
-    children: &StyleVec,
+fn layout_par_impl(
+    elem: &Packed<ParElem>,
     routines: &Routines,
     world: Tracked<dyn World + '_>,
     introspector: Tracked<Introspector>,
@@ -73,12 +73,12 @@ fn layout_inline_impl(
     route: Tracked<Route>,
     locator: Tracked<Locator>,
     styles: StyleChain,
-    consecutive: bool,
     region: Size,
     expand: bool,
+    consecutive: bool,
 ) -> SourceResult<Fragment> {
     let link = LocatorLink::new(locator);
-    let locator = Locator::link(&link);
+    let mut locator = Locator::link(&link).split();
     let mut engine = Engine {
         routines,
         world,
@@ -88,18 +88,51 @@ fn layout_inline_impl(
         route: Route::extend(route),
     };
 
-    let mut locator = locator.split();
+    let arenas = Arenas::default();
+    let children = (engine.routines.realize)(
+        RealizationKind::LayoutPar,
+        &mut engine,
+        &mut locator,
+        &arenas,
+        &elem.body,
+        styles,
+    )?;
+
+    layout_inline(
+        &mut engine,
+        &children,
+        &mut locator,
+        styles,
+        region,
+        expand,
+        true,
+        consecutive,
+    )
+}
 
+/// Lays out realized content with inline layout.
+#[allow(clippy::too_many_arguments)]
+pub fn layout_inline<'a>(
+    engine: &mut Engine,
+    children: &[Pair<'a>],
+    locator: &mut SplitLocator<'a>,
+    styles: StyleChain<'a>,
+    region: Size,
+    expand: bool,
+    paragraph: bool,
+    consecutive: bool,
+) -> SourceResult<Fragment> {
     // Collect all text into one string for BiDi analysis.
     let (text, segments, spans) =
-        collect(children, &mut engine, &mut locator, &styles, region, consecutive)?;
+        collect(children, engine, locator, styles, region, consecutive, paragraph)?;
 
-    // Perform BiDi analysis and then prepares paragraph layout.
-    let p = prepare(&mut engine, children, &text, segments, spans, styles)?;
+    // Perform BiDi analysis and performs some preparation steps before we
+    // proceed to line breaking.
+    let p = prepare(engine, children, &text, segments, spans, styles, paragraph)?;
 
-    // Break the paragraph into lines.
-    let lines = linebreak(&engine, &p, region.x - p.hang);
+    // Break the text into lines.
+    let lines = linebreak(engine, &p, region.x - p.hang);
 
     // Turn the selected lines into frames.
-    finalize(&mut engine, &p, &lines, styles, region, expand, &mut locator)
+    finalize(engine, &p, &lines, styles, region, expand, locator)
 }
diff --git a/crates/typst-layout/src/inline/prepare.rs b/crates/typst-layout/src/inline/prepare.rs
index 2dd79aec..e26c9b14 100644
--- a/crates/typst-layout/src/inline/prepare.rs
+++ b/crates/typst-layout/src/inline/prepare.rs
@@ -1,23 +1,26 @@
 use typst_library::foundations::{Resolve, Smart};
 use typst_library::layout::{Abs, AlignElem, Dir, Em, FixedAlignment};
 use typst_library::model::Linebreaks;
+use typst_library::routines::Pair;
 use typst_library::text::{Costs, Lang, TextElem};
+use typst_utils::SliceExt;
 use unicode_bidi::{BidiInfo, Level as BidiLevel};
 
 use super::*;
 
-/// A paragraph representation in which children are already layouted and text
-/// is already preshaped.
+/// A representation in which children are already layouted and text is already
+/// preshaped.
 ///
 /// In many cases, we can directly reuse these results when constructing a line.
 /// Only when a line break falls onto a text index that is not safe-to-break per
 /// rustybuzz, we have to reshape that portion.
 pub struct Preparation<'a> {
-    /// The paragraph's full text.
+    /// The full text.
     pub text: &'a str,
-    /// Bidirectional text embedding levels for the paragraph.
+    /// Bidirectional text embedding levels.
     ///
-    /// This is `None` if the paragraph is BiDi-uniform (all the base direction).
+    /// This is `None` if all text directions are uniform (all the base
+    /// direction).
     pub bidi: Option<BidiInfo<'a>>,
     /// Text runs, spacing and layouted elements.
     pub items: Vec<(Range, Item<'a>)>,
@@ -33,15 +36,15 @@ pub struct Preparation<'a> {
     pub dir: Dir,
     /// The text language if it's the same for all children.
     pub lang: Option<Lang>,
-    /// The paragraph's resolved horizontal alignment.
+    /// The resolved horizontal alignment.
     pub align: FixedAlignment,
-    /// Whether to justify the paragraph.
+    /// Whether to justify text.
     pub justify: bool,
-    /// The paragraph's hanging indent.
+    /// Hanging indent to apply.
     pub hang: Abs,
     /// Whether to add spacing between CJK and Latin characters.
     pub cjk_latin_spacing: bool,
-    /// Whether font fallback is enabled for this paragraph.
+    /// Whether font fallback is enabled.
     pub fallback: bool,
     /// How to determine line breaks.
     pub linebreaks: Smart<Linebreaks>,
@@ -71,17 +74,18 @@ impl<'a> Preparation<'a> {
     }
 }
 
-/// Performs BiDi analysis and then prepares paragraph layout by building a
+/// Performs BiDi analysis and then prepares further layout by building a
 /// representation on which we can do line breaking without layouting each and
 /// every line from scratch.
 #[typst_macros::time]
 pub fn prepare<'a>(
     engine: &mut Engine,
-    children: &'a StyleVec,
+    children: &[Pair<'a>],
     text: &'a str,
     segments: Vec<Segment<'a>>,
     spans: SpanMapper,
     styles: StyleChain<'a>,
+    paragraph: bool,
 ) -> SourceResult<Preparation<'a>> {
     let dir = TextElem::dir_in(styles);
     let default_level = match dir {
@@ -125,19 +129,22 @@ pub fn prepare<'a>(
         add_cjk_latin_spacing(&mut items);
     }
 
+    // Only apply hanging indent to real paragraphs.
+    let hang = if paragraph { ParElem::hanging_indent_in(styles) } else { Abs::zero() };
+
     Ok(Preparation {
         text,
         bidi: is_bidi.then_some(bidi),
         items,
         indices,
         spans,
-        hyphenate: children.shared_get(styles, TextElem::hyphenate_in),
+        hyphenate: shared_get(children, styles, TextElem::hyphenate_in),
         costs: TextElem::costs_in(styles),
         dir,
-        lang: children.shared_get(styles, TextElem::lang_in),
+        lang: shared_get(children, styles, TextElem::lang_in),
         align: AlignElem::alignment_in(styles).resolve(styles).x,
         justify: ParElem::justify_in(styles),
-        hang: ParElem::hanging_indent_in(styles),
+        hang,
         cjk_latin_spacing,
         fallback: TextElem::fallback_in(styles),
         linebreaks: ParElem::linebreaks_in(styles),
@@ -145,6 +152,19 @@ pub fn prepare<'a>(
     })
 }
 
+/// Get a style property, but only if it is the same for all of the children.
+fn shared_get<T: PartialEq>(
+    children: &[Pair],
+    styles: StyleChain<'_>,
+    getter: fn(StyleChain) -> T,
+) -> Option<T> {
+    let value = getter(styles);
+    children
+        .group_by_key(|&(_, s)| s)
+        .all(|(s, _)| getter(s) == value)
+        .then_some(value)
+}
+
 /// Add some spacing between Han characters and western characters. See
 /// Requirements for Chinese Text Layout, Section 3.2.2 Mixed Text Composition
 /// in Horizontal Written Mode
diff --git a/crates/typst-layout/src/inline/shaping.rs b/crates/typst-layout/src/inline/shaping.rs
index 2ed95f14..b688981a 100644
--- a/crates/typst-layout/src/inline/shaping.rs
+++ b/crates/typst-layout/src/inline/shaping.rs
@@ -29,7 +29,7 @@ use crate::modifiers::{FrameModifiers, FrameModify};
 /// frame.
 #[derive(Clone)]
 pub struct ShapedText<'a> {
-    /// The start of the text in the full paragraph.
+    /// The start of the text in the full text.
     pub base: usize,
     /// The text that was shaped.
     pub text: &'a str,
@@ -66,9 +66,9 @@ pub struct ShapedGlyph {
     pub y_offset: Em,
     /// The adjustability of the glyph.
     pub adjustability: Adjustability,
-    /// The byte range of this glyph's cluster in the full paragraph. A cluster
-    /// is a sequence of one or multiple glyphs that cannot be separated and
-    /// must always be treated as a union.
+    /// The byte range of this glyph's cluster in the full inline layout. A
+    /// cluster is a sequence of one or multiple glyphs that cannot be separated
+    /// and must always be treated as a union.
     ///
     /// The range values of the glyphs in a [`ShapedText`] should not overlap
     /// with each other, and they should be monotonically increasing (for
@@ -405,7 +405,7 @@ impl<'a> ShapedText<'a> {
     /// Reshape a range of the shaped text, reusing information from this
     /// shaping process if possible.
     ///
-    /// The text `range` is relative to the whole paragraph.
+    /// The text `range` is relative to the whole inline layout.
     pub fn reshape(&'a self, engine: &Engine, text_range: Range) -> ShapedText<'a> {
         let text = &self.text[text_range.start - self.base..text_range.end - self.base];
         if let Some(glyphs) = self.slice_safe_to_break(text_range.clone()) {