diff options
| -rw-r--r-- | Cargo.lock | 10 | ||||
| -rw-r--r-- | Cargo.toml | 61 | ||||
| -rw-r--r-- | benches/oneshot.rs | 13 | ||||
| -rw-r--r-- | macros/src/lib.rs | 4 | ||||
| -rw-r--r-- | src/frame.rs | 2 | ||||
| -rw-r--r-- | src/lib.rs | 19 | ||||
| -rw-r--r-- | src/library/prelude.rs | 6 | ||||
| -rw-r--r-- | src/library/text/lang.rs | 2 | ||||
| -rw-r--r-- | src/library/text/quotes.rs | 2 | ||||
| -rw-r--r-- | src/library/text/raw.rs | 4 | ||||
| -rw-r--r-- | src/library/utility/mod.rs | 4 | ||||
| -rw-r--r-- | src/library/utility/string.rs | 2 | ||||
| -rw-r--r-- | src/main.rs | 2 | ||||
| -rw-r--r-- | src/model/args.rs (renamed from src/eval/args.rs) | 0 | ||||
| -rw-r--r-- | src/model/array.rs (renamed from src/eval/array.rs) | 4 | ||||
| -rw-r--r-- | src/model/capture.rs (renamed from src/eval/capture.rs) | 2 | ||||
| -rw-r--r-- | src/model/cast.rs (renamed from src/eval/cast.rs) | 27 | ||||
| -rw-r--r-- | src/model/content.rs | 31 | ||||
| -rw-r--r-- | src/model/dict.rs (renamed from src/eval/dict.rs) | 4 | ||||
| -rw-r--r-- | src/model/eval.rs (renamed from src/eval/mod.rs) | 40 | ||||
| -rw-r--r-- | src/model/func.rs (renamed from src/eval/func.rs) | 7 | ||||
| -rw-r--r-- | src/model/layout.rs | 17 | ||||
| -rw-r--r-- | src/model/methods.rs (renamed from src/eval/methods.rs) | 0 | ||||
| -rw-r--r-- | src/model/mod.rs | 34 | ||||
| -rw-r--r-- | src/model/ops.rs (renamed from src/eval/ops.rs) | 9 | ||||
| -rw-r--r-- | src/model/property.rs | 3 | ||||
| -rw-r--r-- | src/model/raw.rs (renamed from src/eval/raw.rs) | 3 | ||||
| -rw-r--r-- | src/model/recipe.rs | 6 | ||||
| -rw-r--r-- | src/model/scope.rs (renamed from src/eval/scope.rs) | 0 | ||||
| -rw-r--r-- | src/model/show.rs | 3 | ||||
| -rw-r--r-- | src/model/str.rs (renamed from src/eval/str.rs) | 85 | ||||
| -rw-r--r-- | src/model/styles.rs | 2 | ||||
| -rw-r--r-- | src/model/value.rs (renamed from src/eval/value.rs) | 15 | ||||
| -rw-r--r-- | src/model/vm.rs (renamed from src/eval/vm.rs) | 3 | ||||
| -rw-r--r-- | src/syntax/highlight.rs | 6 | ||||
| -rw-r--r-- | src/syntax/incremental.rs (renamed from src/parse/incremental.rs) | 10 | ||||
| -rw-r--r-- | src/syntax/mod.rs | 567 | ||||
| -rw-r--r-- | src/syntax/node.rs | 548 | ||||
| -rw-r--r-- | src/syntax/parser.rs (renamed from src/parse/parser.rs) | 3 | ||||
| -rw-r--r-- | src/syntax/parsing.rs (renamed from src/parse/mod.rs) | 42 | ||||
| -rw-r--r-- | src/syntax/resolve.rs (renamed from src/parse/resolve.rs) | 3 | ||||
| -rw-r--r-- | src/syntax/source.rs (renamed from src/source.rs) | 2 | ||||
| -rw-r--r-- | src/syntax/span.rs | 6 | ||||
| -rw-r--r-- | src/syntax/tokens.rs (renamed from src/parse/tokens.rs) | 4 | ||||
| -rw-r--r-- | tests/typeset.rs | 6 |
45 files changed, 801 insertions, 822 deletions
@@ -375,15 +375,6 @@ dependencies = [ ] [[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - -[[package]] name = "getrandom" version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1126,7 +1117,6 @@ dependencies = [ "dirs", "elsa", "flate2", - "fxhash", "hypher", "iai", "image", @@ -11,25 +11,23 @@ typst-macros = { path = "./macros" } # Utilities bitflags = "1" bytemuck = "1" -fxhash = "0.2" +comemo = "0.1" once_cell = "1" +regex = "1" serde = { version = "1", features = ["derive"] } +siphasher = "0.3" typed-arena = "2" unscanny = "0.1" -regex = "1" - -# Incremental compilation -comemo = "0.1" # Text and font handling hypher = "0.1" kurbo = "0.8" -ttf-parser = "0.17" rustybuzz = "0.5" +ttf-parser = "0.17" unicode-bidi = "0.3.5" +unicode-script = "0.5" unicode-segmentation = "1" unicode-xid = "0.2" -unicode-script = "0.5" xi-unicode = "0.3" # Raster and vector graphics handling @@ -37,12 +35,12 @@ image = { version = "0.24", default-features = false, features = ["png", "jpeg", usvg = { version = "0.22", default-features = false } # External implementation of user-facing features -syntect = { version = "5", default-features = false, features = ["default-syntaxes", "regex-fancy"] } -rex = { git = "https://github.com/laurmaedje/ReX" } -unicode-math = { git = "https://github.com/s3bk/unicode-math/" } -lipsum = { git = "https://github.com/reknih/lipsum" } csv = "1" +lipsum = { git = "https://github.com/reknih/lipsum" } +rex = { git = "https://github.com/laurmaedje/ReX" } serde_json = "1" +syntect = { version = "5", default-features = false, features = ["default-syntaxes", "regex-fancy"] } +unicode-math = { git = "https://github.com/s3bk/unicode-math/" } # PDF export miniz_oxide = "0.5" @@ -50,52 +48,50 @@ pdf-writer = "0.6" subsetter = "0.1" svg2pdf = "0.4" -# Raster export / rendering -tiny-skia = "0.6.2" +# Rendering +flate2 = "1" pixglyph = { git = "https://github.com/typst/pixglyph" } resvg = { version = "0.22", default-features = false } roxmltree = "0.14" -flate2 = "1" +tiny-skia = "0.6.2" -# Command line interface / tests -pico-args = { version = "0.4", optional = true } +# Command line interface +chrono = { version = "0.4", default-features = false, features = ["clock", "std"], optional = true } codespan-reporting = { version = "0.11", optional = true } -same-file = { version = "1", optional = true } -walkdir = { version = "2", optional = true } -elsa = { version = "1.7", optional = true } dirs = { version = "4", optional = true } +elsa = { version = "1.7", optional = true } memmap2 = { version = "0.5", optional = true } -siphasher = { version = "0.3", optional = true } notify = { version = "5", optional = true } -chrono = { version = "0.4", default-features = false, features = ["clock", "std"], optional = true } +pico-args = { version = "0.4", optional = true } +same-file = { version = "1", optional = true } +walkdir = { version = "2", optional = true } [dev-dependencies] iai = { git = "https://github.com/reknih/iai" } walkdir = "2" +elsa = "1.7" + +[workspace] +members = ["macros"] [features] -default = ["tests"] -tests = ["same-file", "walkdir", "elsa", "siphasher"] cli = [ - "pico-args", + "chrono", "codespan-reporting", "dirs", + "elsa", "memmap2", + "notify", + "pico-args", "same-file", "walkdir", - "elsa", - "siphasher", - "notify", - "chrono", ] [profile.dev] -# Faster compilation -debug = 0 +debug = 0 # Faster compilation [profile.dev.package."*"] -# Faster test execution -opt-level = 2 +opt-level = 2 # Faster test execution [[bin]] name = "typst" @@ -103,7 +99,6 @@ required-features = ["cli"] [[test]] name = "typeset" -required-features = ["tests"] harness = false [[bench]] diff --git a/benches/oneshot.rs b/benches/oneshot.rs index d4727512..1d82f44d 100644 --- a/benches/oneshot.rs +++ b/benches/oneshot.rs @@ -6,8 +6,7 @@ use unscanny::Scanner; use typst::diag::{FileError, FileResult}; use typst::font::{Font, FontBook}; -use typst::parse::{TokenMode, Tokens}; -use typst::source::{Source, SourceId}; +use typst::syntax::{Source, SourceId, TokenMode, Tokens}; use typst::util::Buffer; use typst::{Config, World}; @@ -55,7 +54,7 @@ fn bench_tokenize(iai: &mut Iai) { } fn bench_parse(iai: &mut Iai) { - iai.run(|| typst::parse::parse(TEXT)); + iai.run(|| typst::syntax::parse(TEXT)); } fn bench_edit(iai: &mut Iai) { @@ -77,15 +76,15 @@ fn bench_highlight(iai: &mut Iai) { fn bench_eval(iai: &mut Iai) { let world = BenchWorld::new(); let id = world.source.id(); - let route = typst::eval::Route::default(); - iai.run(|| typst::eval::eval(world.track(), route.track(), id).unwrap()); + let route = typst::model::Route::default(); + iai.run(|| typst::model::eval(world.track(), route.track(), id).unwrap()); } fn bench_layout(iai: &mut Iai) { let world = BenchWorld::new(); let id = world.source.id(); - let route = typst::eval::Route::default(); - let module = typst::eval::eval(world.track(), route.track(), id).unwrap(); + let route = typst::model::Route::default(); + let module = typst::model::eval(world.track(), route.track(), id).unwrap(); iai.run(|| typst::model::layout(world.track(), &module.content)); } diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 052e7fcf..62e27a09 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -69,12 +69,12 @@ fn expand(stream: TokenStream2, mut impl_block: syn::ItemImpl) -> Result<TokenSt use std::any::TypeId; use std::marker::PhantomData; use once_cell::sync::Lazy; - use crate::{eval, model}; + use crate::model; use super::*; #impl_block - impl<#params> eval::Node for #self_ty { + impl<#params> model::Node for #self_ty { const SHOWABLE: bool = #showable; #construct #set diff --git a/src/frame.rs b/src/frame.rs index 287bbf90..c7827a9c 100644 --- a/src/frame.rs +++ b/src/frame.rs @@ -4,13 +4,13 @@ use std::fmt::{self, Debug, Formatter, Write}; use std::num::NonZeroUsize; use std::sync::Arc; -use crate::eval::{Dict, Value}; use crate::font::Font; use crate::geom::{ Align, Em, Length, Numeric, Paint, Point, Shape, Size, Spec, Transform, }; use crate::image::Image; use crate::library::text::Lang; +use crate::model::{Dict, Value}; use crate::util::EcoString; /// A finished layout with elements at fixed positions. @@ -17,12 +17,12 @@ //! - **Exporting:** The finished layout can be exported into a supported //! format. Currently, the only supported output format is [PDF]. //! -//! [tokens]: parse::Tokens -//! [parsed]: parse::parse +//! [tokens]: syntax::Tokens +//! [parsed]: syntax::parse //! [syntax tree]: syntax::SyntaxNode //! [AST]: syntax::ast -//! [evaluate]: eval::eval -//! [module]: eval::Module +//! [evaluate]: model::eval +//! [module]: model::Module //! [content]: model::Content //! [layouted]: model::layout //! [PDF]: export::pdf @@ -38,15 +38,12 @@ pub mod geom; #[macro_use] pub mod diag; #[macro_use] -pub mod eval; +pub mod model; pub mod export; pub mod font; pub mod frame; pub mod image; pub mod library; -pub mod model; -pub mod parse; -pub mod source; pub mod syntax; use std::path::{Path, PathBuf}; @@ -54,11 +51,11 @@ use std::path::{Path, PathBuf}; use comemo::{Prehashed, Track}; use crate::diag::{FileResult, SourceResult}; -use crate::eval::{Route, Scope}; use crate::font::{Font, FontBook}; use crate::frame::Frame; use crate::model::StyleMap; -use crate::source::{Source, SourceId}; +use crate::model::{Route, Scope}; +use crate::syntax::{Source, SourceId}; use crate::util::Buffer; /// Typeset a source file into a collection of layouted frames. @@ -71,7 +68,7 @@ pub fn typeset( main: SourceId, ) -> SourceResult<Vec<Frame>> { let route = Route::default(); - let module = eval::eval(world.track(), route.track(), main)?; + let module = model::eval(world.track(), route.track(), main)?; model::layout(world.track(), &module.content) } diff --git a/src/library/prelude.rs b/src/library/prelude.rs index 03bef51e..7a4284f1 100644 --- a/src/library/prelude.rs +++ b/src/library/prelude.rs @@ -12,12 +12,12 @@ pub use typst_macros::node; pub use crate::diag::{ with_alternative, At, FileError, FileResult, SourceError, SourceResult, StrResult, }; -pub use crate::eval::{ +pub use crate::frame::*; +pub use crate::geom::*; +pub use crate::model::{ Arg, Args, Array, Cast, Dict, Dynamic, Func, Node, RawAlign, RawLength, RawStroke, Scope, Smart, Str, Value, Vm, }; -pub use crate::frame::*; -pub use crate::geom::*; pub use crate::model::{ Content, Fold, Key, Layout, LayoutNode, Regions, Resolve, Selector, Show, ShowNode, StyleChain, StyleMap, StyleVec, diff --git a/src/library/text/lang.rs b/src/library/text/lang.rs index b75b3cd8..f2193f05 100644 --- a/src/library/text/lang.rs +++ b/src/library/text/lang.rs @@ -1,5 +1,5 @@ -use crate::eval::Value; use crate::geom::Dir; +use crate::model::Value; /// A code for a natural language. #[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] diff --git a/src/library/text/quotes.rs b/src/library/text/quotes.rs index 98402ca4..0a22646a 100644 --- a/src/library/text/quotes.rs +++ b/src/library/text/quotes.rs @@ -1,5 +1,5 @@ use super::{Lang, Region}; -use crate::parse::is_newline; +use crate::syntax::is_newline; /// State machine for smart quote subtitution. #[derive(Debug, Clone)] diff --git a/src/library/text/raw.rs b/src/library/text/raw.rs index 8b0874f8..35160073 100644 --- a/src/library/text/raw.rs +++ b/src/library/text/raw.rs @@ -72,8 +72,8 @@ impl Show for RawNode { let mut realized = if matches!(lang.as_deref(), Some("typ" | "typst" | "typc")) { let root = match lang.as_deref() { - Some("typc") => crate::parse::parse_code(&self.text), - _ => crate::parse::parse(&self.text), + Some("typc") => crate::syntax::parse_code(&self.text), + _ => crate::syntax::parse(&self.text), }; let mut seq = vec![]; diff --git a/src/library/utility/mod.rs b/src/library/utility/mod.rs index d9b19d64..2d637d29 100644 --- a/src/library/utility/mod.rs +++ b/src/library/utility/mod.rs @@ -12,9 +12,9 @@ pub use string::*; use comemo::Track; -use crate::eval::{Eval, Route, Scopes, Vm}; use crate::library::prelude::*; -use crate::source::Source; +use crate::model::{Eval, Route, Scopes, Vm}; +use crate::syntax::Source; /// The name of a value's type. pub fn type_(_: &mut Vm, args: &mut Args) -> SourceResult<Value> { diff --git a/src/library/utility/string.rs b/src/library/utility/string.rs index 91a990a9..66f127d1 100644 --- a/src/library/utility/string.rs +++ b/src/library/utility/string.rs @@ -1,5 +1,5 @@ -use crate::eval::Regex; use crate::library::prelude::*; +use crate::model::Regex; /// The string representation of a value. pub fn repr(_: &mut Vm, args: &mut Args) -> SourceResult<Value> { diff --git a/src/main.rs b/src/main.rs index ef441091..e32bb8c6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,7 +21,7 @@ use walkdir::WalkDir; use typst::diag::{FileError, FileResult, SourceError, StrResult}; use typst::font::{Font, FontBook, FontInfo, FontVariant}; -use typst::source::{Source, SourceId}; +use typst::syntax::{Source, SourceId}; use typst::util::{Buffer, PathExt}; use typst::{Config, World}; diff --git a/src/eval/args.rs b/src/model/args.rs index f95fbf08..f95fbf08 100644 --- a/src/eval/args.rs +++ b/src/model/args.rs diff --git a/src/eval/array.rs b/src/model/array.rs index b77ce93c..196f02ec 100644 --- a/src/eval/array.rs +++ b/src/model/array.rs @@ -12,11 +12,11 @@ use crate::util::ArcExt; #[allow(unused_macros)] macro_rules! array { ($value:expr; $count:expr) => { - $crate::eval::Array::from_vec(vec![$value.into(); $count]) + $crate::model::Array::from_vec(vec![$value.into(); $count]) }; ($($value:expr),* $(,)?) => { - $crate::eval::Array::from_vec(vec![$($value.into()),*]) + $crate::model::Array::from_vec(vec![$($value.into()),*]) }; } diff --git a/src/eval/capture.rs b/src/model/capture.rs index 289d31e1..c4c107b2 100644 --- a/src/eval/capture.rs +++ b/src/model/capture.rs @@ -133,7 +133,7 @@ impl<'a> CapturesVisitor<'a> { #[cfg(test)] mod tests { use super::*; - use crate::parse::parse; + use crate::syntax::parse; #[track_caller] fn test(text: &str, result: &[&str]) { diff --git a/src/eval/cast.rs b/src/model/cast.rs index 99c34810..00a3fe45 100644 --- a/src/eval/cast.rs +++ b/src/model/cast.rs @@ -1,9 +1,8 @@ use std::num::NonZeroUsize; -use super::{Regex, Value}; +use super::{Content, Layout, LayoutNode, Pattern, Regex, Value}; use crate::diag::{with_alternative, StrResult}; use crate::geom::{Corners, Dir, Paint, Sides}; -use crate::model::{Content, Layout, LayoutNode, Pattern}; use crate::syntax::Spanned; use crate::util::EcoString; @@ -19,20 +18,20 @@ pub trait Cast<V = Value>: Sized { /// Implement traits for dynamic types. macro_rules! dynamic { ($type:ty: $name:literal, $($tts:tt)*) => { - impl $crate::eval::Type for $type { + impl $crate::model::Type for $type { const TYPE_NAME: &'static str = $name; } castable! { $type, - Expected: <Self as $crate::eval::Type>::TYPE_NAME, + Expected: <Self as $crate::model::Type>::TYPE_NAME, $($tts)* @this: Self => this.clone(), } - impl From<$type> for $crate::eval::Value { + impl From<$type> for $crate::model::Value { fn from(v: $type) -> Self { - $crate::eval::Value::Dyn($crate::eval::Dynamic::new(v)) + $crate::model::Value::Dyn($crate::model::Dynamic::new(v)) } } }; @@ -41,12 +40,12 @@ macro_rules! dynamic { /// Make a type castable from a value. macro_rules! castable { ($type:ty: $inner:ty) => { - impl $crate::eval::Cast<$crate::eval::Value> for $type { - fn is(value: &$crate::eval::Value) -> bool { + impl $crate::model::Cast<$crate::model::Value> for $type { + fn is(value: &$crate::model::Value) -> bool { <$inner>::is(value) } - fn cast(value: $crate::eval::Value) -> $crate::diag::StrResult<Self> { + fn cast(value: $crate::model::Value) -> $crate::diag::StrResult<Self> { <$inner>::cast(value).map(Self) } } @@ -59,22 +58,22 @@ macro_rules! castable { $(@$dyn_in:ident: $dyn_type:ty => $dyn_out:expr,)* ) => { #[allow(unreachable_patterns)] - impl $crate::eval::Cast<$crate::eval::Value> for $type { - fn is(value: &$crate::eval::Value) -> bool { + impl $crate::model::Cast<$crate::model::Value> for $type { + fn is(value: &$crate::model::Value) -> bool { #[allow(unused_variables)] match value { $($pattern => true,)* - $crate::eval::Value::Dyn(dynamic) => { + $crate::model::Value::Dyn(dynamic) => { false $(|| dynamic.is::<$dyn_type>())* } _ => false, } } - fn cast(value: $crate::eval::Value) -> $crate::diag::StrResult<Self> { + fn cast(value: $crate::model::Value) -> $crate::diag::StrResult<Self> { let found = match value { $($pattern => return Ok($out),)* - $crate::eval::Value::Dyn(dynamic) => { + $crate::model::Value::Dyn(dynamic) => { $(if let Some($dyn_in) = dynamic.downcast::<$dyn_type>() { return Ok($dyn_out); })* diff --git a/src/model/content.rs b/src/model/content.rs index 7828a3cd..5f0536c3 100644 --- a/src/model/content.rs +++ b/src/model/content.rs @@ -21,21 +21,6 @@ use crate::library::text::{ use crate::util::EcoString; use crate::World; -/// Layout content into a collection of pages. -/// -/// Relayouts until all pinned locations are converged. -#[comemo::memoize] -pub fn layout(world: Tracked<dyn World>, content: &Content) -> SourceResult<Vec<Frame>> { - let styles = StyleChain::with_root(&world.config().styles); - let scratch = Scratch::default(); - - let mut builder = Builder::new(world, &scratch, true); - builder.accept(content, styles)?; - - let (doc, shared) = builder.into_doc(styles)?; - doc.layout(world, shared) -} - /// Composable representation of styled content. /// /// This results from: @@ -332,7 +317,7 @@ impl Sum for Content { } /// Builds a document or a flow node from content. -struct Builder<'a> { +pub(super) struct Builder<'a> { /// The core context. world: Tracked<'a, dyn World>, /// Scratch arenas for building. @@ -349,7 +334,7 @@ struct Builder<'a> { /// Temporary storage arenas for building. #[derive(Default)] -struct Scratch<'a> { +pub(super) struct Scratch<'a> { /// An arena where intermediate style chains are stored. styles: Arena<StyleChain<'a>>, /// An arena where intermediate content resulting from show rules is stored. @@ -357,7 +342,11 @@ struct Scratch<'a> { } impl<'a> Builder<'a> { - fn new(world: Tracked<'a, dyn World>, scratch: &'a Scratch<'a>, top: bool) -> Self { + pub fn new( + world: Tracked<'a, dyn World>, + scratch: &'a Scratch<'a>, + top: bool, + ) -> Self { Self { world, scratch, @@ -368,7 +357,7 @@ impl<'a> Builder<'a> { } } - fn into_doc( + pub fn into_doc( mut self, styles: StyleChain<'a>, ) -> SourceResult<(DocNode, StyleChain<'a>)> { @@ -377,7 +366,7 @@ impl<'a> Builder<'a> { Ok((DocNode(pages), shared)) } - fn into_flow( + pub fn into_flow( mut self, styles: StyleChain<'a>, ) -> SourceResult<(FlowNode, StyleChain<'a>)> { @@ -386,7 +375,7 @@ impl<'a> Builder<'a> { Ok((FlowNode(children), shared)) } - fn accept( + pub fn accept( &mut self, content: &'a Content, styles: StyleChain<'a>, diff --git a/src/eval/dict.rs b/src/model/dict.rs index b95ead31..3e4fd956 100644 --- a/src/eval/dict.rs +++ b/src/model/dict.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use super::{Args, Array, Func, Str, Value, Vm}; use crate::diag::{SourceResult, StrResult}; -use crate::parse::is_ident; +use crate::syntax::is_ident; use crate::syntax::Spanned; use crate::util::ArcExt; @@ -16,7 +16,7 @@ macro_rules! dict { #[allow(unused_mut)] let mut map = std::collections::BTreeMap::new(); $(map.insert($key.into(), $value.into());)* - $crate::eval::Dict::from_map(map) + $crate::model::Dict::from_map(map) }}; } diff --git a/src/eval/mod.rs b/src/model/eval.rs index 0a3d6545..aa5f0378 100644 --- a/src/eval/mod.rs +++ b/src/model/eval.rs @@ -1,50 +1,20 @@ //! Evaluation of markup into modules. -#[macro_use] -mod cast; -#[macro_use] -mod array; -#[macro_use] -mod dict; -#[macro_use] -mod str; -#[macro_use] -mod value; -mod args; -mod capture; -mod func; -pub mod methods; -pub mod ops; -mod raw; -mod scope; -mod vm; - -pub use self::str::*; -pub use args::*; -pub use array::*; -pub use capture::*; -pub use cast::*; -pub use dict::*; -pub use func::*; -pub use raw::*; -pub use scope::*; -pub use typst_macros::node; -pub use value::*; -pub use vm::*; - use std::collections::BTreeMap; use std::sync::Arc; use comemo::{Track, Tracked}; use unicode_segmentation::UnicodeSegmentation; +use super::{ + methods, ops, Arg, Args, Array, CapturesVisitor, Closure, Content, Dict, Flow, Func, + Pattern, Recipe, Scope, Scopes, StyleEntry, StyleMap, Value, Vm, +}; use crate::diag::{At, SourceResult, StrResult, Trace, Tracepoint}; use crate::geom::{Angle, Em, Fraction, Length, Ratio}; use crate::library; -use crate::model::{Content, Pattern, Recipe, StyleEntry, StyleMap}; -use crate::source::SourceId; use crate::syntax::ast::TypedNode; -use crate::syntax::{ast, Span, Spanned, Unit}; +use crate::syntax::{ast, SourceId, Span, Spanned, Unit}; use crate::util::EcoString; use crate::World; diff --git a/src/eval/func.rs b/src/model/func.rs index c307b237..a4f63aa1 100644 --- a/src/eval/func.rs +++ b/src/model/func.rs @@ -4,11 +4,12 @@ use std::sync::Arc; use comemo::{Track, Tracked}; -use super::{Args, Eval, Flow, Route, Scope, Scopes, Value, Vm}; +use super::{ + Args, Content, Eval, Flow, NodeId, Route, Scope, Scopes, StyleMap, Value, Vm, +}; use crate::diag::{SourceResult, StrResult}; -use crate::model::{Content, NodeId, StyleMap}; -use crate::source::SourceId; use crate::syntax::ast::Expr; +use crate::syntax::SourceId; use crate::util::EcoString; use crate::World; diff --git a/src/model/layout.rs b/src/model/layout.rs index 8064afff..09888ba5 100644 --- a/src/model/layout.rs +++ b/src/model/layout.rs @@ -8,8 +8,8 @@ use std::sync::Arc; use comemo::{Prehashed, Tracked}; use super::{Barrier, NodeId, Resolve, StyleChain, StyleEntry}; +use super::{Builder, Content, RawAlign, RawLength, Scratch}; use crate::diag::SourceResult; -use crate::eval::{RawAlign, RawLength}; use crate::frame::{Element, Frame}; use crate::geom::{ Align, Geometry, Length, Paint, Point, Relative, Sides, Size, Spec, Stroke, @@ -18,6 +18,21 @@ use crate::library::graphics::MoveNode; use crate::library::layout::{AlignNode, PadNode}; use crate::World; +/// Layout content into a collection of pages. +/// +/// Relayouts until all pinned locations are converged. +#[comemo::memoize] +pub fn layout(world: Tracked<dyn World>, content: &Content) -> SourceResult<Vec<Frame>> { + let styles = StyleChain::with_root(&world.config().styles); + let scratch = Scratch::default(); + + let mut builder = Builder::new(world, &scratch, true); + builder.accept(content, styles)?; + + let (doc, shared) = builder.into_doc(styles)?; + doc.layout(world, shared) +} + /// A node that can be layouted into a sequence of regions. /// /// Layouting returns one frame per used region. diff --git a/src/eval/methods.rs b/src/model/methods.rs index 57fff681..57fff681 100644 --- a/src/eval/methods.rs +++ b/src/model/methods.rs diff --git a/src/model/mod.rs b/src/model/mod.rs index 5c8b82c0..0ea5cbdd 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -1,18 +1,50 @@ -//! Styled and structured representation of layoutable content. +//! Layout and computation model. #[macro_use] mod styles; mod collapse; mod content; +mod eval; mod layout; mod property; mod recipe; mod show; +#[macro_use] +mod cast; +#[macro_use] +mod array; +#[macro_use] +mod dict; +#[macro_use] +mod str; +#[macro_use] +mod value; +mod args; +mod capture; +mod func; +pub mod methods; +pub mod ops; +mod raw; +mod scope; +mod vm; +pub use self::str::*; +pub use args::*; +pub use array::*; +pub use capture::*; +pub use cast::*; pub use collapse::*; pub use content::*; +pub use dict::*; +pub use eval::*; +pub use func::*; pub use layout::*; pub use property::*; +pub use raw::*; pub use recipe::*; +pub use scope::*; pub use show::*; pub use styles::*; +pub use typst_macros::node; +pub use value::*; +pub use vm::*; diff --git a/src/eval/ops.rs b/src/model/ops.rs index 7e465320..c521f704 100644 --- a/src/eval/ops.rs +++ b/src/model/ops.rs @@ -5,7 +5,6 @@ use std::cmp::Ordering; use super::{RawAlign, RawLength, RawStroke, Regex, Smart, Value}; use crate::diag::StrResult; use crate::geom::{Numeric, Relative, Spec, SpecAxis}; -use crate::model; use Value::*; /// Bail with a type mismatch error. @@ -21,8 +20,8 @@ pub fn join(lhs: Value, rhs: Value) -> StrResult<Value> { (a, None) => a, (None, b) => b, (Str(a), Str(b)) => Str(a + b), - (Str(a), Content(b)) => Content(model::Content::Text(a.into()) + b), - (Content(a), Str(b)) => Content(a + model::Content::Text(b.into())), + (Str(a), Content(b)) => Content(super::Content::Text(a.into()) + b), + (Content(a), Str(b)) => Content(a + super::Content::Text(b.into())), (Content(a), Content(b)) => Content(a + b), (Array(a), Array(b)) => Array(a + b), (Dict(a), Dict(b)) => Dict(a + b), @@ -87,8 +86,8 @@ pub fn add(lhs: Value, rhs: Value) -> StrResult<Value> { (Str(a), Str(b)) => Str(a + b), (Content(a), Content(b)) => Content(a + b), - (Content(a), Str(b)) => Content(a + model::Content::Text(b.into())), - (Str(a), Content(b)) => Content(model::Content::Text(a.into()) + b), + (Content(a), Str(b)) => Content(a + super::Content::Text(b.into())), + (Str(a), Content(b)) => Content(super::Content::Text(a.into()) + b), (Array(a), Array(b)) => Array(a + b), (Dict(a), Dict(b)) => Dict(a + b), diff --git a/src/model/property.rs b/src/model/property.rs index ab4f02e3..ed2ab1d0 100644 --- a/src/model/property.rs +++ b/src/model/property.rs @@ -5,8 +5,7 @@ use std::sync::Arc; use comemo::Prehashed; -use super::{Interruption, NodeId, StyleChain}; -use crate::eval::{RawLength, Smart}; +use super::{Interruption, NodeId, RawLength, Smart, StyleChain}; use crate::geom::{Corners, Length, Numeric, Relative, Sides, Spec}; use crate::library::layout::PageNode; use crate::library::structure::{DescNode, EnumNode, ListNode}; diff --git a/src/eval/raw.rs b/src/model/raw.rs index 9cf346b1..b40a88ec 100644 --- a/src/eval/raw.rs +++ b/src/model/raw.rs @@ -2,12 +2,11 @@ use std::cmp::Ordering; use std::fmt::{self, Debug, Formatter}; use std::ops::{Add, Div, Mul, Neg}; -use super::{Smart, Value}; +use super::{Fold, Resolve, Smart, StyleChain, Value}; use crate::geom::{ Align, Em, Get, Length, Numeric, Paint, Relative, Spec, SpecAxis, Stroke, }; use crate::library::text::TextNode; -use crate::model::{Fold, Resolve, StyleChain}; /// The unresolved alignment representation. #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] diff --git a/src/model/recipe.rs b/src/model/recipe.rs index 27b1be42..05ef07a6 100644 --- a/src/model/recipe.rs +++ b/src/model/recipe.rs @@ -2,9 +2,11 @@ use std::fmt::{self, Debug, Formatter}; use comemo::Tracked; -use super::{Content, Interruption, NodeId, Show, ShowNode, StyleChain, StyleEntry}; +use super::{ + Args, Content, Func, Interruption, NodeId, Regex, Show, ShowNode, StyleChain, + StyleEntry, Value, +}; use crate::diag::SourceResult; -use crate::eval::{Args, Func, Regex, Value}; use crate::library::structure::{DescNode, EnumNode, ListNode}; use crate::syntax::Spanned; use crate::World; diff --git a/src/eval/scope.rs b/src/model/scope.rs index 1ab7032c..1ab7032c 100644 --- a/src/eval/scope.rs +++ b/src/model/scope.rs diff --git a/src/model/show.rs b/src/model/show.rs index b30b2264..bff69448 100644 --- a/src/model/show.rs +++ b/src/model/show.rs @@ -4,9 +4,8 @@ use std::sync::Arc; use comemo::{Prehashed, Tracked}; -use super::{Content, NodeId, Selector, StyleChain}; +use super::{Content, Dict, NodeId, Selector, StyleChain}; use crate::diag::SourceResult; -use crate::eval::Dict; use crate::World; /// A node that can be realized given some styles. diff --git a/src/eval/str.rs b/src/model/str.rs index 9d2375d3..62b37845 100644 --- a/src/eval/str.rs +++ b/src/model/str.rs @@ -13,7 +13,7 @@ use crate::util::EcoString; #[allow(unused_macros)] macro_rules! format_str { ($($tts:tt)*) => {{ - $crate::eval::Str::from(format_eco!($($tts)*)) + $crate::model::Str::from(format_eco!($($tts)*)) }}; } @@ -76,69 +76,69 @@ impl Str { } /// Whether the given pattern exists in this string. - pub fn contains(&self, pattern: TextPattern) -> bool { + pub fn contains(&self, pattern: StrPattern) -> bool { match pattern { - TextPattern::Str(pat) => self.0.contains(pat.as_str()), - TextPattern::Regex(re) => re.is_match(self), + StrPattern::Str(pat) => self.0.contains(pat.as_str()), + StrPattern::Regex(re) => re.is_match(self), } } /// Whether this string begins with the given pattern. - pub fn starts_with(&self, pattern: TextPattern) -> bool { + pub fn starts_with(&self, pattern: StrPattern) -> bool { match pattern { - TextPattern::Str(pat) => self.0.starts_with(pat.as_str()), - TextPattern::Regex(re) => re.find(self).map_or(false, |m| m.start() == 0), + StrPattern::Str(pat) => self.0.starts_with(pat.as_str()), + StrPattern::Regex(re) => re.find(self).map_or(false, |m| m.start() == 0), } } /// Whether this string ends with the given pattern. - pub fn ends_with(&self, pattern: TextPattern) -> bool { + pub fn ends_with(&self, pattern: StrPattern) -> bool { match pattern { - TextPattern::Str(pat) => self.0.ends_with(pat.as_str()), - TextPattern::Regex(re) => { + StrPattern::Str(pat) => self.0.ends_with(pat.as_str()), + StrPattern::Regex(re) => { re.find_iter(self).last().map_or(false, |m| m.end() == self.0.len()) } } } /// The text of the pattern's first match in this string. - pub fn find(&self, pattern: TextPattern) -> Option<Self> { + pub fn find(&self, pattern: StrPattern) -> Option<Self> { match pattern { - TextPattern::Str(pat) => self.0.contains(pat.as_str()).then(|| pat), - TextPattern::Regex(re) => re.find(self).map(|m| m.as_str().into()), + StrPattern::Str(pat) => self.0.contains(pat.as_str()).then(|| pat), + StrPattern::Regex(re) => re.find(self).map(|m| m.as_str().into()), } } /// The position of the pattern's first match in this string. - pub fn position(&self, pattern: TextPattern) -> Option<i64> { + pub fn position(&self, pattern: StrPattern) -> Option<i64> { match pattern { - TextPattern::Str(pat) => self.0.find(pat.as_str()).map(|i| i as i64), - TextPattern::Regex(re) => re.find(self).map(|m| m.start() as i64), + StrPattern::Str(pat) => self.0.find(pat.as_str()).map(|i| i as i64), + StrPattern::Regex(re) => re.find(self).map(|m| m.start() as i64), } } /// The start and, text and capture groups (if any) of the first match of /// the pattern in this string. - pub fn match_(&self, pattern: TextPattern) -> Option<Dict> { + pub fn match_(&self, pattern: StrPattern) -> Option<Dict> { match pattern { - TextPattern::Str(pat) => { + StrPattern::Str(pat) => { self.0.match_indices(pat.as_str()).next().map(match_to_dict) } - TextPattern::Regex(re) => re.captures(self).map(captures_to_dict), + StrPattern::Regex(re) => re.captures(self).map(captures_to_dict), } } /// The start, end, text and capture groups (if any) of all matches of the /// pattern in this string. - pub fn matches(&self, pattern: TextPattern) -> Array { + pub fn matches(&self, pattern: StrPattern) -> Array { match pattern { - TextPattern::Str(pat) => self + StrPattern::Str(pat) => self .0 .match_indices(pat.as_str()) .map(match_to_dict) .map(Value::Dict) .collect(), - TextPattern::Regex(re) => re + StrPattern::Regex(re) => re .captures_iter(self) .map(captures_to_dict) .map(Value::Dict) @@ -147,14 +147,14 @@ impl Str { } /// Split this string at whitespace or a specific pattern. - pub fn split(&self, pattern: Option<TextPattern>) -> Array { + pub fn split(&self, pattern: Option<StrPattern>) -> Array { let s = self.as_str(); match pattern { None => s.split_whitespace().map(|v| Value::Str(v.into())).collect(), - Some(TextPattern::Str(pat)) => { + Some(StrPattern::Str(pat)) => { s.split(pat.as_str()).map(|v| Value::Str(v.into())).collect() } - Some(TextPattern::Regex(re)) => { + Some(StrPattern::Regex(re)) => { re.split(s).map(|v| Value::Str(v.into())).collect() } } @@ -166,20 +166,20 @@ impl Str { /// pattern. pub fn trim( &self, - pattern: Option<TextPattern>, - at: Option<TextSide>, + pattern: Option<StrPattern>, + at: Option<StrSide>, repeat: bool, ) -> Self { - let mut start = matches!(at, Some(TextSide::Start) | None); - let end = matches!(at, Some(TextSide::End) | None); + let mut start = matches!(at, Some(StrSide::Start) | None); + let end = matches!(at, Some(StrSide::End) | None); let trimmed = match pattern { None => match at { None => self.0.trim(), - Some(TextSide::Start) => self.0.trim_start(), - Some(TextSide::End) => self.0.trim_end(), + Some(StrSide::Start) => self.0.trim_start(), + Some(StrSide::End) => self.0.trim_end(), }, - Some(TextPattern::Str(pat)) => { + Some(StrPattern::Str(pat)) => { let pat = pat.as_str(); let mut s = self.as_str(); if repeat { @@ -199,7 +199,7 @@ impl Str { } s } - Some(TextPattern::Regex(re)) => { + Some(StrPattern::Regex(re)) => { let s = self.as_str(); let mut last = 0; let mut range = 0 .. s.len(); @@ -239,18 +239,13 @@ impl Str { /// Replace at most `count` occurances of the given pattern with a /// replacement string (beginning from the start). - pub fn replace( - &self, - pattern: TextPattern, - with: Self, - count: Option<usize>, - ) -> Self { + pub fn replace(&self, pattern: StrPattern, with: Self, count: Option<usize>) -> Self { match pattern { - TextPattern::Str(pat) => match count { + StrPattern::Str(pat) => match count { Some(n) => self.0.replacen(pat.as_str(), &with, n).into(), None => self.0.replace(pat.as_str(), &with).into(), }, - TextPattern::Regex(re) => match count { + StrPattern::Regex(re) => match count { Some(n) => re.replacen(self, n, with.as_str()).into(), None => re.replace(self, with.as_str()).into(), }, @@ -440,7 +435,7 @@ impl Hash for Regex { /// A pattern which can be searched for in a string. #[derive(Debug, Clone)] -pub enum TextPattern { +pub enum StrPattern { /// Just a string. Str(Str), /// A regular expression. @@ -448,7 +443,7 @@ pub enum TextPattern { } castable! { - TextPattern, + StrPattern, Expected: "string or regular expression", Value::Str(text) => Self::Str(text), @regex: Regex => Self::Regex(regex.clone()), @@ -456,7 +451,7 @@ castable! { /// A side of a string. #[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd)] -pub enum TextSide { +pub enum StrSide { /// The logical start of the string, may be left or right depending on the /// language. Start, @@ -465,7 +460,7 @@ pub enum TextSide { } castable! { - TextSide, + StrSide, Expected: "start or end", @align: RawAlign => match align { RawAlign::Start => Self::Start, diff --git a/src/model/styles.rs b/src/model/styles.rs index 93b615fc..76199ca1 100644 --- a/src/model/styles.rs +++ b/src/model/styles.rs @@ -107,7 +107,7 @@ impl StyleMap { /// Mark all contained properties as _scoped_. This means that they only /// apply to the first descendant node (of their type) in the hierarchy and /// not its children, too. This is used by - /// [constructors](crate::eval::Node::construct). + /// [constructors](super::Node::construct). pub fn scoped(mut self) -> Self { for entry in &mut self.0 { if let StyleEntry::Property(property) = entry { diff --git a/src/eval/value.rs b/src/model/value.rs index b7bd6d3c..4075ce9c 100644 --- a/src/eval/value.rs +++ b/src/model/value.rs @@ -4,11 +4,12 @@ use std::fmt::{self, Debug, Formatter}; use std::hash::{Hash, Hasher}; use std::sync::Arc; -use super::{ops, Args, Array, Cast, Dict, Func, RawLength, Str}; +use siphasher::sip128::{Hasher128, SipHasher}; + +use super::{ops, Args, Array, Cast, Content, Dict, Func, Layout, RawLength, Str}; use crate::diag::StrResult; use crate::geom::{Angle, Color, Em, Fraction, Length, Ratio, Relative, RgbaColor}; use crate::library::text::RawNode; -use crate::model::{Content, Layout}; use crate::util::EcoString; /// A computational value. @@ -296,7 +297,7 @@ trait Bounds: Debug + Sync + Send + 'static { fn as_any(&self) -> &dyn Any; fn dyn_eq(&self, other: &Dynamic) -> bool; fn dyn_type_name(&self) -> &'static str; - fn hash64(&self) -> u64; + fn hash128(&self) -> u128; } impl<T> Bounds for T @@ -319,19 +320,19 @@ where T::TYPE_NAME } - fn hash64(&self) -> u64 { + fn hash128(&self) -> u128 { // Also hash the TypeId since nodes with different types but // equal data should be different. - let mut state = fxhash::FxHasher64::default(); + let mut state = SipHasher::new(); self.type_id().hash(&mut state); self.hash(&mut state); - state.finish() + state.finish128().as_u128() } } impl Hash for dyn Bounds { fn hash<H: Hasher>(&self, state: &mut H) { - state.write_u64(self.hash64()); + state.write_u128(self.hash128()); } } diff --git a/src/eval/vm.rs b/src/model/vm.rs index 0604e7be..a1b1ba81 100644 --- a/src/eval/vm.rs +++ b/src/model/vm.rs @@ -4,8 +4,7 @@ use comemo::Tracked; use super::{Route, Scopes, Value}; use crate::diag::{SourceError, StrResult}; -use crate::source::SourceId; -use crate::syntax::Span; +use crate::syntax::{SourceId, Span}; use crate::util::PathExt; use crate::World; diff --git a/src/syntax/highlight.rs b/src/syntax/highlight.rs index bfb36078..325b7274 100644 --- a/src/syntax/highlight.rs +++ b/src/syntax/highlight.rs @@ -6,7 +6,7 @@ use std::ops::Range; use syntect::highlighting::{Color, FontStyle, Highlighter, Style, Theme}; use syntect::parsing::Scope; -use super::{NodeKind, SyntaxNode}; +use super::{parse, NodeKind, SyntaxNode}; /// Highlight source text into a standalone HTML document. pub fn highlight_html(text: &str, theme: &Theme) -> String { @@ -28,7 +28,7 @@ pub fn highlight_pre(text: &str, theme: &Theme) -> String { let mut buf = String::new(); buf.push_str("<pre>\n"); - let root = crate::parse::parse(text); + let root = parse(text); highlight_themed(&root, theme, |range, style| { let styled = style != Style::default(); if styled { @@ -401,8 +401,8 @@ impl Category { #[cfg(test)] mod tests { + use super::super::Source; use super::*; - use crate::source::Source; #[test] fn test_highlighting() { diff --git a/src/parse/incremental.rs b/src/syntax/incremental.rs index 4651a784..529defd7 100644 --- a/src/parse/incremental.rs +++ b/src/syntax/incremental.rs @@ -1,10 +1,9 @@ use std::ops::Range; use std::sync::Arc; -use crate::syntax::{InnerNode, NodeKind, Span, SyntaxNode}; - use super::{ - is_newline, parse, reparse_code_block, reparse_content_block, reparse_markup_elements, + is_newline, parse, reparse_code_block, reparse_content_block, + reparse_markup_elements, InnerNode, NodeKind, Span, SyntaxNode, }; /// Refresh the given syntax node with as little parsing as possible. @@ -413,9 +412,8 @@ fn next_at_start(kind: &NodeKind, prev: bool) -> bool { #[rustfmt::skip] mod tests { use super::*; - use crate::parse::parse; - use crate::parse::tests::check; - use crate::source::Source; + use super::super::{parse, Source}; + use super::super::tests::check; #[track_caller] fn test(prev: &str, range: Range<usize>, with: &str, goal: Range<usize>) { diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs index 8b172def..1a23db5f 100644 --- a/src/syntax/mod.rs +++ b/src/syntax/mod.rs @@ -1,558 +1,41 @@ -//! Syntax types. +//! Syntax definition, parsing, and highlighting. pub mod ast; pub mod highlight; +mod incremental; mod kind; +mod node; +mod parser; +mod parsing; +mod resolve; +mod source; mod span; - -use std::fmt::{self, Debug, Formatter}; -use std::ops::Range; -use std::sync::Arc; +mod tokens; pub use kind::*; +pub use node::*; +pub use parsing::*; +pub use source::*; pub use span::*; +pub use tokens::*; -use self::ast::TypedNode; -use crate::diag::SourceError; -use crate::source::SourceId; - -/// An inner or leaf node in the untyped syntax tree. -#[derive(Clone, PartialEq, Hash)] -pub enum SyntaxNode { - /// A reference-counted inner node. - Inner(Arc<InnerNode>), - /// A leaf token. - Leaf(NodeData), -} - -impl SyntaxNode { - /// The metadata of the node. - pub fn data(&self) -> &NodeData { - match self { - Self::Inner(inner) => &inner.data, - Self::Leaf(leaf) => leaf, - } - } - - /// The type of the node. - pub fn kind(&self) -> &NodeKind { - self.data().kind() - } - - /// The length of the node. - pub fn len(&self) -> usize { - self.data().len() - } - - /// The number of descendants, including the node itself. - pub fn descendants(&self) -> usize { - match self { - Self::Inner(inner) => inner.descendants(), - Self::Leaf(_) => 1, - } - } - - /// The span of the node. - pub fn span(&self) -> Span { - self.data().span() - } - - /// Whether the node or its children contain an error. - pub fn erroneous(&self) -> bool { - match self { - Self::Inner(node) => node.erroneous, - Self::Leaf(data) => data.kind.is_error(), - } - } +use incremental::reparse; +use parser::*; - /// The error messages for this node and its descendants. - pub fn errors(&self) -> Vec<SourceError> { - if !self.erroneous() { - return vec![]; - } +#[cfg(test)] +mod tests { + use std::fmt::Debug; - match self.kind() { - NodeKind::Error(pos, message) => { - vec![SourceError::new(self.span(), message.clone()).with_pos(*pos)] - } - _ => self - .children() - .filter(|node| node.erroneous()) - .flat_map(|node| node.errors()) - .collect(), - } - } - - /// The node's children. - pub fn children(&self) -> std::slice::Iter<'_, SyntaxNode> { - match self { - Self::Inner(inner) => inner.children(), - Self::Leaf(_) => [].iter(), - } - } - - /// Convert the node to a typed AST node. - pub fn cast<T>(&self) -> Option<T> + #[track_caller] + pub fn check<T>(text: &str, found: T, expected: T) where - T: TypedNode, + T: Debug + PartialEq, { - T::from_untyped(self) - } - - /// Get the first child that can cast to the AST type `T`. - pub fn cast_first_child<T: TypedNode>(&self) -> Option<T> { - self.children().find_map(Self::cast) - } - - /// Get the last child that can cast to the AST type `T`. - pub fn cast_last_child<T: TypedNode>(&self) -> Option<T> { - self.children().rev().find_map(Self::cast) - } - - /// Change the type of the node. - pub fn convert(&mut self, kind: NodeKind) { - match self { - Self::Inner(inner) => { - let node = Arc::make_mut(inner); - node.erroneous |= kind.is_error(); - node.data.kind = kind; - } - Self::Leaf(leaf) => leaf.kind = kind, - } - } - - /// Set a synthetic span for the node and all its descendants. - pub fn synthesize(&mut self, span: Span) { - match self { - Self::Inner(inner) => Arc::make_mut(inner).synthesize(span), - Self::Leaf(leaf) => leaf.synthesize(span), - } - } - - /// Assign spans to each node. - pub fn numberize(&mut self, id: SourceId, within: Range<u64>) -> NumberingResult { - match self { - Self::Inner(inner) => Arc::make_mut(inner).numberize(id, None, within), - Self::Leaf(leaf) => leaf.numberize(id, within), - } - } - - /// The upper bound of assigned numbers in this subtree. - pub fn upper(&self) -> u64 { - match self { - Self::Inner(inner) => inner.upper(), - Self::Leaf(leaf) => leaf.span().number() + 1, - } - } - - /// If the span points into this node, convert it to a byte range. - pub fn range(&self, span: Span, offset: usize) -> Option<Range<usize>> { - match self { - Self::Inner(inner) => inner.range(span, offset), - Self::Leaf(leaf) => leaf.range(span, offset), - } - } - - /// Returns all leaf descendants of this node (may include itself). - /// - /// This method is slow and only intended for testing. - pub fn leafs(&self) -> Vec<Self> { - if match self { - Self::Inner(inner) => inner.children.is_empty(), - Self::Leaf(_) => true, - } { - vec![self.clone()] - } else { - self.children().flat_map(Self::leafs).collect() - } - } -} - -impl Default for SyntaxNode { - fn default() -> Self { - Self::Leaf(NodeData::new(NodeKind::None, 0)) - } -} - -impl Debug for SyntaxNode { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - match self { - Self::Inner(node) => node.fmt(f), - Self::Leaf(token) => token.fmt(f), - } - } -} - -/// An inner node in the untyped syntax tree. -#[derive(Clone, Hash)] -pub struct InnerNode { - /// Node metadata. - data: NodeData, - /// The number of nodes in the whole subtree, including this node. - descendants: usize, - /// Whether this node or any of its children are erroneous. - erroneous: bool, - /// The upper bound of this node's numbering range. - upper: u64, - /// This node's children, losslessly make up this node. - children: Vec<SyntaxNode>, -} - -impl InnerNode { - /// Creates a new node with the given kind and a single child. - pub fn with_child(kind: NodeKind, child: impl Into<SyntaxNode>) -> Self { - Self::with_children(kind, vec![child.into()]) - } - - /// Creates a new node with the given kind and children. - pub fn with_children(kind: NodeKind, children: Vec<SyntaxNode>) -> Self { - let mut len = 0; - let mut descendants = 1; - let mut erroneous = kind.is_error(); - - for child in &children { - len += child.len(); - descendants += child.descendants(); - erroneous |= child.erroneous(); - } - - Self { - data: NodeData::new(kind, len), - descendants, - erroneous, - upper: 0, - children, - } - } - - /// The node's metadata. - pub fn data(&self) -> &NodeData { - &self.data - } - - /// The node's type. - pub fn kind(&self) -> &NodeKind { - self.data().kind() - } - - /// The node's length. - pub fn len(&self) -> usize { - self.data().len() - } - - /// The node's span. - pub fn span(&self) -> Span { - self.data().span() - } - - /// The number of descendants, including the node itself. - pub fn descendants(&self) -> usize { - self.descendants - } - - /// The node's children. - pub fn children(&self) -> std::slice::Iter<'_, SyntaxNode> { - self.children.iter() - } - - /// Set a synthetic span for the node and all its descendants. - pub fn synthesize(&mut self, span: Span) { - self.data.synthesize(span); - for child in &mut self.children { - child.synthesize(span); - } - } - - /// Assign span numbers `within` an interval to this node's subtree or just - /// a `range` of its children. - pub fn numberize( - &mut self, - id: SourceId, - range: Option<Range<usize>>, - within: Range<u64>, - ) -> NumberingResult { - // Determine how many nodes we will number. - let descendants = match &range { - Some(range) if range.is_empty() => return Ok(()), - Some(range) => self.children[range.clone()] - .iter() - .map(SyntaxNode::descendants) - .sum::<usize>(), - None => self.descendants, - }; - - // Determine the distance between two neighbouring assigned numbers. If - // possible, we try to fit all numbers into the left half of `within` - // so that there is space for future insertions. - let space = within.end - within.start; - let mut stride = space / (2 * descendants as u64); - if stride == 0 { - stride = space / self.descendants as u64; - if stride == 0 { - return Err(Unnumberable); - } - } - - // Number this node itself. - let mut start = within.start; - if range.is_none() { - let end = start + stride; - self.data.numberize(id, start .. end)?; - self.upper = within.end; - start = end; - } - - // Number the children. - let len = self.children.len(); - for child in &mut self.children[range.unwrap_or(0 .. len)] { - let end = start + child.descendants() as u64 * stride; - child.numberize(id, start .. end)?; - start = end; - } - - Ok(()) - } - - /// The upper bound of assigned numbers in this subtree. - pub fn upper(&self) -> u64 { - self.upper - } - - /// If the span points into this node, convert it to a byte range. - pub fn range(&self, span: Span, mut offset: usize) -> Option<Range<usize>> { - // Check whether we found it. - if let Some(range) = self.data.range(span, offset) { - return Some(range); - } - - // The parent of a subtree has a smaller span number than all of its - // descendants. Therefore, we can bail out early if the target span's - // number is smaller than our number. - if span.number() < self.span().number() { - return None; - } - - let mut children = self.children.iter().peekable(); - while let Some(child) = children.next() { - // Every node in this child's subtree has a smaller span number than - // the next sibling. Therefore we only need to recurse if the next - // sibling's span number is larger than the target span's number. - if children - .peek() - .map_or(true, |next| next.span().number() > span.number()) - { - if let Some(range) = child.range(span, offset) { - return Some(range); - } - } - - offset += child.len(); - } - - None - } - - /// The node's children, mutably. - pub(crate) fn children_mut(&mut self) -> &mut [SyntaxNode] { - &mut self.children - } - - /// Replaces a range of children with a replacement. - /// - /// May have mutated the children if it returns `Err(_)`. - pub(crate) fn replace_children( - &mut self, - mut range: Range<usize>, - replacement: Vec<SyntaxNode>, - ) -> NumberingResult { - let superseded = &self.children[range.clone()]; - - // Compute the new byte length. - self.data.len = self.data.len - + replacement.iter().map(SyntaxNode::len).sum::<usize>() - - superseded.iter().map(SyntaxNode::len).sum::<usize>(); - - // Compute the new number of descendants. - self.descendants = self.descendants - + replacement.iter().map(SyntaxNode::descendants).sum::<usize>() - - superseded.iter().map(SyntaxNode::descendants).sum::<usize>(); - - // Determine whether we're still erroneous after the replacement. That's - // the case if - // - any of the new nodes is erroneous, - // - or if we were erroneous before due to a non-superseded node. - self.erroneous = replacement.iter().any(SyntaxNode::erroneous) - || (self.erroneous - && (self.children[.. range.start].iter().any(SyntaxNode::erroneous)) - || self.children[range.end ..].iter().any(SyntaxNode::erroneous)); - - // Perform the replacement. - let replacement_count = replacement.len(); - self.children.splice(range.clone(), replacement); - range.end = range.start + replacement_count; - - // Renumber the new children. Retries until it works, taking - // exponentially more children into account. - let mut left = 0; - let mut right = 0; - let max_left = range.start; - let max_right = self.children.len() - range.end; - loop { - let renumber = range.start - left .. range.end + right; - - // The minimum assignable number is either - // - the upper bound of the node right before the to-be-renumbered - // children, - // - or this inner node's span number plus one if renumbering starts - // at the first child. - let start_number = renumber - .start - .checked_sub(1) - .and_then(|i| self.children.get(i)) - .map_or(self.span().number() + 1, |child| child.upper()); - - // The upper bound for renumbering is either - // - the span number of the first child after the to-be-renumbered - // children, - // - or this node's upper bound if renumbering ends behind the last - // child. - let end_number = self - .children - .get(renumber.end) - .map_or(self.upper(), |next| next.span().number()); - - // Try to renumber. - let within = start_number .. end_number; - let id = self.span().source(); - if self.numberize(id, Some(renumber), within).is_ok() { - return Ok(()); - } - - // If it didn't even work with all children, we give up. - if left == max_left && right == max_right { - return Err(Unnumberable); - } - - // Exponential expansion to both sides. - left = (left + 1).next_power_of_two().min(max_left); - right = (right + 1).next_power_of_two().min(max_right); - } - } - - /// Update this node after changes were made to one of its children. - pub(crate) fn update_parent( - &mut self, - prev_len: usize, - new_len: usize, - prev_descendants: usize, - new_descendants: usize, - ) { - self.data.len = self.data.len + new_len - prev_len; - self.descendants = self.descendants + new_descendants - prev_descendants; - self.erroneous = self.children.iter().any(SyntaxNode::erroneous); - } -} - -impl From<InnerNode> for SyntaxNode { - fn from(node: InnerNode) -> Self { - Arc::new(node).into() - } -} - -impl From<Arc<InnerNode>> for SyntaxNode { - fn from(node: Arc<InnerNode>) -> Self { - Self::Inner(node) - } -} - -impl Debug for InnerNode { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - self.data.fmt(f)?; - if !self.children.is_empty() { - f.write_str(" ")?; - f.debug_list().entries(&self.children).finish()?; + if found != expected { + println!("source: {text:?}"); + println!("expected: {expected:#?}"); + println!("found: {found:#?}"); + panic!("test failed"); } - Ok(()) - } -} - -impl PartialEq for InnerNode { - fn eq(&self, other: &Self) -> bool { - self.data == other.data - && self.descendants == other.descendants - && self.erroneous == other.erroneous - && self.children == other.children - } -} - -/// Data shared between inner and leaf nodes. -#[derive(Clone, Hash)] -pub struct NodeData { - /// What kind of node this is (each kind would have its own struct in a - /// strongly typed AST). - kind: NodeKind, - /// The byte length of the node in the source. - len: usize, - /// The node's span. - span: Span, -} - -impl NodeData { - /// Create new node metadata. - pub fn new(kind: NodeKind, len: usize) -> Self { - Self { len, kind, span: Span::detached() } - } - - /// The node's type. - pub fn kind(&self) -> &NodeKind { - &self.kind - } - - /// The node's length. - pub fn len(&self) -> usize { - self.len - } - - /// The node's span. - pub fn span(&self) -> Span { - self.span - } - - /// Set a synthetic span for the node. - pub fn synthesize(&mut self, span: Span) { - self.span = span; - } - - /// Assign a span to the node. - pub fn numberize(&mut self, id: SourceId, within: Range<u64>) -> NumberingResult { - if within.start < within.end { - self.span = Span::new(id, (within.start + within.end) / 2); - Ok(()) - } else { - Err(Unnumberable) - } - } - - /// If the span points into this node, convert it to a byte range. - pub fn range(&self, span: Span, offset: usize) -> Option<Range<usize>> { - (self.span == span).then(|| offset .. offset + self.len()) - } -} - -impl From<NodeData> for SyntaxNode { - fn from(token: NodeData) -> Self { - Self::Leaf(token) - } -} - -impl Debug for NodeData { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!(f, "{:?}: {}", self.kind, self.len) - } -} - -impl PartialEq for NodeData { - fn eq(&self, other: &Self) -> bool { - self.kind == other.kind && self.len == other.len } } diff --git a/src/syntax/node.rs b/src/syntax/node.rs new file mode 100644 index 00000000..6a7d424a --- /dev/null +++ b/src/syntax/node.rs @@ -0,0 +1,548 @@ +use std::fmt::{self, Debug, Formatter}; +use std::ops::Range; +use std::sync::Arc; + +use super::ast::TypedNode; +use super::{NodeKind, NumberingResult, SourceId, Span, Unnumberable}; +use crate::diag::SourceError; + +/// An inner or leaf node in the untyped syntax tree. +#[derive(Clone, PartialEq, Hash)] +pub enum SyntaxNode { + /// A reference-counted inner node. + Inner(Arc<InnerNode>), + /// A leaf token. + Leaf(NodeData), +} + +impl SyntaxNode { + /// The metadata of the node. + pub fn data(&self) -> &NodeData { + match self { + Self::Inner(inner) => &inner.data, + Self::Leaf(leaf) => leaf, + } + } + + /// The type of the node. + pub fn kind(&self) -> &NodeKind { + self.data().kind() + } + + /// The length of the node. + pub fn len(&self) -> usize { + self.data().len() + } + + /// The number of descendants, including the node itself. + pub fn descendants(&self) -> usize { + match self { + Self::Inner(inner) => inner.descendants(), + Self::Leaf(_) => 1, + } + } + + /// The span of the node. + pub fn span(&self) -> Span { + self.data().span() + } + + /// Whether the node or its children contain an error. + pub fn erroneous(&self) -> bool { + match self { + Self::Inner(node) => node.erroneous, + Self::Leaf(data) => data.kind.is_error(), + } + } + + /// The error messages for this node and its descendants. + pub fn errors(&self) -> Vec<SourceError> { + if !self.erroneous() { + return vec![]; + } + + match self.kind() { + NodeKind::Error(pos, message) => { + vec![SourceError::new(self.span(), message.clone()).with_pos(*pos)] + } + _ => self + .children() + .filter(|node| node.erroneous()) + .flat_map(|node| node.errors()) + .collect(), + } + } + + /// The node's children. + pub fn children(&self) -> std::slice::Iter<'_, SyntaxNode> { + match self { + Self::Inner(inner) => inner.children(), + Self::Leaf(_) => [].iter(), + } + } + + /// Convert the node to a typed AST node. + pub fn cast<T>(&self) -> Option<T> + where + T: TypedNode, + { + T::from_untyped(self) + } + + /// Get the first child that can cast to the AST type `T`. + pub fn cast_first_child<T: TypedNode>(&self) -> Option<T> { + self.children().find_map(Self::cast) + } + + /// Get the last child that can cast to the AST type `T`. + pub fn cast_last_child<T: TypedNode>(&self) -> Option<T> { + self.children().rev().find_map(Self::cast) + } + + /// Change the type of the node. + pub fn convert(&mut self, kind: NodeKind) { + match self { + Self::Inner(inner) => { + let node = Arc::make_mut(inner); + node.erroneous |= kind.is_error(); + node.data.kind = kind; + } + Self::Leaf(leaf) => leaf.kind = kind, + } + } + + /// Set a synthetic span for the node and all its descendants. + pub fn synthesize(&mut self, span: Span) { + match self { + Self::Inner(inner) => Arc::make_mut(inner).synthesize(span), + Self::Leaf(leaf) => leaf.synthesize(span), + } + } + + /// Assign spans to each node. + pub fn numberize(&mut self, id: SourceId, within: Range<u64>) -> NumberingResult { + match self { + Self::Inner(inner) => Arc::make_mut(inner).numberize(id, None, within), + Self::Leaf(leaf) => leaf.numberize(id, within), + } + } + + /// The upper bound of assigned numbers in this subtree. + pub fn upper(&self) -> u64 { + match self { + Self::Inner(inner) => inner.upper(), + Self::Leaf(leaf) => leaf.span().number() + 1, + } + } + + /// If the span points into this node, convert it to a byte range. + pub fn range(&self, span: Span, offset: usize) -> Option<Range<usize>> { + match self { + Self::Inner(inner) => inner.range(span, offset), + Self::Leaf(leaf) => leaf.range(span, offset), + } + } + + /// Returns all leaf descendants of this node (may include itself). + /// + /// This method is slow and only intended for testing. + pub fn leafs(&self) -> Vec<Self> { + if match self { + Self::Inner(inner) => inner.children.is_empty(), + Self::Leaf(_) => true, + } { + vec![self.clone()] + } else { + self.children().flat_map(Self::leafs).collect() + } + } +} + +impl Default for SyntaxNode { + fn default() -> Self { + Self::Leaf(NodeData::new(NodeKind::None, 0)) + } +} + +impl Debug for SyntaxNode { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match self { + Self::Inner(node) => node.fmt(f), + Self::Leaf(token) => token.fmt(f), + } + } +} + +/// An inner node in the untyped syntax tree. +#[derive(Clone, Hash)] +pub struct InnerNode { + /// Node metadata. + data: NodeData, + /// The number of nodes in the whole subtree, including this node. + descendants: usize, + /// Whether this node or any of its children are erroneous. + erroneous: bool, + /// The upper bound of this node's numbering range. + upper: u64, + /// This node's children, losslessly make up this node. + children: Vec<SyntaxNode>, +} + +impl InnerNode { + /// Creates a new node with the given kind and a single child. + pub fn with_child(kind: NodeKind, child: impl Into<SyntaxNode>) -> Self { + Self::with_children(kind, vec![child.into()]) + } + + /// Creates a new node with the given kind and children. + pub fn with_children(kind: NodeKind, children: Vec<SyntaxNode>) -> Self { + let mut len = 0; + let mut descendants = 1; + let mut erroneous = kind.is_error(); + + for child in &children { + len += child.len(); + descendants += child.descendants(); + erroneous |= child.erroneous(); + } + + Self { + data: NodeData::new(kind, len), + descendants, + erroneous, + upper: 0, + children, + } + } + + /// The node's metadata. + pub fn data(&self) -> &NodeData { + &self.data + } + + /// The node's type. + pub fn kind(&self) -> &NodeKind { + self.data().kind() + } + + /// The node's length. + pub fn len(&self) -> usize { + self.data().len() + } + + /// The node's span. + pub fn span(&self) -> Span { + self.data().span() + } + + /// The number of descendants, including the node itself. + pub fn descendants(&self) -> usize { + self.descendants + } + + /// The node's children. + pub fn children(&self) -> std::slice::Iter<'_, SyntaxNode> { + self.children.iter() + } + + /// Set a synthetic span for the node and all its descendants. + pub fn synthesize(&mut self, span: Span) { + self.data.synthesize(span); + for child in &mut self.children { + child.synthesize(span); + } + } + + /// Assign span numbers `within` an interval to this node's subtree or just + /// a `range` of its children. + pub fn numberize( + &mut self, + id: SourceId, + range: Option<Range<usize>>, + within: Range<u64>, + ) -> NumberingResult { + // Determine how many nodes we will number. + let descendants = match &range { + Some(range) if range.is_empty() => return Ok(()), + Some(range) => self.children[range.clone()] + .iter() + .map(SyntaxNode::descendants) + .sum::<usize>(), + None => self.descendants, + }; + + // Determine the distance between two neighbouring assigned numbers. If + // possible, we try to fit all numbers into the left half of `within` + // so that there is space for future insertions. + let space = within.end - within.start; + let mut stride = space / (2 * descendants as u64); + if stride == 0 { + stride = space / self.descendants as u64; + if stride == 0 { + return Err(Unnumberable); + } + } + + // Number this node itself. + let mut start = within.start; + if range.is_none() { + let end = start + stride; + self.data.numberize(id, start .. end)?; + self.upper = within.end; + start = end; + } + + // Number the children. + let len = self.children.len(); + for child in &mut self.children[range.unwrap_or(0 .. len)] { + let end = start + child.descendants() as u64 * stride; + child.numberize(id, start .. end)?; + start = end; + } + + Ok(()) + } + + /// The upper bound of assigned numbers in this subtree. + pub fn upper(&self) -> u64 { + self.upper + } + + /// If the span points into this node, convert it to a byte range. + pub fn range(&self, span: Span, mut offset: usize) -> Option<Range<usize>> { + // Check whether we found it. + if let Some(range) = self.data.range(span, offset) { + return Some(range); + } + + // The parent of a subtree has a smaller span number than all of its + // descendants. Therefore, we can bail out early if the target span's + // number is smaller than our number. + if span.number() < self.span().number() { + return None; + } + + let mut children = self.children.iter().peekable(); + while let Some(child) = children.next() { + // Every node in this child's subtree has a smaller span number than + // the next sibling. Therefore we only need to recurse if the next + // sibling's span number is larger than the target span's number. + if children + .peek() + .map_or(true, |next| next.span().number() > span.number()) + { + if let Some(range) = child.range(span, offset) { + return Some(range); + } + } + + offset += child.len(); + } + + None + } + + /// The node's children, mutably. + pub(crate) fn children_mut(&mut self) -> &mut [SyntaxNode] { + &mut self.children + } + + /// Replaces a range of children with a replacement. + /// + /// May have mutated the children if it returns `Err(_)`. + pub(crate) fn replace_children( + &mut self, + mut range: Range<usize>, + replacement: Vec<SyntaxNode>, + ) -> NumberingResult { + let superseded = &self.children[range.clone()]; + + // Compute the new byte length. + self.data.len = self.data.len + + replacement.iter().map(SyntaxNode::len).sum::<usize>() + - superseded.iter().map(SyntaxNode::len).sum::<usize>(); + + // Compute the new number of descendants. + self.descendants = self.descendants + + replacement.iter().map(SyntaxNode::descendants).sum::<usize>() + - superseded.iter().map(SyntaxNode::descendants).sum::<usize>(); + + // Determine whether we're still erroneous after the replacement. That's + // the case if + // - any of the new nodes is erroneous, + // - or if we were erroneous before due to a non-superseded node. + self.erroneous = replacement.iter().any(SyntaxNode::erroneous) + || (self.erroneous + && (self.children[.. range.start].iter().any(SyntaxNode::erroneous)) + || self.children[range.end ..].iter().any(SyntaxNode::erroneous)); + + // Perform the replacement. + let replacement_count = replacement.len(); + self.children.splice(range.clone(), replacement); + range.end = range.start + replacement_count; + + // Renumber the new children. Retries until it works, taking + // exponentially more children into account. + let mut left = 0; + let mut right = 0; + let max_left = range.start; + let max_right = self.children.len() - range.end; + loop { + let renumber = range.start - left .. range.end + right; + + // The minimum assignable number is either + // - the upper bound of the node right before the to-be-renumbered + // children, + // - or this inner node's span number plus one if renumbering starts + // at the first child. + let start_number = renumber + .start + .checked_sub(1) + .and_then(|i| self.children.get(i)) + .map_or(self.span().number() + 1, |child| child.upper()); + + // The upper bound for renumbering is either + // - the span number of the first child after the to-be-renumbered + // children, + // - or this node's upper bound if renumbering ends behind the last + // child. + let end_number = self + .children + .get(renumber.end) + .map_or(self.upper(), |next| next.span().number()); + + // Try to renumber. + let within = start_number .. end_number; + let id = self.span().source(); + if self.numberize(id, Some(renumber), within).is_ok() { + return Ok(()); + } + + // If it didn't even work with all children, we give up. + if left == max_left && right == max_right { + return Err(Unnumberable); + } + + // Exponential expansion to both sides. + left = (left + 1).next_power_of_two().min(max_left); + right = (right + 1).next_power_of_two().min(max_right); + } + } + + /// Update this node after changes were made to one of its children. + pub(crate) fn update_parent( + &mut self, + prev_len: usize, + new_len: usize, + prev_descendants: usize, + new_descendants: usize, + ) { + self.data.len = self.data.len + new_len - prev_len; + self.descendants = self.descendants + new_descendants - prev_descendants; + self.erroneous = self.children.iter().any(SyntaxNode::erroneous); + } +} + +impl From<InnerNode> for SyntaxNode { + fn from(node: InnerNode) -> Self { + Arc::new(node).into() + } +} + +impl From<Arc<InnerNode>> for SyntaxNode { + fn from(node: Arc<InnerNode>) -> Self { + Self::Inner(node) + } +} + +impl Debug for InnerNode { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + self.data.fmt(f)?; + if !self.children.is_empty() { + f.write_str(" ")?; + f.debug_list().entries(&self.children).finish()?; + } + Ok(()) + } +} + +impl PartialEq for InnerNode { + fn eq(&self, other: &Self) -> bool { + self.data == other.data + && self.descendants == other.descendants + && self.erroneous == other.erroneous + && self.children == other.children + } +} + +/// Data shared between inner and leaf nodes. +#[derive(Clone, Hash)] +pub struct NodeData { + /// What kind of node this is (each kind would have its own struct in a + /// strongly typed AST). + pub(super) kind: NodeKind, + /// The byte length of the node in the source. + len: usize, + /// The node's span. + span: Span, +} + +impl NodeData { + /// Create new node metadata. + pub fn new(kind: NodeKind, len: usize) -> Self { + Self { len, kind, span: Span::detached() } + } + + /// The node's type. + pub fn kind(&self) -> &NodeKind { + &self.kind + } + + /// The node's length. + pub fn len(&self) -> usize { + self.len + } + + /// The node's span. + pub fn span(&self) -> Span { + self.span + } + + /// Set a synthetic span for the node. + pub fn synthesize(&mut self, span: Span) { + self.span = span; + } + + /// Assign a span to the node. + pub fn numberize(&mut self, id: SourceId, within: Range<u64>) -> NumberingResult { + if within.start < within.end { + self.span = Span::new(id, (within.start + within.end) / 2); + Ok(()) + } else { + Err(Unnumberable) + } + } + + /// If the span points into this node, convert it to a byte range. + pub fn range(&self, span: Span, offset: usize) -> Option<Range<usize>> { + (self.span == span).then(|| offset .. offset + self.len()) + } +} + +impl From<NodeData> for SyntaxNode { + fn from(token: NodeData) -> Self { + Self::Leaf(token) + } +} + +impl Debug for NodeData { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "{:?}: {}", self.kind, self.len) + } +} + +impl PartialEq for NodeData { + fn eq(&self, other: &Self) -> bool { + self.kind == other.kind && self.len == other.len + } +} diff --git a/src/parse/parser.rs b/src/syntax/parser.rs index 3dbb7d50..83b333f4 100644 --- a/src/parse/parser.rs +++ b/src/syntax/parser.rs @@ -2,8 +2,7 @@ use std::fmt::{self, Display, Formatter}; use std::mem; use std::ops::Range; -use super::{TokenMode, Tokens}; -use crate::syntax::{ErrorPos, InnerNode, NodeData, NodeKind, SyntaxNode}; +use super::{ErrorPos, InnerNode, NodeData, NodeKind, SyntaxNode, TokenMode, Tokens}; use crate::util::EcoString; /// A convenient token-based parser. diff --git a/src/parse/mod.rs b/src/syntax/parsing.rs index ac8ec6eb..10b4c4c2 100644 --- a/src/parse/mod.rs +++ b/src/syntax/parsing.rs @@ -1,18 +1,10 @@ -//! Parsing and tokenization. - -mod incremental; -mod parser; -mod resolve; -mod tokens; - -pub use incremental::*; -pub use parser::*; -pub use tokens::*; - use std::collections::HashSet; -use crate::syntax::ast::{Assoc, BinOp, UnOp}; -use crate::syntax::{ErrorPos, NodeKind, SyntaxNode}; +use super::ast::{Assoc, BinOp, UnOp}; +use super::{ + ErrorPos, Group, Marker, NodeKind, ParseError, ParseResult, Parser, SyntaxNode, + TokenMode, +}; use crate::util::EcoString; /// Parse a source file. @@ -32,7 +24,7 @@ pub fn parse_code(text: &str) -> SyntaxNode { /// Reparse a code block. /// /// Returns `Some` if all of the input was consumed. -fn reparse_code_block( +pub(crate) fn reparse_code_block( prefix: &str, text: &str, end_pos: usize, @@ -56,7 +48,7 @@ fn reparse_code_block( /// Reparse a content block. /// /// Returns `Some` if all of the input was consumed. -fn reparse_content_block( +pub(crate) fn reparse_content_block( prefix: &str, text: &str, end_pos: usize, @@ -80,7 +72,7 @@ fn reparse_content_block( /// Reparse a sequence markup elements without the topmost node. /// /// Returns `Some` if all of the input was consumed. -fn reparse_markup_elements( +pub(crate) fn reparse_markup_elements( prefix: &str, text: &str, end_pos: usize, @@ -1146,21 +1138,3 @@ fn body(p: &mut Parser) -> ParseResult { } } } - -#[cfg(test)] -mod tests { - use std::fmt::Debug; - - #[track_caller] - pub fn check<T>(text: &str, found: T, expected: T) - where - T: Debug + PartialEq, - { - if found != expected { - println!("source: {text:?}"); - println!("expected: {expected:#?}"); - println!("found: {found:#?}"); - panic!("test failed"); - } - } -} diff --git a/src/parse/resolve.rs b/src/syntax/resolve.rs index 9fde0cf4..2ad35cec 100644 --- a/src/parse/resolve.rs +++ b/src/syntax/resolve.rs @@ -1,7 +1,6 @@ use unscanny::Scanner; -use super::{is_ident, is_newline}; -use crate::syntax::RawKind; +use super::{is_ident, is_newline, RawKind}; use crate::util::EcoString; /// Resolve all escape sequences in a string. diff --git a/src/source.rs b/src/syntax/source.rs index 69e72d6b..1b87b1c9 100644 --- a/src/source.rs +++ b/src/syntax/source.rs @@ -9,8 +9,8 @@ use comemo::Prehashed; use unscanny::Scanner; use crate::diag::SourceResult; -use crate::parse::{is_newline, parse, reparse}; use crate::syntax::ast::Markup; +use crate::syntax::{is_newline, parse, reparse}; use crate::syntax::{Span, SyntaxNode}; use crate::util::{PathExt, StrExt}; diff --git a/src/syntax/span.rs b/src/syntax/span.rs index 744aa123..d4d9a8f6 100644 --- a/src/syntax/span.rs +++ b/src/syntax/span.rs @@ -2,7 +2,7 @@ use std::fmt::{self, Debug, Display, Formatter}; use std::num::NonZeroU64; use std::ops::Range; -use crate::syntax::SourceId; +use super::SourceId; /// A value with a span locating it in the source code. #[derive(Copy, Clone, Eq, PartialEq, Hash)] @@ -42,8 +42,8 @@ impl<T: Debug> Debug for Spanned<T> { /// A unique identifier for a syntax node. /// /// This is used throughout the compiler to track which source section an error -/// or element stems from. Can be [mapped back](crate::source::Source::range) -/// to a byte range for user facing display. +/// or element stems from. Can be [mapped back](super::Source::range) to a byte +/// range for user facing display. /// /// Span ids are ordered in the tree to enable quickly finding the node with /// some id: diff --git a/src/parse/tokens.rs b/src/syntax/tokens.rs index 73c64d1e..8e1b6944 100644 --- a/src/parse/tokens.rs +++ b/src/syntax/tokens.rs @@ -4,8 +4,8 @@ use unicode_xid::UnicodeXID; use unscanny::Scanner; use super::resolve::{resolve_hex, resolve_raw, resolve_string}; +use super::{ErrorPos, NodeKind, RawKind, Unit}; use crate::geom::{AngleUnit, LengthUnit}; -use crate::syntax::{ErrorPos, NodeKind, RawKind, Unit}; use crate::util::EcoString; /// An iterator over the tokens of a string of source code. @@ -710,8 +710,8 @@ fn is_math_id_continue(c: char) -> bool { #[cfg(test)] #[allow(non_snake_case)] mod tests { + use super::super::tests::check; use super::*; - use crate::parse::tests::check; use ErrorPos::*; use NodeKind::*; diff --git a/tests/typeset.rs b/tests/typeset.rs index 9eab55c8..235fcab3 100644 --- a/tests/typeset.rs +++ b/tests/typeset.rs @@ -15,15 +15,13 @@ use unscanny::Scanner; use walkdir::WalkDir; use typst::diag::{FileError, FileResult}; -use typst::eval::{Smart, Value}; use typst::font::{Font, FontBook}; use typst::frame::{Element, Frame}; use typst::geom::{Length, RgbaColor, Sides}; use typst::library::layout::PageNode; use typst::library::text::{TextNode, TextSize}; -use typst::model::StyleMap; -use typst::source::{Source, SourceId}; -use typst::syntax::SyntaxNode; +use typst::model::{Smart, StyleMap, Value}; +use typst::syntax::{Source, SourceId, SyntaxNode}; use typst::util::{Buffer, PathExt}; use typst::{bail, Config, World}; |