diff options
| author | Laurenz <laurmaedje@gmail.com> | 2020-01-26 15:51:13 +0100 |
|---|---|---|
| committer | Laurenz <laurmaedje@gmail.com> | 2020-01-26 15:51:13 +0100 |
| commit | 20fb4e7c379b79b84d9884d5f2c89d781c5793e2 (patch) | |
| tree | a1eef90680afa2b43cb1ce0a687c837fd78810e7 /src/syntax | |
| parent | 0a087cd28bbee5fcdffbb9d49b0ba9f413ad7f92 (diff) | |
Document everything 📜
Diffstat (limited to 'src/syntax')
| -rw-r--r-- | src/syntax/expr.rs | 88 | ||||
| -rw-r--r-- | src/syntax/func/keys.rs | 62 | ||||
| -rw-r--r-- | src/syntax/func/maps.rs | 65 | ||||
| -rw-r--r-- | src/syntax/func/mod.rs | 56 | ||||
| -rw-r--r-- | src/syntax/func/values.rs | 106 | ||||
| -rw-r--r-- | src/syntax/mod.rs | 50 | ||||
| -rw-r--r-- | src/syntax/parsing.rs | 48 | ||||
| -rw-r--r-- | src/syntax/scope.rs | 2 | ||||
| -rw-r--r-- | src/syntax/span.rs | 34 | ||||
| -rw-r--r-- | src/syntax/tokens.rs | 34 |
10 files changed, 439 insertions, 106 deletions
diff --git a/src/syntax/expr.rs b/src/syntax/expr.rs index b4c0dfaa..879e5fae 100644 --- a/src/syntax/expr.rs +++ b/src/syntax/expr.rs @@ -1,8 +1,10 @@ +//! Expressions in function headers. + use std::fmt::{self, Display, Formatter}; use crate::error::Errors; use crate::size::Size; -use super::func::{keys::Key, values::Value}; +use super::func::{Key, Value}; use super::span::{Span, Spanned}; use super::tokens::is_identifier; @@ -10,16 +12,24 @@ use super::tokens::is_identifier; /// An argument or return value. #[derive(Clone, PartialEq)] pub enum Expr { + /// An identifier: `ident`. Ident(Ident), + /// A string: `"string"`. Str(String), + /// A number: `1.2, 200%`. Number(f64), + /// A size: `2cm, 5.2in`. Size(Size), + /// A bool: `true, false`. Bool(bool), + /// A tuple: `(false, 12cm, "hi")`. Tuple(Tuple), + /// An object: `{ fit: false, size: 12pt }`. Object(Object), } impl Expr { + /// A natural-language name of the type of this expression, e.g. "identifier". pub fn name(&self) -> &'static str { use Expr::*; match self { @@ -34,11 +44,21 @@ impl Expr { } } -/// An identifier. +/// A unicode identifier. +/// +/// The identifier must be valid! This is checked in [`Ident::new`] or +/// [`is_identifier`]. +/// +/// # Example +/// ```typst +/// [func: "hi", ident] +/// ^^^^ ^^^^^ +/// ``` #[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] pub struct Ident(pub String); impl Ident { + /// Create a new identifier from a string checking that it is valid. pub fn new<S>(ident: S) -> Option<Ident> where S: AsRef<str> + Into<String> { if is_identifier(ident.as_ref()) { Some(Ident(ident.into())) @@ -47,26 +67,37 @@ impl Ident { } } + /// Return a reference to the underlying string. pub fn as_str(&self) -> &str { self.0.as_str() } } -/// A sequence of expressions. +/// An untyped sequence of expressions. +/// +/// # Example +/// ```typst +/// (false, 12cm, "hi") +/// ``` #[derive(Clone, PartialEq)] pub struct Tuple { + /// The elements of the tuple. pub items: Vec<Spanned<Expr>>, } impl Tuple { + /// Create an empty tuple. pub fn new() -> Tuple { Tuple { items: vec![] } } + /// Add an element. pub fn add(&mut self, item: Spanned<Expr>) { self.items.push(item); } + /// Extract (and remove) the first matching value and remove and generate + /// errors for all previous items that did not match. pub fn get<V: Value>(&mut self, errors: &mut Errors) -> Option<V::Output> { while !self.items.is_empty() { let expr = self.items.remove(0); @@ -79,6 +110,8 @@ impl Tuple { None } + /// Extract and return an iterator over all values that match and generate + /// errors for all items that do not match. pub fn get_all<'a, V: Value>(&'a mut self, errors: &'a mut Errors) -> impl Iterator<Item=V::Output> + 'a { self.items.drain(..).filter_map(move |expr| { @@ -92,36 +125,63 @@ impl Tuple { } /// A key-value collection of identifiers and associated expressions. +/// +/// The pairs themselves are not spanned, but the combined spans can easily be +/// retrieved by merging the spans of key and value as happening in +/// [`FuncArg::span`](super::func::FuncArg::span). +/// +/// # Example +/// ```typst +/// { fit: false, size: 12cm, items: (1, 2, 3) } +/// ``` #[derive(Clone, PartialEq)] pub struct Object { + /// The key-value pairs of the object. pub pairs: Vec<Pair>, } /// A key-value pair in an object. #[derive(Clone, PartialEq)] pub struct Pair { + /// The key part. + /// ```typst + /// key: value + /// ^^^ + /// ``` pub key: Spanned<Ident>, + /// The value part. + /// ```typst + /// key: value + /// ^^^^^ + /// ``` pub value: Spanned<Expr>, } impl Object { + /// Create an empty object. pub fn new() -> Object { Object { pairs: vec![] } } - pub fn add(&mut self, key: Spanned<Ident>, value: Spanned<Expr>) { - self.pairs.push(Pair { key, value }); - } - - pub fn add_pair(&mut self, pair: Pair) { + /// Add a pair to object. + pub fn add(&mut self, pair: Pair) { self.pairs.push(pair); } + /// Extract (and remove) a pair with the given key string and matching + /// value. + /// + /// Inserts an error if the value does not match. If the key is not + /// contained, no error is inserted. pub fn get<V: Value>(&mut self, errors: &mut Errors, key: &str) -> Option<V::Output> { let index = self.pairs.iter().position(|pair| pair.key.v.as_str() == key)?; self.get_index::<V>(errors, index) } + /// Extract (and remove) a pair with a matching key and value. + /// + /// Inserts an error if the value does not match. If no matching key is + /// found, no error is inserted. pub fn get_with_key<K: Key, V: Value>( &mut self, errors: &mut Errors, @@ -135,6 +195,9 @@ impl Object { None } + /// Extract (and remove) all pairs with matching keys and values. + /// + /// Inserts errors for values that do not match. pub fn get_all<'a, K: Key, V: Value>( &'a mut self, errors: &'a mut Errors, @@ -157,6 +220,13 @@ impl Object { }).filter_map(|x| x) } + /// Extract all key value pairs with span information. + /// + /// The spans are over both key and value, like so: + /// ```typst + /// { key: value } + /// ^^^^^^^^^^ + /// ``` pub fn get_all_spanned<'a, K: Key + 'a, V: Value + 'a>( &'a mut self, errors: &'a mut Errors, @@ -165,6 +235,8 @@ impl Object { .map(|(k, v)| Spanned::new((k.v, v.v), Span::merge(k.span, v.span))) } + /// Extract the argument at the given index and insert an error if the value + /// does not match. fn get_index<V: Value>(&mut self, errors: &mut Errors, index: usize) -> Option<V::Output> { let expr = self.pairs.remove(index).value; let span = expr.span; diff --git a/src/syntax/func/keys.rs b/src/syntax/func/keys.rs index 116cd4e6..d77447c8 100644 --- a/src/syntax/func/keys.rs +++ b/src/syntax/func/keys.rs @@ -1,3 +1,5 @@ +//! Key types for identifying keyword arguments. + use crate::layout::prelude::*; use super::values::AlignmentValue::{self, *}; use super::*; @@ -6,10 +8,55 @@ use self::AxisKey::*; use self::PaddingKey::*; - +/// Key types are used to extract keyword arguments from +/// [`Objects`](crate::syntax::expr::Object). They represent the key part of a +/// keyword argument. +/// ```typst +/// [func: key=value] +/// ^^^ +/// ``` +/// +/// A key type has an associated output type, which is returned when parsing +/// this key from a string. Most of the time, the output type is simply the key +/// itself, as in the implementation for the [`AxisKey`]: +/// ``` +/// # use typstc::syntax::func::Key; +/// # use typstc::syntax::span::Spanned; +/// # #[derive(Eq, PartialEq)] enum Axis { Horizontal, Vertical, Primary, Secondary } +/// # #[derive(Eq, PartialEq)] enum AxisKey { Specific(Axis), Generic(Axis) } +/// # use Axis::*; +/// # use AxisKey::*; +/// impl Key for AxisKey { +/// type Output = Self; +/// +/// fn parse(key: Spanned<&str>) -> Option<Self::Output> { +/// match key.v { +/// "horizontal" | "h" => Some(Specific(Horizontal)), +/// "vertical" | "v" => Some(Specific(Vertical)), +/// "primary" | "p" => Some(Generic(Primary)), +/// "secondary" | "s" => Some(Generic(Secondary)), +/// _ => None, +/// } +/// } +/// } +/// ``` +/// +/// The axis key would also be useful to identify axes when describing +/// dimensions of objects, as in `width=3cm`, because these are also properties +/// that are stored per axis. However, here the used keyword arguments are +/// actually different (`width` instead of `horizontal`)! Therefore we cannot +/// just use the axis key. +/// +/// To fix this, there is another type [`ExtentKey`] which implements `Key` and +/// has the associated output type axis key. The extent key struct itself has no +/// fields and is only used to extract the axis key. This way, we can specify +/// which argument kind we want without duplicating the type in the background. pub trait Key { + /// The type to parse into. type Output: Eq; + /// Parse a key string into the output type if the string is valid for this + /// key. fn parse(key: Spanned<&str>) -> Option<Self::Output>; } @@ -21,6 +68,7 @@ impl<K: Key> Key for Spanned<K> { } } +/// Implements [`Key`] for types that just need to match on strings. macro_rules! key { ($type:ty, $output:ty, $($($p:pat)|* => $r:expr),* $(,)?) => { impl Key for $type { @@ -36,8 +84,9 @@ macro_rules! key { }; } -/// An argument key which identifies a layouting axis. +/// A key which identifies a layouting axis. #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] +#[allow(missing_docs)] pub enum AxisKey { Generic(GenericAxis), Specific(SpecificAxis), @@ -68,6 +117,8 @@ key!(AxisKey, Self, "secondary" | "s" => Generic(Secondary), ); +/// A key which parses into an [`AxisKey`] but uses typical extent keywords +/// instead of axis keywords, e.g. `width` instead of `horizontal`. pub struct ExtentKey; key!(ExtentKey, AxisKey, @@ -77,8 +128,13 @@ key!(ExtentKey, AxisKey, "secondary-size" | "ss" => Generic(Secondary), ); -/// An argument key which identifies an axis, but allows for positional +/// A key which identifies an axis, but alternatively allows for two positional /// arguments with unspecified axes. +/// +/// This type does not implement `Key` in itself since it cannot be parsed from +/// a string. Rather, [`AxisKeys`](AxisKey) and positional arguments should be +/// parsed separately and mapped onto this key, as happening in the +/// [`PosAxisMap`](super::maps::PosAxisMap). #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] pub enum PosAxisKey { /// The first positional argument. diff --git a/src/syntax/func/maps.rs b/src/syntax/func/maps.rs index 8941024e..eb4c8394 100644 --- a/src/syntax/func/maps.rs +++ b/src/syntax/func/maps.rs @@ -9,40 +9,46 @@ use super::values::*; use super::*; -/// A deduplicating map type useful for storing possibly redundant arguments. +/// A map which deduplicates redundant arguments. +/// +/// Whenever a duplicate argument is inserted into the map, through the +/// functions `from_iter`, `insert` or `extend` an errors is added to the error +/// list that needs to be passed to those functions. +/// +/// All entries need to have span information to enable the error reporting. #[derive(Debug, Clone, PartialEq)] pub struct DedupMap<K, V> where K: Eq { map: Vec<Spanned<(K, V)>>, } impl<K, V> DedupMap<K, V> where K: Eq { + /// Create a new deduplicating map. pub fn new() -> DedupMap<K, V> { DedupMap { map: vec![] } } + /// Create a new map from an iterator of spanned keys and values. pub fn from_iter<I>(errors: &mut Errors, iter: I) -> DedupMap<K, V> where I: IntoIterator<Item=Spanned<(K, V)>> { let mut map = DedupMap::new(); - for Spanned { v: (key, value), span } in iter.into_iter() { - map.insert(errors, key, value, span); - } + map.extend(errors, iter); map } - /// Add a key-value pair. - pub fn insert(&mut self, errors: &mut Errors, key: K, value: V, span: Span) { - if self.map.iter().any(|e| e.v.0 == key) { - errors.push(err!(span; "duplicate argument")); + /// Add a spanned key-value pair. + pub fn insert(&mut self, errors: &mut Errors, entry: Spanned<(K, V)>) { + if self.map.iter().any(|e| e.v.0 == entry.v.0) { + errors.push(err!(entry.span; "duplicate argument")); } else { - self.map.push(Spanned { v: (key, value), span }); + self.map.push(entry); } } - /// Add multiple key-value pairs. + /// Add multiple spanned key-value pairs. pub fn extend<I>(&mut self, errors: &mut Errors, items: I) where I: IntoIterator<Item=Spanned<(K, V)>> { - for Spanned { v: (k, v), span } in items.into_iter() { - self.insert(errors, k, v, span); + for item in items.into_iter() { + self.insert(errors, item); } } @@ -65,16 +71,15 @@ impl<K, V> DedupMap<K, V> where K: Eq { } /// Create a new map where keys and values are mapped to new keys and - /// values. - /// - /// Returns an error if a new key is duplicate. + /// values. When the mapping introduces new duplicates, errors are + /// generated. pub fn dedup<F, K2, V2>(&self, errors: &mut Errors, mut f: F) -> DedupMap<K2, V2> where F: FnMut(&K, &V) -> (K2, V2), K2: Eq { let mut map = DedupMap::new(); for Spanned { v: (key, value), span } in self.map.iter() { let (key, value) = f(key, value); - map.insert(errors, key, value, *span); + map.insert(errors, Spanned { v: (key, value), span: *span }); } map @@ -86,11 +91,12 @@ impl<K, V> DedupMap<K, V> where K: Eq { } } -/// A map for storing a value for two axes given by keyword arguments. +/// A map for storing a value for axes given by keyword arguments. #[derive(Debug, Clone, PartialEq)] pub struct AxisMap<V>(DedupMap<AxisKey, V>); impl<V: Clone> AxisMap<V> { + /// Parse an axis map from the object. pub fn parse<KT: Key<Output=AxisKey>, VT: Value<Output=V>>( errors: &mut Errors, object: &mut Object, @@ -105,12 +111,13 @@ impl<V: Clone> AxisMap<V> { } } -/// A map for extracting values for two axes that are given through two -/// positional or keyword arguments. +/// A map for storing values for axes that are given through a combination of +/// (two) positional and keyword arguments. #[derive(Debug, Clone, PartialEq)] pub struct PosAxisMap<V>(DedupMap<PosAxisKey, V>); impl<V: Clone> PosAxisMap<V> { + /// Parse a positional/axis map from the function arguments. pub fn parse<KT: Key<Output=AxisKey>, VT: Value<Output=V>>( errors: &mut Errors, args: &mut FuncArgs, @@ -118,8 +125,8 @@ impl<V: Clone> PosAxisMap<V> { let mut map = DedupMap::new(); for &key in &[PosAxisKey::First, PosAxisKey::Second] { - if let Some(value) = args.pos.get::<Spanned<VT>>(errors) { - map.insert(errors, key, value.v, value.span); + if let Some(Spanned { v, span }) = args.pos.get::<Spanned<VT>>(errors) { + map.insert(errors, Spanned { v: (key, v), span }) } } @@ -133,7 +140,8 @@ impl<V: Clone> PosAxisMap<V> { PosAxisMap(map) } - /// Deduplicate from positional or specific to generic axes. + /// Deduplicate from positional arguments and keyword arguments for generic + /// or specific axes to just generic axes. pub fn dedup<F>( &self, errors: &mut Errors, @@ -151,17 +159,19 @@ impl<V: Clone> PosAxisMap<V> { } } -/// A map for extracting padding for a set of specifications given for all -/// sides, opposing sides or single sides. +/// A map for storing padding given for a combination of all sides, opposing +/// sides or single sides. #[derive(Debug, Clone, PartialEq)] pub struct PaddingMap(DedupMap<PaddingKey<AxisKey>, Option<PSize>>); impl PaddingMap { + /// Parse a padding map from the function arguments. pub fn parse(errors: &mut Errors, args: &mut FuncArgs) -> PaddingMap { let mut map = DedupMap::new(); - if let Some(psize) = args.pos.get::<Spanned<Defaultable<PSize>>>(errors) { - map.insert(errors, PaddingKey::All, psize.v, psize.span); + let all = args.pos.get::<Spanned<Defaultable<PSize>>>(errors); + if let Some(Spanned { v, span }) = all { + map.insert(errors, Spanned { v: (PaddingKey::All, v), span }); } let paddings: Vec<_> = args.key @@ -187,8 +197,9 @@ impl PaddingMap { All => All, Both(axis) => Both(axis.to_specific(axes)), Side(axis, alignment) => { + let generic = axis.to_generic(axes); let axis = axis.to_specific(axes); - Side(axis, alignment.to_specific(axes, axis)) + Side(axis, alignment.to_specific(axes, generic)) } }, val) }); diff --git a/src/syntax/func/mod.rs b/src/syntax/func/mod.rs index e66b4d6a..0c5b4447 100644 --- a/src/syntax/func/mod.rs +++ b/src/syntax/func/mod.rs @@ -1,25 +1,38 @@ +//! Primitives for argument parsing in library functions. + use crate::error::{Error, Errors}; use super::expr::{Expr, Ident, Tuple, Object, Pair}; use super::span::{Span, Spanned}; -pub mod maps; -pub mod keys; -pub mod values; +pub_use_mod!(maps); +pub_use_mod!(keys); +pub_use_mod!(values); +/// The parsed header of a function. #[derive(Debug, Clone, PartialEq)] pub struct FuncHeader { + /// The function name, that is: + /// ```typst + /// [box: w=5cm] + /// ^^^ + /// ``` pub name: Spanned<Ident>, + /// The arguments passed to the function. pub args: FuncArgs, } +/// The positional and keyword arguments passed to a function. #[derive(Debug, Clone, PartialEq)] pub struct FuncArgs { + /// The positional arguments. pub pos: Tuple, + /// They keyword arguments. pub key: Object, } impl FuncArgs { + /// Create new empty function arguments. pub fn new() -> FuncArgs { FuncArgs { pos: Tuple::new(), @@ -30,40 +43,32 @@ impl FuncArgs { /// Add an argument. pub fn add(&mut self, arg: FuncArg) { match arg { - FuncArg::Pos(item) => self.add_pos(item), - FuncArg::Key(pair) => self.add_key_pair(pair), + FuncArg::Pos(item) => self.pos.add(item), + FuncArg::Key(pair) => self.key.add(pair), } } - /// Add a positional argument. - pub fn add_pos(&mut self, item: Spanned<Expr>) { - self.pos.add(item); - } - - /// Add a keyword argument. - pub fn add_key(&mut self, key: Spanned<Ident>, value: Spanned<Expr>) { - self.key.add(key, value); - } - - /// Add a keyword argument from an existing pair. - pub fn add_key_pair(&mut self, pair: Pair) { - self.key.add_pair(pair); - } - + /// Iterate over all arguments. pub fn into_iter(self) -> impl Iterator<Item=FuncArg> { self.pos.items.into_iter().map(|item| FuncArg::Pos(item)) .chain(self.key.pairs.into_iter().map(|pair| FuncArg::Key(pair))) } } +/// Either a positional or keyword argument. #[derive(Debug, Clone, PartialEq)] pub enum FuncArg { + /// A positional argument. Pos(Spanned<Expr>), + /// A keyword argument. Key(Pair), } impl FuncArg { - /// The span or the value or combined span of key and value. + /// The full span of this argument. + /// + /// In case of a positional argument this is just the span of the expression + /// and in case of a keyword argument the combined span of key and value. pub fn span(&self) -> Span { match self { FuncArg::Pos(item) => item.span, @@ -72,14 +77,17 @@ impl FuncArg { } } +/// Extra methods on [`Options`](Option) used for argument parsing. pub trait OptionExt: Sized { - fn or_missing(self, errors: &mut Errors, span: Span, what: &str) -> Self; + /// Add an error about a missing argument `arg` with the given span if the + /// option is `None`. + fn or_missing(self, errors: &mut Errors, span: Span, arg: &str) -> Self; } impl<T> OptionExt for Option<T> { - fn or_missing(self, errors: &mut Errors, span: Span, what: &str) -> Self { + fn or_missing(self, errors: &mut Errors, span: Span, arg: &str) -> Self { if self.is_none() { - errors.push(err!(span; "missing argument: {}", what)); + errors.push(err!(span; "missing argument: {}", arg)); } self } diff --git a/src/syntax/func/values.rs b/src/syntax/func/values.rs index a767aef6..515d6a43 100644 --- a/src/syntax/func/values.rs +++ b/src/syntax/func/values.rs @@ -1,3 +1,5 @@ +//! Value types for extracting function arguments. + use std::fmt::{self, Display, Formatter}; use std::marker::PhantomData; use toddle::query::{FontStyle, FontWeight}; @@ -10,9 +12,65 @@ use super::*; use self::AlignmentValue::*; +/// Value types are used to extract the values of positional and keyword +/// arguments from [`Tuples`](crate::syntax::expr::Tuple) and +/// [`Objects`](crate::syntax::expr::Object). They represent the value part of +/// an argument. +/// ```typst +/// [func: value, key=value] +/// ^^^^^ ^^^^^ +/// ``` +/// +/// Similarly to the [`Key`] trait, this trait has an associated output type +/// which the values are parsed into. Most of the time this is just `Self`, as +/// in the implementation for `bool`: +/// ``` +/// # use typstc::err; +/// # use typstc::error::Error; +/// # use typstc::syntax::expr::Expr; +/// # use typstc::syntax::func::Value; +/// # use typstc::syntax::span::Spanned; +/// # struct Bool; /* +/// impl Value for bool { +/// # */ impl Value for Bool { +/// # type Output = bool; /* +/// type Output = Self; +/// # */ +/// +/// fn parse(expr: Spanned<Expr>) -> Result<Self::Output, Error> { +/// match expr.v { +/// Expr::Bool(b) => Ok(b), +/// other => Err(err!("expected bool, found {}", other.name())), +/// } +/// } +/// } +/// ``` +/// +/// However, sometimes the `Output` type is not just `Self`. For example, there +/// is a value called `Defaultable<V>` which acts as follows: +/// ``` +/// # use typstc::syntax::func::{FuncArgs, Defaultable}; +/// # use typstc::size::Size; +/// # let mut args = FuncArgs::new(); +/// # let mut errors = vec![]; +/// args.key.get::<Defaultable<Size>>(&mut errors, "size"); +/// ``` +/// This will yield. +/// ```typst +/// [func: size=2cm] => Some(Size::cm(2.0)) +/// [func: size=default] => None +/// ``` +/// +/// The type `Defaultable` has no fields and is only used for extracting the +/// option value. This prevents us from having a `Defaultable<V>` type which is +/// essentially simply a bad [`Option`] replacement without the good utility +/// functions. pub trait Value { + /// The type to parse into. type Output; + /// Parse an expression into this value or return an error if the expression + /// is valid for this value type. fn parse(expr: Spanned<Expr>) -> Result<Self::Output, Error>; } @@ -25,6 +83,7 @@ impl<V: Value> Value for Spanned<V> { } } +/// Implements [`Value`] for types that just need to match on expressions. macro_rules! value { ($type:ty, $output:ty, $name:expr, $($p:pat => $r:expr),* $(,)?) => { impl Value for $type { @@ -57,6 +116,8 @@ value!(ScaleSize, Self, "number or size", Expr::Number(scale) => ScaleSize::Scaled(scale as f32), ); +/// A value type that matches [`Expr::Ident`] and [`Expr::Str`] and returns a +/// String. pub struct StringLike; value!(StringLike, String, "identifier or string", @@ -64,15 +125,18 @@ value!(StringLike, String, "identifier or string", Expr::Str(s) => s, ); -pub struct Defaultable<T>(PhantomData<T>); +/// A value type that matches the string `"default"` or a value type `V` and +/// returns `Option::Some(V::Output)` for a value and `Option::None` for +/// `"default"`. +pub struct Defaultable<V>(PhantomData<V>); -impl<T: Value> Value for Defaultable<T> { - type Output = Option<T::Output>; +impl<V: Value> Value for Defaultable<V> { + type Output = Option<V::Output>; fn parse(expr: Spanned<Expr>) -> Result<Self::Output, Error> { match expr.v { Expr::Ident(ident) if ident.as_str() == "default" => Ok(None), - _ => T::parse(expr).map(Some) + _ => V::parse(expr).map(Some) } } } @@ -135,8 +199,12 @@ impl Value for Direction { } } +/// A value type that matches identifiers that are valid alignments like +/// `origin` or `right`. #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] +#[allow(missing_docs)] pub enum AlignmentValue { + /// A generic alignment. Align(Alignment), Left, Top, @@ -145,26 +213,26 @@ pub enum AlignmentValue { } impl AlignmentValue { - /// The generic axis this alignment corresponds to in the given system of - /// layouting axes. `None` if the alignment is generic. - pub fn axis(self, axes: LayoutAxes) -> Option<GenericAxis> { + /// The specific axis this alignment corresponds to. `None` if the alignment + /// is generic. + pub fn axis(self) -> Option<SpecificAxis> { match self { - Left | Right => Some(Horizontal.to_generic(axes)), - Top | Bottom => Some(Vertical.to_generic(axes)), + Left | Right => Some(Horizontal), + Top | Bottom => Some(Vertical), Align(_) => None, } } - /// The generic version of this alignment in the given system of layouting - /// axes. + /// The generic version of this alignment on the given axis in the given + /// system of layouting axes. /// /// Returns `None` if the alignment is invalid for the given axis. pub fn to_generic(self, axes: LayoutAxes, axis: GenericAxis) -> Option<Alignment> { let specific = axis.to_specific(axes); - let start = match axes.get(axis).is_positive() { - true => Origin, - false => End, - }; + let positive = axes.get(axis).is_positive(); + + // The alignment matching the origin of the positive coordinate direction. + let start = if positive { Origin } else { End }; match (self, specific) { (Align(alignment), _) => Some(alignment), @@ -174,10 +242,10 @@ impl AlignmentValue { } } - /// The specific version of this alignment in the given system of layouting - /// axes. - pub fn to_specific(self, axes: LayoutAxes, axis: SpecificAxis) -> AlignmentValue { - let direction = axes.get_specific(axis); + /// The specific version of this alignment on the given axis in the given + /// system of layouting axes. + pub fn to_specific(self, axes: LayoutAxes, axis: GenericAxis) -> AlignmentValue { + let direction = axes.get(axis); if let Align(alignment) = self { match (direction, alignment) { (LeftToRight, Origin) | (RightToLeft, End) => Left, diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs index d5afbca6..4430f6e8 100644 --- a/src/syntax/mod.rs +++ b/src/syntax/mod.rs @@ -1,4 +1,4 @@ -//! Tokenization and parsing of source code. +//! Syntax models, parsing and tokenization. use std::any::Any; use std::fmt::Debug; @@ -17,14 +17,19 @@ pub_use_mod!(parsing); pub_use_mod!(tokens); +/// Represents a parsed piece of source that can be layouted and in the future +/// also be queried for information used for refactorings, autocomplete, etc. #[async_trait(?Send)] pub trait Model: Debug + ModelBounds { + /// Layout the model into a sequence of commands processed by a + /// [`ModelLayouter`](crate::layout::ModelLayouter). async fn layout<'a>(&'a self, ctx: LayoutContext<'_, '_>) -> Layouted<Commands<'a>>; } /// A tree representation of source code. #[derive(Debug, Clone, PartialEq)] pub struct SyntaxModel { + /// The syntactical elements making up this model. pub nodes: SpanVec<Node>, } @@ -50,22 +55,22 @@ impl Model for SyntaxModel { } } -/// A node in the syntax tree. +/// A node in the [syntax model](SyntaxModel). #[derive(Debug, Clone)] pub enum Node { - /// A number of whitespace characters containing less than two newlines. + /// Whitespace containing less than two newlines. Space, - /// Whitespace characters with more than two newlines. + /// Whitespace with more than two newlines. Newline, /// Plain text. Text(String), - /// Italics enabled / disabled. + /// Italics were enabled / disabled. ToggleItalic, - /// Bolder enabled / disabled. + /// Bolder was enabled / disabled. ToggleBolder, - /// Monospace enabled / disabled. + /// Monospace was enabled / disabled. ToggleMonospace, - /// A submodel. + /// A submodel, typically a function invocation. Model(Box<dyn Model>), } @@ -85,15 +90,34 @@ impl PartialEq for Node { } } +/// Decorations for semantic syntax highlighting. #[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub enum Decoration { + /// A valid function name: + /// ```typst + /// [box] + /// ^^^ + /// ``` ValidFuncName, + + /// An invalid function name: + /// ```typst + /// [blabla] + /// ^^^^^^ + /// ``` InvalidFuncName, + + /// The key of a keyword argument: + /// ```typst + /// [box: width=5cm] + /// ^^^^^ + /// ``` ArgumentKey, } impl dyn Model { + /// Downcast this model to a concrete type implementing [`Model`]. pub fn downcast<T>(&self) -> Option<&T> where T: Model + 'static { self.as_any().downcast_ref::<T>() } @@ -111,9 +135,19 @@ impl Clone for Box<dyn Model> { } } +/// This trait describes bounds necessary for types implementing [`Model`]. It is +/// automatically implemented for all types that are [`Model`], [`PartialEq`], +/// [`Clone`] and `'static`. +/// +/// It is necessary to make models comparable and clonable. pub trait ModelBounds { + /// Convert into a `dyn Any`. fn as_any(&self) -> &dyn Any; + + /// Check for equality with another model. fn bound_eq(&self, other: &dyn Model) -> bool; + + /// Clone into a boxed model trait object. fn bound_clone(&self) -> Box<dyn Model>; } diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs index 3e3e827f..9d128a46 100644 --- a/src/syntax/parsing.rs +++ b/src/syntax/parsing.rs @@ -1,3 +1,5 @@ +//! Parsing of source code into syntax models. + use crate::error::Errors; use super::expr::*; use super::func::{FuncHeader, FuncArgs, FuncArg}; @@ -14,13 +16,19 @@ pub struct ParseContext<'a> { pub scope: &'a Scope, } +/// The result of parsing: Some parsed thing, errors and decorations for syntax +/// highlighting. pub struct Parsed<T> { + /// The result of the parsing process. pub output: T, + /// Errors that arose in the parsing process. pub errors: Errors, + /// Decorations for semantic syntax highlighting. pub decorations: SpanVec<Decoration>, } impl<T> Parsed<T> { + /// Map the output type and keep errors and decorations. pub fn map<F, U>(self, f: F) -> Parsed<U> where F: FnOnce(T) -> U { Parsed { output: f(self.output), @@ -30,17 +38,24 @@ impl<T> Parsed<T> { } } +/// Parse source code into a syntax model. +/// +/// All errors and decorations are offset by the `start` position. pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed<SyntaxModel> { let mut model = SyntaxModel::new(); let mut errors = Vec::new(); let mut decorations = Vec::new(); + // We always start in body mode. The header tokenization mode is only used + // in the `FuncParser`. let mut tokens = Tokens::new(start, src, TokenizationMode::Body); while let Some(token) = tokens.next() { let span = token.span; let node = match token.v { + // Only at least two newlines mean a _real_ newline indicating a + // paragraph break. Token::Space(newlines) => if newlines >= 2 { Node::Newline } else { @@ -50,6 +65,9 @@ pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed<SyntaxMode Token::Function { header, body, terminated } => { let parsed: Parsed<Node> = FuncParser::new(header, body, ctx).parse(); + // Collect the errors and decorations from the function parsing, + // but offset their spans by the start of the function since + // they are function-local. errors.extend(offset_spans(parsed.errors, span.start)); decorations.extend(offset_spans(parsed.decorations, span.start)); @@ -79,16 +97,30 @@ pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed<SyntaxMode Parsed { output: model, errors, decorations } } +/// Performs the function parsing. struct FuncParser<'s> { ctx: ParseContext<'s>, errors: Errors, decorations: SpanVec<Decoration>, + + /// ```typst + /// [tokens][body] + /// ^^^^^^ + /// ``` tokens: Tokens<'s>, peeked: Option<Option<Spanned<Token<'s>>>>, + + /// The spanned body string if there is a body. The string itself is just + /// the parsed without the brackets, while the span includes the brackets. + /// ```typst + /// [tokens][body] + /// ^^^^^^ + /// ``` body: Option<Spanned<&'s str>>, } impl<'s> FuncParser<'s> { + /// Create a new function parser. fn new( header: &'s str, body: Option<Spanned<&'s str>>, @@ -104,11 +136,15 @@ impl<'s> FuncParser<'s> { } } + /// Do the parsing. fn parse(mut self) -> Parsed<Node> { let parsed = if let Some(header) = self.parse_func_header() { let name = header.name.v.as_str(); let (parser, deco) = match self.ctx.scope.get_parser(name) { + // A valid function. Ok(parser) => (parser, Decoration::ValidFuncName), + + // The fallback parser was returned. Invalid function. Err(parser) => { self.errors.push(err!(header.name.span; "unknown function")); (parser, Decoration::InvalidFuncName) @@ -139,6 +175,7 @@ impl<'s> FuncParser<'s> { } } + /// Parse the header tokens. fn parse_func_header(&mut self) -> Option<FuncHeader> { let start = self.pos(); self.skip_whitespace(); @@ -166,6 +203,7 @@ impl<'s> FuncParser<'s> { Some(FuncHeader { name, args }) } + /// Parse the function arguments after a colon. fn parse_func_args(&mut self) -> FuncArgs { let mut args = FuncArgs::new(); @@ -226,7 +264,7 @@ impl<'s> FuncParser<'s> { arg } - /// Parse a atomic or compound (tuple / object) expression. + /// Parse an atomic or compound (tuple / object) expression. fn parse_expr(&mut self) -> Option<Spanned<Expr>> { let first = self.peek()?; let spanned = |v| Spanned { v, span: first.span }; @@ -301,7 +339,8 @@ impl<'s> FuncParser<'s> { self.errors.push(err!(Span::at(pos); "expected {}", thing)); } - /// Add a found-error if `found` is some and a positional error, otherwise. + /// Add a expected-found-error if `found` is `Some` and an expected-error + /// otherwise. fn expected_found_or_at( &mut self, thing: &str, @@ -315,7 +354,7 @@ impl<'s> FuncParser<'s> { } /// Consume tokens until the function returns true and only consume the last - /// token if instructed to. + /// token if instructed to so by `eat_match`. fn eat_until<F>(&mut self, mut f: F, eat_match: bool) where F: FnMut(Token<'s>) -> bool { while let Some(token) = self.peek() { @@ -342,11 +381,12 @@ impl<'s> FuncParser<'s> { *self.peeked.get_or_insert_with(|| iter.next()) } + /// Peek at the unspanned value of the next token. fn peekv(&mut self) -> Option<Token<'s>> { self.peek().map(Spanned::value) } - /// The position at the end of the last eat token / start of the peekable + /// The position at the end of the last eaten token / start of the peekable /// token. fn pos(&self) -> Position { self.peeked.flatten() diff --git a/src/syntax/scope.rs b/src/syntax/scope.rs index 2aae331d..895ee498 100644 --- a/src/syntax/scope.rs +++ b/src/syntax/scope.rs @@ -1,3 +1,5 @@ +//! Scopes containing function parsers. + use std::collections::HashMap; use std::fmt::{self, Debug, Formatter}; diff --git a/src/syntax/span.rs b/src/syntax/span.rs index f5bd4caf..ad1358cf 100644 --- a/src/syntax/span.rs +++ b/src/syntax/span.rs @@ -5,18 +5,20 @@ use std::ops::{Add, Sub}; use serde::Serialize; -/// A line-column position in source code. +/// Zero-indexed line-column position in source code. #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize)] pub struct Position { - /// The 0-indexed line (inclusive). + /// The zero-indexed line. pub line: usize, - /// The 0-indexed column (inclusive). + /// The zero-indexed column. pub column: usize, } impl Position { + /// The line 0, column 0 position. pub const ZERO: Position = Position { line: 0, column: 0 }; + /// Crete a new instance from line and column. pub fn new(line: usize, column: usize) -> Position { Position { line, column } } @@ -58,20 +60,25 @@ impl Sub for Position { } } -/// Describes a slice of source code. +/// Locates a slice of source code. #[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)] pub struct Span { + /// The inclusive start position. pub start: Position, + /// The inclusive end position. pub end: Position, } impl Span { + /// A dummy span. pub const ZERO: Span = Span { start: Position::ZERO, end: Position::ZERO }; + /// Create a new span from start and end positions. pub fn new(start: Position, end: Position) -> Span { Span { start, end } } + /// Create a new span with the earlier start and later end position. pub fn merge(a: Span, b: Span) -> Span { Span { start: a.start.min(b.start), @@ -79,14 +86,15 @@ impl Span { } } + /// Create a span including just a single position. pub fn at(pos: Position) -> Span { Span { start: pos, end: pos } } - pub fn expand(&mut self, other: Span) { - *self = Span::merge(*self, other) - } - + /// Offset a span by a start position. + /// + /// This is, for example, used to translate error spans from function local + /// to global. pub fn offset(self, start: Position) -> Span { Span { start: start + self.start, @@ -95,26 +103,32 @@ impl Span { } } -/// Annotates a value with the part of the source code it corresponds to. +/// A value with the span it corresponds to in the source code. #[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)] pub struct Spanned<T> { + /// The value. pub v: T, + /// The corresponding span. pub span: Span, } impl<T> Spanned<T> { + /// Create a new instance from a value and its span. pub fn new(v: T, span: Span) -> Spanned<T> { Spanned { v, span } } + /// Access the value. pub fn value(self) -> T { self.v } + /// Map the value using a function while keeping the span. pub fn map<V, F>(self, f: F) -> Spanned<V> where F: FnOnce(T) -> V { Spanned { v: f(self.v), span: self.span } } + /// Maps the span while keeping the value. pub fn map_span<F>(mut self, f: F) -> Spanned<T> where F: FnOnce(Span) -> Span { self.span = f(self.span); self @@ -124,6 +138,8 @@ impl<T> Spanned<T> { /// A vector of spanned things. pub type SpanVec<T> = Vec<Spanned<T>>; +/// [Offset](Span::offset) all spans in a vector of spanned things by a start +/// position. pub fn offset_spans<T>(vec: SpanVec<T>, start: Position) -> impl Iterator<Item=Spanned<T>> { vec.into_iter().map(move |s| s.map_span(|span| span.offset(start))) } diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs index 0a8e2f17..40d2a526 100644 --- a/src/syntax/tokens.rs +++ b/src/syntax/tokens.rs @@ -16,16 +16,31 @@ pub enum Token<'s> { /// number of newlines that were contained in the whitespace. Space(usize), - /// A line comment with inner string contents `//<&'s str>\n`. + /// A line comment with inner string contents `//<str>\n`. LineComment(&'s str), - /// A block comment with inner string contents `/*<&'s str>*/`. The comment + /// A block comment with inner string contents `/*<str>*/`. The comment /// can contain nested block comments. BlockComment(&'s str), - /// A function invocation `[<header>][<body>]`. + /// A function invocation. Function { + /// The header string: + /// ```typst + /// [header: args][body] + /// ^^^^^^^^^^^^ + /// ``` header: &'s str, + /// The spanned body string: + /// ```typst + /// [header][hello *world*] + /// ^^^^^^^^^^^^^ + /// ``` + /// + /// The span includes the brackets while the string does not. body: Option<Spanned<&'s str>>, + /// Whether the last closing bracket was present. + /// - `[func]` or `[func][body]` => terminated + /// - `[func` or `[func][body` => not terminated terminated: bool, }, @@ -48,7 +63,12 @@ pub enum Token<'s> { /// An identifier in a function header: `center`. ExprIdent(&'s str), /// A quoted string in a function header: `"..."`. - ExprStr { string: &'s str, terminated: bool }, + ExprStr { + /// The string inside the quotes. + string: &'s str, + /// Whether the closing quote was present. + terminated: bool + }, /// A number in a function header: `3.14`. ExprNumber(f64), /// A size in a function header: `12pt`. @@ -110,13 +130,19 @@ pub struct Tokens<'s> { index: usize, } +/// Whether to tokenize in header mode which yields expression, comma and +/// similar tokens or in body mode which yields text and star, underscore, +/// backtick tokens. #[derive(Debug, Copy, Clone, Eq, PartialEq)] +#[allow(missing_docs)] pub enum TokenizationMode { Header, Body, } impl<'s> Tokens<'s> { + /// Create a new token iterator with the given mode where the first token + /// span starts an the given `start` position. pub fn new(start: Position, src: &'s str, mode: TokenizationMode) -> Tokens<'s> { Tokens { src, |
