From 20fb4e7c379b79b84d9884d5f2c89d781c5793e2 Mon Sep 17 00:00:00 2001
From: Laurenz <laurmaedje@gmail.com>
Date: Sun, 26 Jan 2020 15:51:13 +0100
Subject: =?UTF-8?q?Document=20everything=20=F0=9F=93=9C?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/syntax/expr.rs        |  88 ++++++++++++++++++++++++++++++++++----
 src/syntax/func/keys.rs   |  62 +++++++++++++++++++++++++--
 src/syntax/func/maps.rs   |  65 ++++++++++++++++------------
 src/syntax/func/mod.rs    |  56 +++++++++++++-----------
 src/syntax/func/values.rs | 106 +++++++++++++++++++++++++++++++++++++---------
 src/syntax/mod.rs         |  50 ++++++++++++++++++----
 src/syntax/parsing.rs     |  48 +++++++++++++++++++--
 src/syntax/scope.rs       |   2 +
 src/syntax/span.rs        |  34 +++++++++++----
 src/syntax/tokens.rs      |  34 +++++++++++++--
 10 files changed, 439 insertions(+), 106 deletions(-)

(limited to 'src/syntax')

diff --git a/src/syntax/expr.rs b/src/syntax/expr.rs
index b4c0dfaa..879e5fae 100644
--- a/src/syntax/expr.rs
+++ b/src/syntax/expr.rs
@@ -1,8 +1,10 @@
+//! Expressions in function headers.
+
 use std::fmt::{self, Display, Formatter};
 
 use crate::error::Errors;
 use crate::size::Size;
-use super::func::{keys::Key, values::Value};
+use super::func::{Key, Value};
 use super::span::{Span, Spanned};
 use super::tokens::is_identifier;
 
@@ -10,16 +12,24 @@ use super::tokens::is_identifier;
 /// An argument or return value.
 #[derive(Clone, PartialEq)]
 pub enum Expr {
+    /// An identifier: `ident`.
     Ident(Ident),
+    /// A string: `"string"`.
     Str(String),
+    /// A number: `1.2, 200%`.
     Number(f64),
+    /// A size: `2cm, 5.2in`.
     Size(Size),
+    /// A bool: `true, false`.
     Bool(bool),
+    /// A tuple: `(false, 12cm, "hi")`.
     Tuple(Tuple),
+    /// An object: `{ fit: false, size: 12pt }`.
     Object(Object),
 }
 
 impl Expr {
+    /// A natural-language name of the type of this expression, e.g. "identifier".
     pub fn name(&self) -> &'static str {
         use Expr::*;
         match self {
@@ -34,11 +44,21 @@ impl Expr {
     }
 }
 
-/// An identifier.
+/// A unicode identifier.
+///
+/// The identifier must be valid! This is checked in [`Ident::new`] or
+/// [`is_identifier`].
+///
+/// # Example
+/// ```typst
+/// [func: "hi", ident]
+///  ^^^^        ^^^^^
+/// ```
 #[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
 pub struct Ident(pub String);
 
 impl Ident {
+    /// Create a new identifier from a string checking that it is valid.
     pub fn new<S>(ident: S) -> Option<Ident> where S: AsRef<str> + Into<String> {
         if is_identifier(ident.as_ref()) {
             Some(Ident(ident.into()))
@@ -47,26 +67,37 @@ impl Ident {
         }
     }
 
+    /// Return a reference to the underlying string.
     pub fn as_str(&self) -> &str {
         self.0.as_str()
     }
 }
 
-/// A sequence of expressions.
+/// An untyped sequence of expressions.
+///
+/// # Example
+/// ```typst
+/// (false, 12cm, "hi")
+/// ```
 #[derive(Clone, PartialEq)]
 pub struct Tuple {
+    /// The elements of the tuple.
     pub items: Vec<Spanned<Expr>>,
 }
 
 impl Tuple {
+    /// Create an empty tuple.
     pub fn new() -> Tuple {
         Tuple { items: vec![] }
     }
 
+    /// Add an element.
     pub fn add(&mut self, item: Spanned<Expr>) {
         self.items.push(item);
     }
 
+    /// Extract (and remove) the first matching value and remove and generate
+    /// errors for all previous items that did not match.
     pub fn get<V: Value>(&mut self, errors: &mut Errors) -> Option<V::Output> {
         while !self.items.is_empty() {
             let expr = self.items.remove(0);
@@ -79,6 +110,8 @@ impl Tuple {
         None
     }
 
+    /// Extract and return an iterator over all values that match and generate
+    /// errors for all items that do not match.
     pub fn get_all<'a, V: Value>(&'a mut self, errors: &'a mut Errors)
     -> impl Iterator<Item=V::Output> + 'a {
         self.items.drain(..).filter_map(move |expr| {
@@ -92,36 +125,63 @@ impl Tuple {
 }
 
 /// A key-value collection of identifiers and associated expressions.
+///
+/// The pairs themselves are not spanned, but the combined spans can easily be
+/// retrieved by merging the spans of key and value as happening in
+/// [`FuncArg::span`](super::func::FuncArg::span).
+///
+/// # Example
+/// ```typst
+/// { fit: false, size: 12cm, items: (1, 2, 3) }
+/// ```
 #[derive(Clone, PartialEq)]
 pub struct Object {
+    /// The key-value pairs of the object.
     pub pairs: Vec<Pair>,
 }
 
 /// A key-value pair in an object.
 #[derive(Clone, PartialEq)]
 pub struct Pair {
+    /// The key part.
+    /// ```typst
+    /// key: value
+    /// ^^^
+    /// ```
     pub key: Spanned<Ident>,
+    /// The value part.
+    /// ```typst
+    /// key: value
+    ///      ^^^^^
+    /// ```
     pub value: Spanned<Expr>,
 }
 
 impl Object {
+    /// Create an empty object.
     pub fn new() -> Object {
         Object { pairs: vec![] }
     }
 
-    pub fn add(&mut self, key: Spanned<Ident>, value: Spanned<Expr>) {
-        self.pairs.push(Pair { key, value });
-    }
-
-    pub fn add_pair(&mut self, pair: Pair) {
+    /// Add a pair to object.
+    pub fn add(&mut self, pair: Pair) {
         self.pairs.push(pair);
     }
 
+    /// Extract (and remove) a pair with the given key string and matching
+    /// value.
+    ///
+    /// Inserts an error if the value does not match. If the key is not
+    /// contained, no error is inserted.
     pub fn get<V: Value>(&mut self, errors: &mut Errors, key: &str) -> Option<V::Output> {
         let index = self.pairs.iter().position(|pair| pair.key.v.as_str() == key)?;
         self.get_index::<V>(errors, index)
     }
 
+    /// Extract (and remove) a pair with a matching key and value.
+    ///
+    /// Inserts an error if the value does not match. If no matching key is
+    /// found, no error is inserted.
     pub fn get_with_key<K: Key, V: Value>(
         &mut self,
         errors: &mut Errors,
@@ -135,6 +195,9 @@ impl Object {
         None
     }
 
+    /// Extract (and remove) all pairs with matching keys and values.
+    ///
+    /// Inserts errors for values that do not match.
     pub fn get_all<'a, K: Key, V: Value>(
         &'a mut self,
         errors: &'a mut Errors,
@@ -157,6 +220,13 @@ impl Object {
         }).filter_map(|x| x)
     }
 
+    /// Extract all key value pairs with span information.
+    ///
+    /// The spans are over both key and value, like so:
+    /// ```typst
+    /// { key: value }
+    ///   ^^^^^^^^^^
+    /// ```
     pub fn get_all_spanned<'a, K: Key + 'a, V: Value + 'a>(
         &'a mut self,
         errors: &'a mut Errors,
@@ -165,6 +235,8 @@ impl Object {
             .map(|(k, v)| Spanned::new((k.v, v.v), Span::merge(k.span, v.span)))
     }
 
+    /// Extract the argument at the given index and insert an error if the value
+    /// does not match.
     fn get_index<V: Value>(&mut self, errors: &mut Errors, index: usize) -> Option<V::Output> {
         let expr = self.pairs.remove(index).value;
         let span = expr.span;
diff --git a/src/syntax/func/keys.rs b/src/syntax/func/keys.rs
index 116cd4e6..d77447c8 100644
--- a/src/syntax/func/keys.rs
+++ b/src/syntax/func/keys.rs
@@ -1,3 +1,5 @@
+//! Key types for identifying keyword arguments.
+
 use crate::layout::prelude::*;
 use super::values::AlignmentValue::{self, *};
 use super::*;
@@ -6,10 +8,55 @@ use self::AxisKey::*;
 use self::PaddingKey::*;
 
 
-
+/// Key types are used to extract keyword arguments from
+/// [`Objects`](crate::syntax::expr::Object). They represent the key part of a
+/// keyword argument.
+/// ```typst
+/// [func: key=value]
+///        ^^^
+/// ```
+///
+/// A key type has an associated output type, which is returned when parsing
+/// this key from a string. Most of the time, the output type is simply the key
+/// itself, as in the implementation for the [`AxisKey`]:
+/// ```
+/// # use typstc::syntax::func::Key;
+/// # use typstc::syntax::span::Spanned;
+/// # #[derive(Eq, PartialEq)] enum Axis { Horizontal, Vertical, Primary, Secondary }
+/// # #[derive(Eq, PartialEq)] enum AxisKey { Specific(Axis), Generic(Axis) }
+/// # use Axis::*;
+/// # use AxisKey::*;
+/// impl Key for AxisKey {
+///     type Output = Self;
+///
+///     fn parse(key: Spanned<&str>) -> Option<Self::Output> {
+///         match key.v {
+///             "horizontal" | "h" => Some(Specific(Horizontal)),
+///             "vertical"   | "v" => Some(Specific(Vertical)),
+///             "primary"    | "p" => Some(Generic(Primary)),
+///             "secondary"  | "s" => Some(Generic(Secondary)),
+///             _ => None,
+///         }
+///     }
+/// }
+/// ```
+///
+/// The axis key would also be useful to identify axes when describing
+/// dimensions of objects, as in `width=3cm`, because these are also properties
+/// that are stored per axis. However, here the used keyword arguments are
+/// actually different (`width` instead of `horizontal`)! Therefore we cannot
+/// just use the axis key.
+///
+/// To fix this, there is another type [`ExtentKey`] which implements `Key` and
+/// has the associated output type axis key. The extent key struct itself has no
+/// fields and is only used to extract the axis key. This way, we can specify
+/// which argument kind we want without duplicating the type in the background.
 pub trait Key {
+    /// The type to parse into.
     type Output: Eq;
 
+    /// Parse a key string into the output type if the string is valid for this
+    /// key.
     fn parse(key: Spanned<&str>) -> Option<Self::Output>;
 }
 
@@ -21,6 +68,7 @@ impl<K: Key> Key for Spanned<K> {
     }
 }
 
+/// Implements [`Key`] for types that just need to match on strings.
 macro_rules! key {
     ($type:ty, $output:ty, $($($p:pat)|* => $r:expr),* $(,)?) => {
         impl Key for $type {
@@ -36,8 +84,9 @@ macro_rules! key {
     };
 }
 
-/// An argument key which identifies a layouting axis.
+/// A key which identifies a layouting axis.
 #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
+#[allow(missing_docs)]
 pub enum AxisKey {
     Generic(GenericAxis),
     Specific(SpecificAxis),
@@ -68,6 +117,8 @@ key!(AxisKey, Self,
     "secondary"  | "s" => Generic(Secondary),
 );
 
+/// A key which parses into an [`AxisKey`] but uses typical extent keywords
+/// instead of axis keywords, e.g. `width` instead of `horizontal`.
 pub struct ExtentKey;
 
 key!(ExtentKey, AxisKey,
@@ -77,8 +128,13 @@ key!(ExtentKey, AxisKey,
     "secondary-size" | "ss" => Generic(Secondary),
 );
 
-/// An argument key which identifies an axis, but allows for positional
+/// A key which identifies an axis, but alternatively allows for two positional
 /// arguments with unspecified axes.
+///
+/// This type does not implement `Key` in itself since it cannot be parsed from
+/// a string. Rather, [`AxisKeys`](AxisKey) and positional arguments should be
+/// parsed separately and mapped onto this key, as happening in the
+/// [`PosAxisMap`](super::maps::PosAxisMap).
 #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
 pub enum PosAxisKey {
     /// The first positional argument.
diff --git a/src/syntax/func/maps.rs b/src/syntax/func/maps.rs
index 8941024e..eb4c8394 100644
--- a/src/syntax/func/maps.rs
+++ b/src/syntax/func/maps.rs
@@ -9,40 +9,46 @@ use super::values::*;
 use super::*;
 
 
-/// A deduplicating map type useful for storing possibly redundant arguments.
+/// A map which deduplicates redundant arguments.
+///
+/// Whenever a duplicate argument is inserted into the map, through the
+/// functions `from_iter`, `insert` or `extend` an errors is added to the error
+/// list that needs to be passed to those functions.
+///
+/// All entries need to have span information to enable the error reporting.
 #[derive(Debug, Clone, PartialEq)]
 pub struct DedupMap<K, V> where K: Eq {
     map: Vec<Spanned<(K, V)>>,
 }
 
 impl<K, V> DedupMap<K, V> where K: Eq {
+    /// Create a new deduplicating map.
     pub fn new() -> DedupMap<K, V> {
         DedupMap { map: vec![] }
     }
 
+    /// Create a new map from an iterator of spanned keys and values.
     pub fn from_iter<I>(errors: &mut Errors, iter: I) -> DedupMap<K, V>
     where I: IntoIterator<Item=Spanned<(K, V)>> {
         let mut map = DedupMap::new();
-        for Spanned { v: (key, value), span } in iter.into_iter() {
-            map.insert(errors, key, value, span);
-        }
+        map.extend(errors, iter);
         map
     }
 
-    /// Add a key-value pair.
-    pub fn insert(&mut self, errors: &mut Errors, key: K, value: V, span: Span) {
-        if self.map.iter().any(|e| e.v.0 == key) {
-            errors.push(err!(span; "duplicate argument"));
+    /// Add a spanned key-value pair.
+    pub fn insert(&mut self, errors: &mut Errors, entry: Spanned<(K, V)>) {
+        if self.map.iter().any(|e| e.v.0 == entry.v.0) {
+            errors.push(err!(entry.span; "duplicate argument"));
         } else {
-            self.map.push(Spanned { v: (key, value), span });
+            self.map.push(entry);
         }
     }
 
-    /// Add multiple key-value pairs.
+    /// Add multiple spanned key-value pairs.
     pub fn extend<I>(&mut self, errors: &mut Errors, items: I)
     where I: IntoIterator<Item=Spanned<(K, V)>> {
-        for Spanned { v: (k, v), span } in items.into_iter() {
-            self.insert(errors, k, v, span);
+        for item in items.into_iter() {
+            self.insert(errors, item);
         }
     }
 
@@ -65,16 +71,15 @@ impl<K, V> DedupMap<K, V> where K: Eq {
     }
 
     /// Create a new map where keys and values are mapped to new keys and
-    /// values.
-    ///
-    /// Returns an error if a new key is duplicate.
+    /// values. When the mapping introduces new duplicates, errors are
+    /// generated.
     pub fn dedup<F, K2, V2>(&self, errors: &mut Errors, mut f: F) -> DedupMap<K2, V2>
     where F: FnMut(&K, &V) -> (K2, V2), K2: Eq {
         let mut map = DedupMap::new();
 
         for Spanned { v: (key, value), span } in self.map.iter() {
             let (key, value) = f(key, value);
-            map.insert(errors, key, value, *span);
+            map.insert(errors, Spanned { v: (key, value), span: *span });
         }
 
         map
@@ -86,11 +91,12 @@ impl<K, V> DedupMap<K, V> where K: Eq {
     }
 }
 
-/// A map for storing a value for two axes given by keyword arguments.
+/// A map for storing a value for axes given by keyword arguments.
 #[derive(Debug, Clone, PartialEq)]
 pub struct AxisMap<V>(DedupMap<AxisKey, V>);
 
 impl<V: Clone> AxisMap<V> {
+    /// Parse an axis map from the object.
     pub fn parse<KT: Key<Output=AxisKey>, VT: Value<Output=V>>(
         errors: &mut Errors,
         object: &mut Object,
@@ -105,12 +111,13 @@ impl<V: Clone> AxisMap<V> {
     }
 }
 
-/// A map for extracting values for two axes that are given through two
-/// positional or keyword arguments.
+/// A map for storing values for axes that are given through a combination of
+/// (two) positional and keyword arguments.
 #[derive(Debug, Clone, PartialEq)]
 pub struct PosAxisMap<V>(DedupMap<PosAxisKey, V>);
 
 impl<V: Clone> PosAxisMap<V> {
+    /// Parse a positional/axis map from the function arguments.
     pub fn parse<KT: Key<Output=AxisKey>, VT: Value<Output=V>>(
         errors: &mut Errors,
         args: &mut FuncArgs,
@@ -118,8 +125,8 @@ impl<V: Clone> PosAxisMap<V> {
         let mut map = DedupMap::new();
 
         for &key in &[PosAxisKey::First, PosAxisKey::Second] {
-            if let Some(value) = args.pos.get::<Spanned<VT>>(errors) {
-                map.insert(errors, key, value.v, value.span);
+            if let Some(Spanned { v, span }) = args.pos.get::<Spanned<VT>>(errors) {
+                map.insert(errors, Spanned { v: (key, v), span })
             }
         }
 
@@ -133,7 +140,8 @@ impl<V: Clone> PosAxisMap<V> {
         PosAxisMap(map)
     }
 
-    /// Deduplicate from positional or specific to generic axes.
+    /// Deduplicate from positional arguments and keyword arguments for generic
+    /// or specific axes to just generic axes.
     pub fn dedup<F>(
         &self,
         errors: &mut Errors,
@@ -151,17 +159,19 @@ impl<V: Clone> PosAxisMap<V> {
     }
 }
 
-/// A map for extracting padding for a set of specifications given for all
-/// sides, opposing sides or single sides.
+/// A map for storing padding given for a combination of all sides, opposing
+/// sides or single sides.
 #[derive(Debug, Clone, PartialEq)]
 pub struct PaddingMap(DedupMap<PaddingKey<AxisKey>, Option<PSize>>);
 
 impl PaddingMap {
+    /// Parse a padding map from the function arguments.
     pub fn parse(errors: &mut Errors, args: &mut FuncArgs) -> PaddingMap {
         let mut map = DedupMap::new();
 
-        if let Some(psize) = args.pos.get::<Spanned<Defaultable<PSize>>>(errors) {
-            map.insert(errors, PaddingKey::All, psize.v, psize.span);
+        let all = args.pos.get::<Spanned<Defaultable<PSize>>>(errors);
+        if let Some(Spanned { v, span }) = all {
+            map.insert(errors, Spanned { v: (PaddingKey::All, v), span });
         }
 
         let paddings: Vec<_> = args.key
@@ -187,8 +197,9 @@ impl PaddingMap {
                 All => All,
                 Both(axis) => Both(axis.to_specific(axes)),
                 Side(axis, alignment) => {
+                    let generic = axis.to_generic(axes);
                     let axis = axis.to_specific(axes);
-                    Side(axis, alignment.to_specific(axes, axis))
+                    Side(axis, alignment.to_specific(axes, generic))
                 }
             }, val)
         });
diff --git a/src/syntax/func/mod.rs b/src/syntax/func/mod.rs
index e66b4d6a..0c5b4447 100644
--- a/src/syntax/func/mod.rs
+++ b/src/syntax/func/mod.rs
@@ -1,25 +1,38 @@
+//! Primitives for argument parsing in library functions.
+
 use crate::error::{Error, Errors};
 use super::expr::{Expr, Ident, Tuple, Object, Pair};
 use super::span::{Span, Spanned};
 
-pub mod maps;
-pub mod keys;
-pub mod values;
+pub_use_mod!(maps);
+pub_use_mod!(keys);
+pub_use_mod!(values);
 
 
+/// The parsed header of a function.
 #[derive(Debug, Clone, PartialEq)]
 pub struct FuncHeader {
+    /// The function name, that is:
+    /// ```typst
+    /// [box: w=5cm]
+    ///  ^^^
+    /// ```
     pub name: Spanned<Ident>,
+    /// The arguments passed to the function.
     pub args: FuncArgs,
 }
 
+/// The positional and keyword arguments passed to a function.
 #[derive(Debug, Clone, PartialEq)]
 pub struct FuncArgs {
+    /// The positional arguments.
     pub pos: Tuple,
+    /// They keyword arguments.
     pub key: Object,
 }
 
 impl FuncArgs {
+    /// Create new empty function arguments.
     pub fn new() -> FuncArgs {
         FuncArgs {
             pos: Tuple::new(),
@@ -30,40 +43,32 @@ impl FuncArgs {
     /// Add an argument.
     pub fn add(&mut self, arg: FuncArg) {
         match arg {
-            FuncArg::Pos(item) => self.add_pos(item),
-            FuncArg::Key(pair) => self.add_key_pair(pair),
+            FuncArg::Pos(item) => self.pos.add(item),
+            FuncArg::Key(pair) => self.key.add(pair),
         }
     }
 
-    /// Add a positional argument.
-    pub fn add_pos(&mut self, item: Spanned<Expr>) {
-        self.pos.add(item);
-    }
-
-    /// Add a keyword argument.
-    pub fn add_key(&mut self, key: Spanned<Ident>, value: Spanned<Expr>) {
-        self.key.add(key, value);
-    }
-
-    /// Add a keyword argument from an existing pair.
-    pub fn add_key_pair(&mut self, pair: Pair) {
-        self.key.add_pair(pair);
-    }
-
+    /// Iterate over all arguments.
     pub fn into_iter(self) -> impl Iterator<Item=FuncArg> {
         self.pos.items.into_iter().map(|item| FuncArg::Pos(item))
             .chain(self.key.pairs.into_iter().map(|pair| FuncArg::Key(pair)))
     }
 }
 
+/// Either a positional or keyword argument.
 #[derive(Debug, Clone, PartialEq)]
 pub enum FuncArg {
+    /// A positional argument.
     Pos(Spanned<Expr>),
+    /// A keyword argument.
     Key(Pair),
 }
 
 impl FuncArg {
-    /// The span or the value or combined span of key and value.
+    /// The full span of this argument.
+    ///
+    /// In case of a positional argument this is just the span of the expression
+    /// and in case of a keyword argument the combined span of key and value.
     pub fn span(&self) -> Span {
         match self {
             FuncArg::Pos(item) => item.span,
@@ -72,14 +77,17 @@ impl FuncArg {
     }
 }
 
+/// Extra methods on [`Options`](Option) used for argument parsing.
 pub trait OptionExt: Sized {
-    fn or_missing(self, errors: &mut Errors, span: Span, what: &str) -> Self;
+    /// Add an error about a missing argument `arg` with the given span if the
+    /// option is `None`.
+    fn or_missing(self, errors: &mut Errors, span: Span, arg: &str) -> Self;
 }
 
 impl<T> OptionExt for Option<T> {
-    fn or_missing(self, errors: &mut Errors, span: Span, what: &str) -> Self {
+    fn or_missing(self, errors: &mut Errors, span: Span, arg: &str) -> Self {
         if self.is_none() {
-            errors.push(err!(span; "missing argument: {}", what));
+            errors.push(err!(span; "missing argument: {}", arg));
         }
         self
     }
diff --git a/src/syntax/func/values.rs b/src/syntax/func/values.rs
index a767aef6..515d6a43 100644
--- a/src/syntax/func/values.rs
+++ b/src/syntax/func/values.rs
@@ -1,3 +1,5 @@
+//! Value types for extracting function arguments.
+
 use std::fmt::{self, Display, Formatter};
 use std::marker::PhantomData;
 use toddle::query::{FontStyle, FontWeight};
@@ -10,9 +12,65 @@ use super::*;
 use self::AlignmentValue::*;
 
 
+/// Value types are used to extract the values of positional and keyword
+/// arguments from [`Tuples`](crate::syntax::expr::Tuple) and
+/// [`Objects`](crate::syntax::expr::Object). They represent the value part of
+/// an argument.
+/// ```typst
+/// [func: value, key=value]
+///        ^^^^^      ^^^^^
+/// ```
+///
+/// Similarly to the [`Key`] trait, this trait has an associated output type
+/// which the values are parsed into. Most of the time this is just `Self`, as
+/// in the implementation for `bool`:
+/// ```
+/// # use typstc::err;
+/// # use typstc::error::Error;
+/// # use typstc::syntax::expr::Expr;
+/// # use typstc::syntax::func::Value;
+/// # use typstc::syntax::span::Spanned;
+/// # struct Bool; /*
+/// impl Value for bool {
+/// # */ impl Value for Bool {
+///     # type Output = bool; /*
+///     type Output = Self;
+///     # */
+///
+///     fn parse(expr: Spanned<Expr>) -> Result<Self::Output, Error> {
+///         match expr.v {
+///             Expr::Bool(b) => Ok(b),
+///             other => Err(err!("expected bool, found {}", other.name())),
+///         }
+///     }
+/// }
+/// ```
+///
+/// However, sometimes the `Output` type is not just `Self`. For example, there
+/// is a value called `Defaultable<V>` which acts as follows:
+/// ```
+/// # use typstc::syntax::func::{FuncArgs, Defaultable};
+/// # use typstc::size::Size;
+/// # let mut args = FuncArgs::new();
+/// # let mut errors = vec![];
+/// args.key.get::<Defaultable<Size>>(&mut errors, "size");
+/// ```
+/// This will yield.
+/// ```typst
+/// [func: size=2cm]     => Some(Size::cm(2.0))
+/// [func: size=default] => None
+/// ```
+///
+/// The type `Defaultable` has no fields and is only used for extracting the
+/// option value. This prevents us from having a `Defaultable<V>` type which is
+/// essentially simply a bad [`Option`] replacement without the good utility
+/// functions.
 pub trait Value {
+    /// The type to parse into.
     type Output;
 
+    /// Parse an expression into this value or return an error if the expression
+    /// is valid for this value type.
     fn parse(expr: Spanned<Expr>) -> Result<Self::Output, Error>;
 }
 
@@ -25,6 +83,7 @@ impl<V: Value> Value for Spanned<V> {
     }
 }
 
+/// Implements [`Value`] for types that just need to match on expressions.
 macro_rules! value {
     ($type:ty, $output:ty, $name:expr, $($p:pat => $r:expr),* $(,)?) => {
         impl Value for $type {
@@ -57,6 +116,8 @@ value!(ScaleSize, Self, "number or size",
     Expr::Number(scale) => ScaleSize::Scaled(scale as f32),
 );
 
+/// A value type that matches [`Expr::Ident`] and [`Expr::Str`] and returns a
+/// String.
 pub struct StringLike;
 
 value!(StringLike, String, "identifier or string",
@@ -64,15 +125,18 @@ value!(StringLike, String, "identifier or string",
     Expr::Str(s) => s,
 );
 
-pub struct Defaultable<T>(PhantomData<T>);
+/// A value type that matches the string `"default"` or a value type `V` and
+/// returns `Option::Some(V::Output)` for a value and `Option::None` for
+/// `"default"`.
+pub struct Defaultable<V>(PhantomData<V>);
 
-impl<T: Value> Value for Defaultable<T> {
-    type Output = Option<T::Output>;
+impl<V: Value> Value for Defaultable<V> {
+    type Output = Option<V::Output>;
 
     fn parse(expr: Spanned<Expr>) -> Result<Self::Output, Error> {
         match expr.v {
             Expr::Ident(ident) if ident.as_str() == "default" => Ok(None),
-            _ => T::parse(expr).map(Some)
+            _ => V::parse(expr).map(Some)
         }
     }
 }
@@ -135,8 +199,12 @@ impl Value for Direction {
     }
 }
 
+/// A value type that matches identifiers that are valid alignments like
+/// `origin` or `right`.
 #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
+#[allow(missing_docs)]
 pub enum AlignmentValue {
+    /// A generic alignment.
     Align(Alignment),
     Left,
     Top,
@@ -145,26 +213,26 @@ pub enum AlignmentValue {
 }
 
 impl AlignmentValue {
-    /// The generic axis this alignment corresponds to in the given system of
-    /// layouting axes. `None` if the alignment is generic.
-    pub fn axis(self, axes: LayoutAxes) -> Option<GenericAxis> {
+    /// The specific axis this alignment corresponds to. `None` if the alignment
+    /// is generic.
+    pub fn axis(self) -> Option<SpecificAxis> {
         match self {
-            Left | Right => Some(Horizontal.to_generic(axes)),
-            Top | Bottom => Some(Vertical.to_generic(axes)),
+            Left | Right => Some(Horizontal),
+            Top | Bottom => Some(Vertical),
             Align(_) => None,
         }
     }
 
-    /// The generic version of this alignment in the given system of layouting
-    /// axes.
+    /// The generic version of this alignment on the given axis in the given
+    /// system of layouting axes.
     ///
     /// Returns `None` if the alignment is invalid for the given axis.
     pub fn to_generic(self, axes: LayoutAxes, axis: GenericAxis) -> Option<Alignment> {
         let specific = axis.to_specific(axes);
-        let start = match axes.get(axis).is_positive() {
-            true => Origin,
-            false => End,
-        };
+        let positive = axes.get(axis).is_positive();
+
+        // The alignment matching the origin of the positive coordinate direction.
+        let start = if positive { Origin } else { End };
 
         match (self, specific) {
             (Align(alignment), _) => Some(alignment),
@@ -174,10 +242,10 @@ impl AlignmentValue {
         }
     }
 
-    /// The specific version of this alignment in the given system of layouting
-    /// axes.
-    pub fn to_specific(self, axes: LayoutAxes, axis: SpecificAxis) -> AlignmentValue {
-        let direction = axes.get_specific(axis);
+    /// The specific version of this alignment on the given axis in the given
+    /// system of layouting axes.
+    pub fn to_specific(self, axes: LayoutAxes, axis: GenericAxis) -> AlignmentValue {
+        let direction = axes.get(axis);
         if let Align(alignment) = self {
             match (direction, alignment) {
                 (LeftToRight, Origin) | (RightToLeft, End) => Left,
diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs
index d5afbca6..4430f6e8 100644
--- a/src/syntax/mod.rs
+++ b/src/syntax/mod.rs
@@ -1,4 +1,4 @@
-//! Tokenization and parsing of source code.
+//! Syntax models, parsing and tokenization.
 
 use std::any::Any;
 use std::fmt::Debug;
@@ -17,14 +17,19 @@ pub_use_mod!(parsing);
 pub_use_mod!(tokens);
 
 
+/// Represents a parsed piece of source that can be layouted and in the future
+/// also be queried for information used for refactorings, autocomplete, etc.
 #[async_trait(?Send)]
 pub trait Model: Debug + ModelBounds {
+    /// Layout the model into a sequence of commands processed by a
+    /// [`ModelLayouter`](crate::layout::ModelLayouter).
     async fn layout<'a>(&'a self, ctx: LayoutContext<'_, '_>) -> Layouted<Commands<'a>>;
 }
 
 /// A tree representation of source code.
 #[derive(Debug, Clone, PartialEq)]
 pub struct SyntaxModel {
+    /// The syntactical elements making up this model.
     pub nodes: SpanVec<Node>,
 }
 
@@ -50,22 +55,22 @@ impl Model for SyntaxModel {
     }
 }
 
-/// A node in the syntax tree.
+/// A node in the [syntax model](SyntaxModel).
 #[derive(Debug, Clone)]
 pub enum Node {
-    /// A number of whitespace characters containing less than two newlines.
+    /// Whitespace containing less than two newlines.
     Space,
-    /// Whitespace characters with more than two newlines.
+    /// Whitespace with more than two newlines.
     Newline,
     /// Plain text.
     Text(String),
-    /// Italics enabled / disabled.
+    /// Italics were enabled / disabled.
     ToggleItalic,
-    /// Bolder enabled / disabled.
+    /// Bolder was enabled / disabled.
     ToggleBolder,
-    /// Monospace enabled / disabled.
+    /// Monospace was enabled / disabled.
     ToggleMonospace,
-    /// A submodel.
+    /// A submodel, typically a function invocation.
     Model(Box<dyn Model>),
 }
 
@@ -85,15 +90,34 @@ impl PartialEq for Node {
     }
 }
 
+/// Decorations for semantic syntax highlighting.
 #[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub enum Decoration {
+    /// A valid function name:
+    /// ```typst
+    /// [box]
+    ///  ^^^
+    /// ```
     ValidFuncName,
+
+    /// An invalid function name:
+    /// ```typst
+    /// [blabla]
+    ///  ^^^^^^
+    /// ```
     InvalidFuncName,
+
+    /// The key of a keyword argument:
+    /// ```typst
+    /// [box: width=5cm]
+    ///       ^^^^^
+    /// ```
     ArgumentKey,
 }
 
 impl dyn Model {
+    /// Downcast this model to a concrete type implementing [`Model`].
     pub fn downcast<T>(&self) -> Option<&T> where T: Model + 'static {
         self.as_any().downcast_ref::<T>()
     }
@@ -111,9 +135,19 @@ impl Clone for Box<dyn Model> {
     }
 }
 
+/// This trait describes bounds necessary for types implementing [`Model`]. It is
+/// automatically implemented for all types that are [`Model`], [`PartialEq`],
+/// [`Clone`] and `'static`.
+///
+/// It is necessary to make models comparable and clonable.
 pub trait ModelBounds {
+    /// Convert into a `dyn Any`.
     fn as_any(&self) -> &dyn Any;
+
+    /// Check for equality with another model.
     fn bound_eq(&self, other: &dyn Model) -> bool;
+
+    /// Clone into a boxed model trait object.
     fn bound_clone(&self) -> Box<dyn Model>;
 }
 
diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs
index 3e3e827f..9d128a46 100644
--- a/src/syntax/parsing.rs
+++ b/src/syntax/parsing.rs
@@ -1,3 +1,5 @@
+//! Parsing of source code into syntax models.
+
 use crate::error::Errors;
 use super::expr::*;
 use super::func::{FuncHeader, FuncArgs, FuncArg};
@@ -14,13 +16,19 @@ pub struct ParseContext<'a> {
     pub scope: &'a Scope,
 }
 
+/// The result of parsing: Some parsed thing, errors and decorations for syntax
+/// highlighting.
 pub struct Parsed<T> {
+    /// The result of the parsing process.
     pub output: T,
+    /// Errors that arose in the parsing process.
     pub errors: Errors,
+    /// Decorations for semantic syntax highlighting.
     pub decorations: SpanVec<Decoration>,
 }
 
 impl<T> Parsed<T> {
+    /// Map the output type and keep errors and decorations.
     pub fn map<F, U>(self, f: F) -> Parsed<U> where F: FnOnce(T) -> U {
         Parsed {
             output: f(self.output),
@@ -30,17 +38,24 @@ impl<T> Parsed<T> {
     }
 }
 
+/// Parse source code into a syntax model.
+///
+/// All errors and decorations are offset by the `start` position.
 pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed<SyntaxModel> {
     let mut model = SyntaxModel::new();
     let mut errors = Vec::new();
     let mut decorations = Vec::new();
 
+    // We always start in body mode. The header tokenization mode is only used
+    // in the `FuncParser`.
     let mut tokens = Tokens::new(start, src, TokenizationMode::Body);
 
     while let Some(token) = tokens.next() {
         let span = token.span;
 
         let node = match token.v {
+            // Only at least two newlines mean a _real_ newline indicating a
+            // paragraph break.
             Token::Space(newlines) => if newlines >= 2 {
                 Node::Newline
             } else {
@@ -50,6 +65,9 @@ pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed<SyntaxMode
             Token::Function { header, body, terminated } => {
                 let parsed: Parsed<Node> = FuncParser::new(header, body, ctx).parse();
 
+                // Collect the errors and decorations from the function parsing,
+                // but offset their spans by the start of the function since
+                // they are function-local.
                 errors.extend(offset_spans(parsed.errors, span.start));
                 decorations.extend(offset_spans(parsed.decorations, span.start));
 
@@ -79,16 +97,30 @@ pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed<SyntaxMode
     Parsed { output: model, errors, decorations }
 }
 
+/// Performs the function parsing.
 struct FuncParser<'s> {
     ctx: ParseContext<'s>,
     errors: Errors,
     decorations: SpanVec<Decoration>,
+
+    /// ```typst
+    /// [tokens][body]
+    ///  ^^^^^^
+    /// ```
     tokens: Tokens<'s>,
     peeked: Option<Option<Spanned<Token<'s>>>>,
+
+    /// The spanned body string if there is a body. The string itself is just
+    /// the parsed without the brackets, while the span includes the brackets.
+    /// ```typst
+    /// [tokens][body]
+    ///         ^^^^^^
+    /// ```
     body: Option<Spanned<&'s str>>,
 }
 
 impl<'s> FuncParser<'s> {
+    /// Create a new function parser.
     fn new(
         header: &'s str,
         body: Option<Spanned<&'s str>>,
@@ -104,11 +136,15 @@ impl<'s> FuncParser<'s> {
         }
     }
 
+    /// Do the parsing.
     fn parse(mut self) -> Parsed<Node> {
         let parsed = if let Some(header) = self.parse_func_header() {
             let name = header.name.v.as_str();
             let (parser, deco) = match self.ctx.scope.get_parser(name) {
+                // A valid function.
                 Ok(parser) => (parser, Decoration::ValidFuncName),
+
+                // The fallback parser was returned. Invalid function.
                 Err(parser) => {
                     self.errors.push(err!(header.name.span; "unknown function"));
                     (parser, Decoration::InvalidFuncName)
@@ -139,6 +175,7 @@ impl<'s> FuncParser<'s> {
         }
     }
 
+    /// Parse the header tokens.
     fn parse_func_header(&mut self) -> Option<FuncHeader> {
         let start = self.pos();
         self.skip_whitespace();
@@ -166,6 +203,7 @@ impl<'s> FuncParser<'s> {
         Some(FuncHeader { name, args })
     }
 
+    /// Parse the function arguments after a colon.
     fn parse_func_args(&mut self) -> FuncArgs {
         let mut args = FuncArgs::new();
 
@@ -226,7 +264,7 @@ impl<'s> FuncParser<'s> {
         arg
     }
 
-    /// Parse a atomic or compound (tuple / object) expression.
+    /// Parse an atomic or compound (tuple / object) expression.
     fn parse_expr(&mut self) -> Option<Spanned<Expr>> {
         let first = self.peek()?;
         let spanned = |v| Spanned { v, span: first.span };
@@ -301,7 +339,8 @@ impl<'s> FuncParser<'s> {
         self.errors.push(err!(Span::at(pos); "expected {}", thing));
     }
 
-    /// Add a found-error if `found` is some and a positional error, otherwise.
+    /// Add a expected-found-error if `found` is `Some` and an expected-error
+    /// otherwise.
     fn expected_found_or_at(
         &mut self,
         thing: &str,
@@ -315,7 +354,7 @@ impl<'s> FuncParser<'s> {
     }
 
     /// Consume tokens until the function returns true and only consume the last
-    /// token if instructed to.
+    /// token if instructed to so by `eat_match`.
     fn eat_until<F>(&mut self, mut f: F, eat_match: bool)
     where F: FnMut(Token<'s>) -> bool {
         while let Some(token) = self.peek() {
@@ -342,11 +381,12 @@ impl<'s> FuncParser<'s> {
         *self.peeked.get_or_insert_with(|| iter.next())
     }
 
+    /// Peek at the unspanned value of the next token.
     fn peekv(&mut self) -> Option<Token<'s>> {
         self.peek().map(Spanned::value)
     }
 
-    /// The position at the end of the last eat token / start of the peekable
+    /// The position at the end of the last eaten token / start of the peekable
     /// token.
     fn pos(&self) -> Position {
         self.peeked.flatten()
diff --git a/src/syntax/scope.rs b/src/syntax/scope.rs
index 2aae331d..895ee498 100644
--- a/src/syntax/scope.rs
+++ b/src/syntax/scope.rs
@@ -1,3 +1,5 @@
+//! Scopes containing function parsers.
+
 use std::collections::HashMap;
 use std::fmt::{self, Debug, Formatter};
 
diff --git a/src/syntax/span.rs b/src/syntax/span.rs
index f5bd4caf..ad1358cf 100644
--- a/src/syntax/span.rs
+++ b/src/syntax/span.rs
@@ -5,18 +5,20 @@ use std::ops::{Add, Sub};
 use serde::Serialize;
 
 
-/// A line-column position in source code.
+/// Zero-indexed line-column position in source code.
 #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize)]
 pub struct Position {
-    /// The 0-indexed line (inclusive).
+    /// The zero-indexed line.
     pub line: usize,
-    /// The 0-indexed column (inclusive).
+    /// The zero-indexed column.
     pub column: usize,
 }
 
 impl Position {
+    /// The line 0, column 0 position.
     pub const ZERO: Position = Position { line: 0, column: 0 };
 
+    /// Crete a new instance from line and column.
     pub fn new(line: usize, column: usize) -> Position {
         Position { line, column }
     }
@@ -58,20 +60,25 @@ impl Sub for Position {
     }
 }
 
-/// Describes a slice of source code.
+/// Locates a slice of source code.
 #[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)]
 pub struct Span {
+    /// The inclusive start position.
     pub start: Position,
+    /// The inclusive end position.
     pub end: Position,
 }
 
 impl Span {
+    /// A dummy span.
     pub const ZERO: Span = Span { start: Position::ZERO, end: Position::ZERO };
 
+    /// Create a new span from start and end positions.
     pub fn new(start: Position, end: Position) -> Span {
         Span { start, end }
     }
 
+    /// Create a new span with the earlier start and later end position.
     pub fn merge(a: Span, b: Span) -> Span {
         Span {
             start: a.start.min(b.start),
@@ -79,14 +86,15 @@ impl Span {
         }
     }
 
+    /// Create a span including just a single position.
     pub fn at(pos: Position) -> Span {
         Span { start: pos, end: pos }
     }
 
-    pub fn expand(&mut self, other: Span) {
-        *self = Span::merge(*self, other)
-    }
-
+    /// Offset a span by a start position.
+    ///
+    /// This is, for example, used to translate error spans from function local
+    /// to global.
     pub fn offset(self, start: Position) -> Span {
         Span {
             start: start + self.start,
@@ -95,26 +103,32 @@ impl Span {
     }
 }
 
-/// Annotates a value with the part of the source code it corresponds to.
+/// A value with the span it corresponds to in the source code.
 #[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)]
 pub struct Spanned<T> {
+    /// The value.
     pub v: T,
+    /// The corresponding span.
     pub span: Span,
 }
 
 impl<T> Spanned<T> {
+    /// Create a new instance from a value and its span.
     pub fn new(v: T, span: Span) -> Spanned<T> {
         Spanned { v, span }
     }
 
+    /// Access the value.
     pub fn value(self) -> T {
         self.v
     }
 
+    /// Map the value using a function while keeping the span.
     pub fn map<V, F>(self, f: F) -> Spanned<V> where F: FnOnce(T) -> V {
         Spanned { v: f(self.v), span: self.span }
     }
 
+    /// Maps the span while keeping the value.
     pub fn map_span<F>(mut self, f: F) -> Spanned<T> where F: FnOnce(Span) -> Span {
         self.span = f(self.span);
         self
@@ -124,6 +138,8 @@ impl<T> Spanned<T> {
 /// A vector of spanned things.
 pub type SpanVec<T> = Vec<Spanned<T>>;
 
+/// [Offset](Span::offset) all spans in a vector of spanned things by a start
+/// position.
 pub fn offset_spans<T>(vec: SpanVec<T>, start: Position) -> impl Iterator<Item=Spanned<T>> {
     vec.into_iter().map(move |s| s.map_span(|span| span.offset(start)))
 }
diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs
index 0a8e2f17..40d2a526 100644
--- a/src/syntax/tokens.rs
+++ b/src/syntax/tokens.rs
@@ -16,16 +16,31 @@ pub enum Token<'s> {
     /// number of newlines that were contained in the whitespace.
     Space(usize),
 
-    /// A line comment with inner string contents `//<&'s str>\n`.
+    /// A line comment with inner string contents `//<str>\n`.
     LineComment(&'s str),
-    /// A block comment with inner string contents `/*<&'s str>*/`. The comment
+    /// A block comment with inner string contents `/*<str>*/`. The comment
     /// can contain nested block comments.
     BlockComment(&'s str),
 
-    /// A function invocation `[<header>][<body>]`.
+    /// A function invocation.
     Function {
+        /// The header string:
+        /// ```typst
+        /// [header: args][body]
+        ///  ^^^^^^^^^^^^
+        /// ```
         header: &'s str,
+        /// The spanned body string:
+        /// ```typst
+        /// [header][hello *world*]
+        ///          ^^^^^^^^^^^^^
+        /// ```
+        ///
+        /// The span includes the brackets while the string does not.
         body: Option<Spanned<&'s str>>,
+        /// Whether the last closing bracket was present.
+        /// - `[func]` or `[func][body]` => terminated
+        /// - `[func` or `[func][body` => not terminated
         terminated: bool,
     },
 
@@ -48,7 +63,12 @@ pub enum Token<'s> {
     /// An identifier in a function header: `center`.
     ExprIdent(&'s str),
     /// A quoted string in a function header: `"..."`.
-    ExprStr { string: &'s str, terminated: bool },
+    ExprStr {
+        /// The string inside the quotes.
+        string: &'s str,
+        /// Whether the closing quote was present.
+        terminated: bool
+    },
     /// A number in a function header: `3.14`.
     ExprNumber(f64),
     /// A size in a function header: `12pt`.
@@ -110,13 +130,19 @@ pub struct Tokens<'s> {
     index: usize,
 }
 
+/// Whether to tokenize in header mode which yields expression, comma and
+/// similar tokens or in body mode which yields text and star, underscore,
+/// backtick tokens.
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
+#[allow(missing_docs)]
 pub enum TokenizationMode {
     Header,
     Body,
 }
 
 impl<'s> Tokens<'s> {
+    /// Create a new token iterator with the given mode where the first token
+    /// span starts an the given `start` position.
     pub fn new(start: Position, src: &'s str, mode: TokenizationMode) -> Tokens<'s> {
         Tokens {
             src,
-- 
cgit v1.2.3