diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/error.rs | 10 | ||||
| -rw-r--r-- | src/func/macros.rs | 202 | ||||
| -rw-r--r-- | src/func/mod.rs | 132 | ||||
| -rw-r--r-- | src/layout/line.rs | 72 | ||||
| -rw-r--r-- | src/layout/mod.rs | 42 | ||||
| -rw-r--r-- | src/layout/model.rs | 193 | ||||
| -rw-r--r-- | src/layout/stack.rs | 55 | ||||
| -rw-r--r-- | src/layout/text.rs | 59 | ||||
| -rw-r--r-- | src/layout/tree.rs | 163 | ||||
| -rw-r--r-- | src/lib.rs | 59 | ||||
| -rw-r--r-- | src/syntax/expr.rs | 3 | ||||
| -rw-r--r-- | src/syntax/func.rs | 125 | ||||
| -rw-r--r-- | src/syntax/mod.rs | 382 | ||||
| -rw-r--r-- | src/syntax/parsing.rs | 207 | ||||
| -rw-r--r-- | src/syntax/span.rs | 40 | ||||
| -rw-r--r-- | src/syntax/tokens.rs | 90 |
16 files changed, 894 insertions, 940 deletions
diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 00000000..ed43818f --- /dev/null +++ b/src/error.rs @@ -0,0 +1,10 @@ +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct Error { + pub message: String, +} + +impl Error { + pub fn new(message: impl Into<String>) -> Error { + Error { message: message.into() } + } +} diff --git a/src/func/macros.rs b/src/func/macros.rs index 90c3b11e..1e92d735 100644 --- a/src/func/macros.rs +++ b/src/func/macros.rs @@ -1,131 +1,76 @@ //! Helper types and macros for creating custom functions. -/// Defines function types concisely. #[macro_export] macro_rules! function { - // Parse a unit struct. - ($(#[$outer:meta])* pub struct $type:ident; $($rest:tt)*) => { - $(#[$outer])* pub struct $type; - function!(@meta $type | $($rest)*); + // Entry point. + ($(#[$outer:meta])* $v:vis $storage:ident $name:ident $($r:tt)*) => { + function!(@def($name) $(#[$outer])* $v $storage $name $($r)*); }; - - // Parse a tuple struct. - ($(#[$outer:meta])* pub struct $type:ident($($fields:tt)*); $($rest:tt)*) => { - $(#[$outer])* pub struct $type($($fields)*); - function!(@meta $type | $($rest)*); - }; - - // Parse a struct with fields. - ($(#[$outer:meta])* pub struct $type:ident { $($fields:tt)* } $($rest:tt)*) => { - $(#[$outer])* pub struct $type { $($fields)* } - function!(@meta $type | $($rest)*); - }; - - // Parse an enum. - ($(#[$outer:meta])* pub enum $type:ident { $($fields:tt)* } $($rest:tt)*) => { - $(#[$outer])* pub enum $type { $($fields)* } - function!(@meta $type | $($rest)*); - }; - - // Parse a metadata type definition. - (@meta $type:ident | type Meta = $meta:ty; $($rest:tt)*) => { - function!(@parse $type $meta | $($rest)*); + (@def($name:ident) $definition:item $($r:tt)*) => { + $definition + function!(@meta($name) $($r)*); }; - // Set the metadata to `()` if there is no type definition. - (@meta $type:ident | $($rest:tt)*) => { - function!(@parse $type () | $($rest)*); + // Metadata. + (@meta($name:ident) type Meta = $meta:ty; $($r:tt)*) => { + function!(@parse($name, $meta) $($r)*); }; - - // Parse a `parse(default)`. - (@parse $type:ident $meta:ty | parse(default) $($rest:tt)*) => { - function!(@parse $type $meta | - parse(_args, _body, _ctx, _meta) { Default::default() } - $($rest)* - ); - }; - - // (0-arg) Parse a parse-definition without arguments. - (@parse $type:ident $meta:ty | parse() $code:block $($rest:tt)*) => { - function!(@parse $type $meta | parse(_args, _body, _ctx, _meta) $code $($rest)*); + (@meta($name:ident) $($r:tt)*) => { + function!(@parse($name, ()) $($r)*); }; - // (1-arg) Parse a parse-definition with only the first argument. - (@parse $type:ident $meta:ty | parse($header:ident) $code:block $($rest:tt)*) => { - function!(@parse $type $meta | parse($header, _body, _ctx, _meta) $code $($rest)*); + // Parse trait. + (@parse($($a:tt)*) parse(default) $($r:tt)*) => { + function!(@parse($($a)*) parse() { Default::default() } $($r)*); }; - - // (2-arg) Parse a parse-definition with only the first two arguments. - (@parse $type:ident $meta:ty | - parse($header:ident, $body:pat) $code:block $($rest:tt)* - ) => { - function!(@parse $type $meta | parse($header, $body, _ctx, _meta) $code $($rest)*); + (@parse($($a:tt)*) parse($h:ident, $b:ident, $c:ident, $e:ident, $d:ident) $($r:tt)* ) => { + function!(@parse($($a)*) parse($h, $b, $c, $e, $d, _metadata) $($r)*); }; - - // (3-arg) Parse a parse-definition with only the first three arguments. - (@parse $type:ident $meta:ty | - parse($header:ident, $body:pat, $ctx:pat) $code:block $($rest:tt)* - ) => { - function!(@parse $type $meta | parse($header, $body, $ctx, _meta) $code $($rest)*); - }; - - // (4-arg) Parse a parse-definition with all four arguments. - (@parse $type:ident $meta:ty | - parse($header:ident, $body:pat, $ctx:pat, $metadata:pat) $code:block - $($rest:tt)* - ) => { - impl $crate::func::ParseFunc for $type { + (@parse($name:ident, $meta:ty) parse( + $header:ident, + $body:ident, + $ctx:ident, + $errors:ident, + $decos:ident, + $metadata:ident + ) $code:block $($r:tt)*) => { + impl $crate::func::Parse for $name { type Meta = $meta; fn parse( - header: $crate::syntax::FuncHeader, - $body: Option<&str>, - $ctx: $crate::syntax::ParseContext, - $metadata: Self::Meta, - ) -> $crate::syntax::ParseResult<Self> where Self: Sized { - #[allow(unused_mut)] - let mut $header = header; - let val = $code; - if !$header.args.is_empty() { - return Err($crate::TypesetError::with_message("unexpected arguments")); - } - Ok(val) + #[allow(unused)] mut $header: FuncHeader, + #[allow(unused)] $body: Option<Spanned<&str>>, + #[allow(unused)] $ctx: ParseContext, + #[allow(unused)] $metadata: Self::Meta, + ) -> Parsed<Self> where Self: Sized { + #[allow(unused)] let mut $errors = vec![]; + #[allow(unused)] let mut $decos = vec![]; + let output = $code; + $crate::syntax::Parsed { output, errors: $errors, decorations: $decos } } } - function!(@layout $type | $($rest)*); + function!(@layout($name) $($r)*); }; - // (0-arg) Parse a layout-definition without arguments. - (@layout $type:ident | layout() $code:block) => { - function!(@layout $type | layout(self, _ctx) $code); - }; - - // (1-arg) Parse a layout-definition with only the first argument. - (@layout $type:ident | layout($this:ident) $code:block) => { - function!(@layout $type | layout($this, _ctx) $code); - }; - - // (2-arg) Parse a layout-definition with all arguments. - (@layout $type:ident | layout($this:ident, $ctx:pat) $code:block) => { - impl $crate::func::LayoutFunc for $type { - fn layout<'a, 'life0, 'life1, 'async_trait>( - &'a $this, - $ctx: $crate::layout::LayoutContext<'life0, 'life1> - ) -> std::pin::Pin<Box<dyn std::future::Future< - Output = $crate::layout::LayoutResult< - $crate::func::Commands<'a>> - > + 'async_trait - >> + (@layout($name:ident) layout($this:ident, $ctx:ident, $errors:ident) $code:block) => { + impl $crate::syntax::Model for $name { + fn layout<'a, 'b, 'c, 't>( + #[allow(unused)] &'a $this, + #[allow(unused)] $ctx: $crate::layout::LayoutContext<'b, 'c>, + ) -> $crate::syntax::DynFuture<'t, $crate::layout::Layouted<$crate::func::Commands<'a>>> where - 'a: 'async_trait, - 'life0: 'async_trait, - 'life1: 'async_trait, - Self: 'async_trait, + 'a: 't, + 'b: 't, + 'c: 't, + Self: 't, { - #[allow(unreachable_code)] - Box::pin(async move { Ok($code) }) + Box::pin(async move { + #[allow(unused)] let mut $errors = vec![]; + let output = $code; + $crate::layout::Layouted { output, errors: $errors } + }) } } }; @@ -137,35 +82,30 @@ macro_rules! function { /// - If the function can have a body, use `parse!(optional: body, ctx)`. /// - If the function must have a body, use `parse!(expected: body, ctx)`. #[macro_export] -macro_rules! parse { - (forbidden: $body:expr) => { - if $body.is_some() { - return Err($crate::TypesetError::with_message("unexpected body")); - } - }; - - (optional: $body:expr, $ctx:expr) => ( +macro_rules! body { + (opt: $body:expr, $ctx:expr, $errors:expr, $decos:expr) => ({ + $body.map(|body| { + let parsed = $crate::syntax::parse(body.span.start, body.v, $ctx); + $errors.extend(parsed.errors); + $decos.extend(parsed.decorations); + parsed.output + }) + }); + + (nope: $body:expr, $errors:expr) => { if let Some(body) = $body { - Some($crate::syntax::parse(body, $ctx).0) - } else { - None + $errors.push($crate::err!(body.span, "unexpected body")); } - ); - - (expected: $body:expr, $ctx:expr) => ( - if let Some(body) = $body { - $crate::syntax::parse(body, $ctx).0 - } else { - Err($crate::TypesetError::with_message("unexpected body")) - } - ) + }; } -/// Early-return with a formatted typesetting error or construct an error -/// expression. +/// Construct a spanned error. #[macro_export] -macro_rules! error { - (@unexpected_argument) => (error!(@"unexpected argument")); - (@$($tts:tt)*) => ($crate::TypesetError::with_message(format!($($tts)*))); - ($($tts:tt)*) => (return Err(error!(@$($tts)*));); +macro_rules! err { + ($span:expr, $($args:tt)*) => { + $crate::syntax::Spanned { + v: $crate::error::Error::new(format!($($args)*)), + span: $span, + } + }; } diff --git a/src/func/mod.rs b/src/func/mod.rs index bfc2774c..1ca226c3 100644 --- a/src/func/mod.rs +++ b/src/func/mod.rs @@ -3,7 +3,6 @@ use std::any::Any; use std::collections::HashMap; use std::fmt::{self, Debug, Formatter}; -use async_trait::async_trait; use self::prelude::*; @@ -12,81 +11,34 @@ mod macros; /// Useful imports for creating your own functions. pub mod prelude { - pub use crate::func::{Scope, ParseFunc, LayoutFunc, Command, Commands}; + pub use super::{Scope, Parse, Command, Commands}; pub use crate::layout::prelude::*; - pub use crate::syntax::*; + pub use crate::syntax::prelude::*; pub use crate::size::{Size, Size2D, SizeBox, ValueBox, ScaleSize, FSize, PSize}; pub use crate::style::{LayoutStyle, PageStyle, TextStyle}; pub use Command::*; } -/// Types representing functions that are parsed from source code. -pub trait ParseFunc { +/// Parse a function from source code. +pub trait Parse { type Meta: Clone; /// Parse the header and body into this function given a context. fn parse( header: FuncHeader, - body: Option<&str>, + body: Option<Spanned<&str>>, ctx: ParseContext, metadata: Self::Meta, - ) -> ParseResult<Self> where Self: Sized; + ) -> Parsed<Self> where Self: Sized; } -/// Function types which can be laid out in a layout context. -/// -/// This trait is a supertrait of `[LayoutFuncBounds]` for technical reasons. -/// The trait `[LayoutFuncBounds]` is automatically implemented for types which -/// can be used as functions, that is, all types which fulfill the bounds `Debug -/// + PartialEq + 'static`. -#[async_trait(?Send)] -pub trait LayoutFunc: LayoutFuncBounds { - /// Layout this function in a given context. - /// - /// Returns a sequence of layouting commands which describe what the - /// function is doing. - async fn layout<'a>(&'a self, ctx: LayoutContext<'_, '_>) -> LayoutResult<Commands<'a>>; -} - -impl dyn LayoutFunc { - /// Downcast a function trait object to a concrete function type. - pub fn downcast<F>(&self) -> Option<&F> where F: LayoutFunc + 'static { - self.help_cast_as_any().downcast_ref::<F>() - } -} - -impl PartialEq for dyn LayoutFunc { - fn eq(&self, other: &dyn LayoutFunc) -> bool { - self.help_eq(other) - } -} - -/// A helper trait that describes requirements for types that can implement -/// [`Function`]. -/// -/// Automatically implemented for all types which fulfill to the bounds `Debug + -/// PartialEq + 'static`. There should be no need to implement this manually. -pub trait LayoutFuncBounds: Debug { - /// Cast self into `Any`. - fn help_cast_as_any(&self) -> &dyn Any; - - /// Compare self with another function trait object. - fn help_eq(&self, other: &dyn LayoutFunc) -> bool; -} - -impl<T> LayoutFuncBounds for T where T: Debug + PartialEq + 'static { - fn help_cast_as_any(&self) -> &dyn Any { - self - } - - fn help_eq(&self, other: &dyn LayoutFunc) -> bool { - if let Some(other) = other.help_cast_as_any().downcast_ref::<Self>() { - self == other - } else { - false - } - } -} +/// A function which parses the source of a function into a model type which +/// implements [`Model`]. +type Parser = dyn Fn( + FuncHeader, + Option<Spanned<&str>>, + ParseContext, +) -> Parsed<Box<dyn Model>>; /// A sequence of layouting commands. pub type Commands<'a> = Vec<Command<'a>>; @@ -94,7 +46,7 @@ pub type Commands<'a> = Vec<Command<'a>>; /// Layouting commands from functions to the typesetting engine. #[derive(Debug)] pub enum Command<'a> { - LayoutTree(&'a SyntaxTree), + LayoutSyntaxModel(&'a SyntaxModel), Add(Layout), AddMultiple(MultiLayout), @@ -114,41 +66,17 @@ pub enum Command<'a> { /// A map from identifiers to function parsers. pub struct Scope { parsers: HashMap<String, Box<Parser>>, - debug: Option<Box<Parser>> -} - -/// A function which parses the source of a function into a function type which -/// implements [`LayoutFunc`]. -type Parser = dyn Fn( - FuncHeader, - Option<&str>, - ParseContext -) -> ParseResult<Box<dyn LayoutFunc>>; - -fn make_parser<F>(metadata: <F as ParseFunc>::Meta) -> Box<Parser> -where F: ParseFunc + LayoutFunc + 'static { - Box::new(move |a, b, c| { - F::parse(a, b, c, metadata.clone()) - .map(|f| Box::new(f) as Box<dyn LayoutFunc>) - }) + fallback: Box<Parser> } impl Scope { - /// Create a new empty scope. - pub fn new() -> Scope { - Scope { - parsers: HashMap::new(), - debug: None, - } - } - - /// Create a new scope with a debug parser that is invoked if not other + /// Create a new empty scope with a fallback parser that is invoked when no /// match is found. - pub fn with_debug<F>() -> Scope - where F: ParseFunc<Meta=()> + LayoutFunc + 'static { + pub fn new<F>() -> Scope + where F: Parse<Meta=()> + Model + 'static { Scope { parsers: HashMap::new(), - debug: Some(make_parser::<F>(())), + fallback: parser::<F>(()), } } @@ -159,24 +87,25 @@ impl Scope { /// Associate the given name with a type that is parseable into a function. pub fn add<F>(&mut self, name: &str) - where F: ParseFunc<Meta=()> + LayoutFunc + 'static { + where F: Parse<Meta=()> + Model + 'static { self.add_with_metadata::<F>(name, ()); } /// Add a parseable type with additional metadata that is given to the /// parser (other than the default of `()`). - pub fn add_with_metadata<F>(&mut self, name: &str, metadata: <F as ParseFunc>::Meta) - where F: ParseFunc + LayoutFunc + 'static { + pub fn add_with_metadata<F>(&mut self, name: &str, metadata: <F as Parse>::Meta) + where F: Parse + Model + 'static { self.parsers.insert( name.to_owned(), - make_parser::<F>(metadata), + parser::<F>(metadata), ); } /// Return the parser with the given name if there is one. - pub(crate) fn get_parser(&self, name: &str) -> Option<&Parser> { - self.parsers.get(name).map(|x| &**x) - .or(self.debug.as_ref().map(|x| &**x)) + pub(crate) fn get_parser(&self, name: &str) -> Result<&Parser, &Parser> { + self.parsers.get(name) + .map(|x| &**x) + .ok_or_else(|| &*self.fallback) } } @@ -186,3 +115,10 @@ impl Debug for Scope { write!(f, "{:?}", self.parsers.keys()) } } + +fn parser<F>(metadata: <F as Parse>::Meta) -> Box<Parser> where F: Parse + Model + 'static { + Box::new(move |h, b, c| { + F::parse(h, b, c, metadata.clone()) + .map(|model| Box::new(model) as Box<dyn Model>) + }) +} diff --git a/src/layout/line.rs b/src/layout/line.rs index c4205e81..f7777ae0 100644 --- a/src/layout/line.rs +++ b/src/layout/line.rs @@ -67,18 +67,21 @@ impl LineLayouter { } /// Add a layout to the run. - pub fn add(&mut self, layout: Layout) -> LayoutResult<()> { + pub fn add(&mut self, layout: Layout) { let axes = self.ctx.axes; if let Some(alignment) = self.run.alignment { if layout.alignment.secondary != alignment.secondary { - if self.stack.is_fitting_alignment(layout.alignment) { - self.finish_line()?; + // TODO: Issue warning for non-fitting alignment in + // non-repeating context. + let fitting = self.stack.is_fitting_alignment(layout.alignment); + if !fitting && self.ctx.repeat { + self.finish_space(true); } else { - self.finish_space(true)?; + self.finish_line(); } } else if layout.alignment.primary < alignment.primary { - self.finish_line()?; + self.finish_line(); } else if layout.alignment.primary > alignment.primary { let mut rest_run = LineRun::new(); @@ -92,7 +95,7 @@ impl LineLayouter { rest_run.size.y = self.run.size.y; - self.finish_line()?; + self.finish_line(); self.stack.add_spacing(-rest_run.size.y, SpacingKind::Hard); self.run = rest_run; @@ -105,16 +108,14 @@ impl LineLayouter { let size = layout.dimensions.generalized(axes); - while !self.usable().fits(size) { + if !self.usable().fits(size) { if !self.line_is_empty() { - self.finish_line()?; - } else { - if self.stack.space_is_last() && self.stack.space_is_empty() { - error!("cannot fit box of size {} into usable size of {}", - layout.dimensions, self.usable()); - } + self.finish_line(); + } - self.finish_space(true)?; + // TODO: Issue warning about overflow if there is overflow. + if !self.usable().fits(size) { + self.stack.skip_to_fitting_space(layout.dimensions); } } @@ -124,18 +125,15 @@ impl LineLayouter { self.run.size.x += size.x; self.run.size.y.max_eq(size.y); self.run.last_spacing = LastSpacing::None; - - Ok(()) } /// Add multiple layouts to the run. /// /// This function simply calls `add` repeatedly for each layout. - pub fn add_multiple(&mut self, layouts: MultiLayout) -> LayoutResult<()> { + pub fn add_multiple(&mut self, layouts: MultiLayout) { for layout in layouts { - self.add(layout)?; + self.add(layout); } - Ok(()) } /// The remaining usable size in the run. @@ -180,20 +178,16 @@ impl LineLayouter { } /// Finish the run and add secondary spacing to the underlying stack. - pub fn add_secondary_spacing( - &mut self, - spacing: Size, - kind: SpacingKind - ) -> LayoutResult<()> { - self.finish_line_if_not_empty()?; - Ok(self.stack.add_spacing(spacing, kind)) + pub fn add_secondary_spacing(&mut self, spacing: Size, kind: SpacingKind) { + self.finish_line_if_not_empty(); + self.stack.add_spacing(spacing, kind) } /// Change the layouting axes used by this layouter. - pub fn set_axes(&mut self, axes: LayoutAxes) -> LayoutResult<()> { - self.finish_line_if_not_empty()?; + pub fn set_axes(&mut self, axes: LayoutAxes) { + self.finish_line_if_not_empty(); self.ctx.axes = axes; - Ok(self.stack.set_axes(axes)) + self.stack.set_axes(axes) } /// Change the layouting spaces to use. @@ -224,19 +218,19 @@ impl LineLayouter { } /// Finish the last line and compute the final multi-layout. - pub fn finish(mut self) -> LayoutResult<MultiLayout> { - self.finish_line_if_not_empty()?; + pub fn finish(mut self) -> MultiLayout { + self.finish_line_if_not_empty(); self.stack.finish() } /// Finish the currently active space and start a new one. - pub fn finish_space(&mut self, hard: bool) -> LayoutResult<()> { - self.finish_line_if_not_empty()?; + pub fn finish_space(&mut self, hard: bool) { + self.finish_line_if_not_empty(); self.stack.finish_space(hard) } /// Add the current line to the stack and start a new line. - pub fn finish_line(&mut self) -> LayoutResult<()> { + pub fn finish_line(&mut self) { let mut actions = LayoutActions::new(); let layouts = std::mem::replace(&mut self.run.layouts, vec![]); @@ -257,21 +251,17 @@ impl LineLayouter { alignment: self.run.alignment .unwrap_or(LayoutAlignment::new(Origin, Origin)), actions: actions.to_vec(), - })?; + }); self.run = LineRun::new(); - self.stack.add_spacing(self.ctx.line_spacing, LINE_KIND); - - Ok(()) + self.stack.add_spacing(self.ctx.line_spacing, SpacingKind::LINE); } /// Finish the current line if it is not empty. - fn finish_line_if_not_empty(&mut self) -> LayoutResult<()> { + fn finish_line_if_not_empty(&mut self) { if !self.line_is_empty() { self.finish_line() - } else { - Ok(()) } } } diff --git a/src/layout/mod.rs b/src/layout/mod.rs index 49551945..75d34409 100644 --- a/src/layout/mod.rs +++ b/src/layout/mod.rs @@ -4,23 +4,20 @@ use std::io::{self, Write}; use smallvec::SmallVec; use toddle::query::{SharedFontLoader, FontIndex}; +use crate::error::Error; +use crate::syntax::SpanVec; use crate::size::{Size, Size2D, SizeBox}; use crate::style::LayoutStyle; mod actions; -mod tree; +mod model; mod line; mod stack; mod text; /// Common types for layouting. pub mod prelude { - pub use super::{ - layout, LayoutResult, - MultiLayout, Layout, LayoutContext, LayoutSpaces, LayoutSpace, - LayoutExpansion, LayoutAxes, GenericAxis, SpecificAxis, Direction, - LayoutAlignment, Alignment, SpacingKind, - }; + pub use super::*; pub use GenericAxis::*; pub use SpecificAxis::*; pub use Direction::*; @@ -29,7 +26,7 @@ pub mod prelude { /// Different kinds of layouters (fully re-exported). pub mod layouters { - pub use super::tree::layout; + pub use super::model::layout; pub use super::line::{LineLayouter, LineContext}; pub use super::stack::{StackLayouter, StackContext}; pub use super::text::{layout_text, TextContext}; @@ -40,8 +37,19 @@ pub use self::layouters::*; pub use self::prelude::*; -/// The result type for layouting. -pub type LayoutResult<T> = crate::TypesetResult<T>; +pub struct Layouted<T> { + pub output: T, + pub errors: SpanVec<Error>, +} + +impl<T> Layouted<T> { + pub fn map<F, U>(self, f: F) -> Layouted<U> where F: FnOnce(T) -> U { + Layouted { + output: f(self.output), + errors: self.errors, + } + } +} /// A collection of layouts. pub type MultiLayout = Vec<Layout>; @@ -361,14 +369,16 @@ pub enum SpacingKind { Soft(u32), } -/// The standard spacing kind used for paragraph spacing. -const PARAGRAPH_KIND: SpacingKind = SpacingKind::Soft(1); +impl SpacingKind { + /// The standard spacing kind used for paragraph spacing. + pub const PARAGRAPH: SpacingKind = SpacingKind::Soft(1); -/// The standard spacing kind used for line spacing. -const LINE_KIND: SpacingKind = SpacingKind::Soft(2); + /// The standard spacing kind used for line spacing. + pub const LINE: SpacingKind = SpacingKind::Soft(2); -/// The standard spacing kind used for word spacing. -const WORD_KIND: SpacingKind = SpacingKind::Soft(1); + /// The standard spacing kind used for word spacing. + pub const WORD: SpacingKind = SpacingKind::Soft(1); +} /// The last appeared spacing. #[derive(Debug, Copy, Clone, PartialEq)] diff --git a/src/layout/model.rs b/src/layout/model.rs new file mode 100644 index 00000000..bcec5ceb --- /dev/null +++ b/src/layout/model.rs @@ -0,0 +1,193 @@ +use std::pin::Pin; +use std::future::Future; +use smallvec::smallvec; + +use crate::error::Error; +use crate::func::Command; +use crate::syntax::{Model, DynFuture, SyntaxModel, Node}; +use crate::syntax::{SpanVec, Spanned, Span, offset_spans}; +use super::*; + + +pub async fn layout( + model: &SyntaxModel, + ctx: LayoutContext<'_, '_> +) -> Layouted<MultiLayout> { + let mut layouter = ModelLayouter::new(ctx); + layouter.layout_syntax_model(model).await; + layouter.finish() +} + +#[derive(Debug, Clone)] +struct ModelLayouter<'a, 'p> { + ctx: LayoutContext<'a, 'p>, + layouter: LineLayouter, + style: LayoutStyle, + errors: SpanVec<Error>, +} + +impl<'a, 'p> ModelLayouter<'a, 'p> { + /// Create a new syntax tree layouter. + fn new(ctx: LayoutContext<'a, 'p>) -> ModelLayouter<'a, 'p> { + ModelLayouter { + layouter: LineLayouter::new(LineContext { + spaces: ctx.spaces.clone(), + axes: ctx.axes, + alignment: ctx.alignment, + repeat: ctx.repeat, + debug: ctx.debug, + line_spacing: ctx.style.text.line_spacing(), + }), + style: ctx.style.clone(), + ctx, + errors: vec![], + } + } + + fn layout<'r>( + &'r mut self, + model: Spanned<&'r dyn Model> + ) -> DynFuture<'r, ()> { Box::pin(async move { + let layouted = model.v.layout(LayoutContext { + style: &self.style, + spaces: self.layouter.remaining(), + nested: true, + debug: false, + .. self.ctx + }).await; + + let commands = layouted.output; + self.errors.extend(offset_spans(layouted.errors, model.span.start)); + + for command in commands { + self.execute_command(command, model.span); + } + }) } + + fn execute_command<'r>( + &'r mut self, + command: Command<'r>, + model_span: Span, + ) -> DynFuture<'r, ()> { Box::pin(async move { + use Command::*; + + match command { + LayoutSyntaxModel(model) => self.layout_syntax_model(model).await, + + Add(layout) => self.layouter.add(layout), + AddMultiple(layouts) => self.layouter.add_multiple(layouts), + SpacingFunc(space, kind, axis) => match axis { + Primary => self.layouter.add_primary_spacing(space, kind), + Secondary => self.layouter.add_secondary_spacing(space, kind), + } + + FinishLine => self.layouter.finish_line(), + FinishSpace => self.layouter.finish_space(true), + BreakParagraph => self.layout_paragraph(), + BreakPage => { + if self.ctx.nested { + self.errors.push(Spanned::new( + Error::new( "page break cannot be issued from nested context"), + model_span, + )); + } else { + self.layouter.finish_space(true) + } + } + + SetTextStyle(style) => { + self.layouter.set_line_spacing(style.line_spacing()); + self.style.text = style; + } + SetPageStyle(style) => { + if self.ctx.nested { + self.errors.push(Spanned::new( + Error::new("page style cannot be changed from nested context"), + model_span, + )); + } else { + self.style.page = style; + + let margins = style.margins(); + self.ctx.base = style.dimensions.unpadded(margins); + self.layouter.set_spaces(smallvec![ + LayoutSpace { + dimensions: style.dimensions, + padding: margins, + expansion: LayoutExpansion::new(true, true), + } + ], true); + } + } + + SetAlignment(alignment) => self.ctx.alignment = alignment, + SetAxes(axes) => { + self.layouter.set_axes(axes); + self.ctx.axes = axes; + } + } + }) } + + fn layout_syntax_model<'r>( + &'r mut self, + model: &'r SyntaxModel + ) -> DynFuture<'r, ()> { Box::pin(async move { + use Node::*; + + for node in &model.nodes { + match &node.v { + Space => self.layout_space(), + Newline => self.layout_paragraph(), + Text(text) => self.layout_text(text).await, + + ToggleItalic => self.style.text.variant.style.toggle(), + ToggleBolder => { + let fac = if self.style.text.bolder { -1 } else { 1 }; + self.style.text.variant.weight.0 += 300 * fac; + self.style.text.bolder = !self.style.text.bolder; + } + ToggleMonospace => { + let list = &mut self.style.text.fallback.list; + match list.get(0).map(|s| s.as_str()) { + Some("monospace") => { list.remove(0); }, + _ => list.insert(0, "monospace".to_string()), + } + } + + Node::Model(model) => { + self.layout(Spanned::new(model.as_ref(), node.span)).await; + } + } + } + }) } + + async fn layout_text(&mut self, text: &str) { + self.layouter.add(layout_text(text, TextContext { + loader: &self.ctx.loader, + style: &self.style.text, + axes: self.ctx.axes, + alignment: self.ctx.alignment, + }).await) + } + + fn layout_space(&mut self) { + self.layouter.add_primary_spacing( + self.style.text.word_spacing(), + SpacingKind::WORD, + ); + } + + fn layout_paragraph(&mut self) { + self.layouter.add_secondary_spacing( + self.style.text.paragraph_spacing(), + SpacingKind::PARAGRAPH, + ); + } + + fn finish(self) -> Layouted<MultiLayout> { + Layouted { + output: self.layouter.finish(), + errors: self.errors, + } + } +} diff --git a/src/layout/stack.rs b/src/layout/stack.rs index 80d57424..96b44d04 100644 --- a/src/layout/stack.rs +++ b/src/layout/stack.rs @@ -68,10 +68,12 @@ impl StackLayouter { } /// Add a layout to the stack. - pub fn add(&mut self, layout: Layout) -> LayoutResult<()> { - // If the alignment cannot be fit in this space, finish it. - if !self.update_rulers(layout.alignment) { - self.finish_space(true)?; + pub fn add(&mut self, layout: Layout) { + // If the alignment cannot be fitted in this space, finish it. + // TODO: Issue warning for non-fitting alignment in + // non-repeating context. + if !self.update_rulers(layout.alignment) && self.ctx.repeat { + self.finish_space(true); } // Now, we add a possibly cached soft space. If the secondary alignment @@ -81,14 +83,9 @@ impl StackLayouter { self.add_spacing(spacing, SpacingKind::Hard); } - // Find the first space that fits the layout. - while !self.space.usable.fits(layout.dimensions) { - if self.space_is_last() && self.space_is_empty() { - error!("cannot fit box of size {} into usable size of {}", - layout.dimensions, self.space.usable); - } - - self.finish_space(true)?; + // TODO: Issue warning about overflow if there is overflow. + if !self.space.usable.fits(layout.dimensions) && self.ctx.repeat { + self.skip_to_fitting_space(layout.dimensions); } // Change the usable space and size of the space. @@ -98,18 +95,15 @@ impl StackLayouter { // again. self.space.layouts.push((self.ctx.axes, layout)); self.space.last_spacing = LastSpacing::None; - - Ok(()) } /// Add multiple layouts to the stack. /// /// This function simply calls `add` repeatedly for each layout. - pub fn add_multiple(&mut self, layouts: MultiLayout) -> LayoutResult<()> { + pub fn add_multiple(&mut self, layouts: MultiLayout) { for layout in layouts { - self.add(layout)?; + self.add(layout); } - Ok(()) } /// Add secondary spacing to the stack. @@ -215,6 +209,19 @@ impl StackLayouter { } } + /// Move to the first space that can fit the given dimensions or do nothing + /// if no space is capable of that. + pub fn skip_to_fitting_space(&mut self, dimensions: Size2D) { + let start = self.next_space(); + for (index, space) in self.ctx.spaces[start..].iter().enumerate() { + if space.usable().fits(dimensions) { + self.finish_space(true); + self.start_space(start + index, true); + return; + } + } + } + /// The remaining unpadded, unexpanding spaces. If a multi-layout is laid /// out into these spaces, it will fit into this stack. pub fn remaining(&self) -> LayoutSpaces { @@ -251,19 +258,15 @@ impl StackLayouter { } /// Compute the finished multi-layout. - pub fn finish(mut self) -> LayoutResult<MultiLayout> { + pub fn finish(mut self) -> MultiLayout { if self.space.hard || !self.space_is_empty() { - self.finish_space(false)?; + self.finish_space(false); } - Ok(self.layouts) + self.layouts } /// Finish the current space and start a new one. - pub fn finish_space(&mut self, hard: bool) -> LayoutResult<()> { - if !self.ctx.repeat && hard { - error!("cannot create new space in a non-repeating context"); - } - + pub fn finish_space(&mut self, hard: bool) { let space = self.ctx.spaces[self.space.index]; // ------------------------------------------------------------------ // @@ -376,7 +379,7 @@ impl StackLayouter { // ------------------------------------------------------------------ // // Step 5: Start the next space. - Ok(self.start_space(self.next_space(), hard)) + self.start_space(self.next_space(), hard) } /// Start a new space with the given index. diff --git a/src/layout/text.rs b/src/layout/text.rs index a66e04a2..16ae93da 100644 --- a/src/layout/text.rs +++ b/src/layout/text.rs @@ -6,6 +6,14 @@ use crate::style::TextStyle; use super::*; +/// Layouts text into a box. +/// +/// There is no complex layout involved. The text is simply laid out left- +/// to-right using the correct font for each character. +pub async fn layout_text(text: &str, ctx: TextContext<'_, '_>) -> Layout { + TextLayouter::new(text, ctx).layout().await +} + /// The context for text layouting. /// /// See [`LayoutContext`] for details about the fields. @@ -17,14 +25,6 @@ pub struct TextContext<'a, 'p> { pub alignment: LayoutAlignment, } -/// Layouts text into a box. -/// -/// There is no complex layout involved. The text is simply laid out left- -/// to-right using the correct font for each character. -pub async fn layout_text(text: &str, ctx: TextContext<'_, '_>) -> LayoutResult<Layout> { - TextLayouter::new(text, ctx).layout().await -} - /// Layouts text into boxes. struct TextLayouter<'a, 'p> { ctx: TextContext<'a, 'p>, @@ -49,14 +49,14 @@ impl<'a, 'p> TextLayouter<'a, 'p> { } /// Layout the text - async fn layout(mut self) -> LayoutResult<Layout> { + async fn layout(mut self) -> Layout { if self.ctx.axes.primary.is_positive() { for c in self.text.chars() { - self.layout_char(c).await?; + self.layout_char(c).await; } } else { for c in self.text.chars().rev() { - self.layout_char(c).await?; + self.layout_char(c).await; } } @@ -64,16 +64,20 @@ impl<'a, 'p> TextLayouter<'a, 'p> { self.actions.add(LayoutAction::WriteText(self.buffer)); } - Ok(Layout { + Layout { dimensions: Size2D::new(self.width, self.ctx.style.font_size()), alignment: self.ctx.alignment, actions: self.actions.to_vec(), - }) + } } /// Layout an individual character. - async fn layout_char(&mut self, c: char) -> LayoutResult<()> { - let (index, char_width) = self.select_font(c).await?; + async fn layout_char(&mut self, c: char) { + let (index, char_width) = match self.select_font(c).await { + Some(selected) => selected, + // TODO: Issue warning about missing character. + None => return, + }; self.width += char_width; @@ -88,13 +92,11 @@ impl<'a, 'p> TextLayouter<'a, 'p> { } self.buffer.push(c); - - Ok(()) } /// Select the best font for a character and return its index along with /// the width of the char in the font. - async fn select_font(&mut self, c: char) -> LayoutResult<(FontIndex, Size)> { + async fn select_font(&mut self, c: char) -> Option<(FontIndex, Size)> { let mut loader = self.ctx.loader.borrow_mut(); let query = FontQuery { @@ -104,26 +106,27 @@ impl<'a, 'p> TextLayouter<'a, 'p> { }; if let Some((font, index)) = loader.get(query).await { - let font_unit_ratio = 1.0 / (font.read_table::<Header>()?.units_per_em as f32); + let header = font.read_table::<Header>().ok()?; + let font_unit_ratio = 1.0 / (header.units_per_em as f32); let font_unit_to_size = |x| Size::pt(font_unit_ratio * x); let glyph = font - .read_table::<CharMap>()? - .get(c) - .expect("select_font: font should have char"); + .read_table::<CharMap>() + .ok()? + .get(c)?; let glyph_width = font - .read_table::<HorizontalMetrics>()? - .get(glyph) - .expect("select_font: font should have glyph") + .read_table::<HorizontalMetrics>() + .ok()? + .get(glyph)? .advance_width as f32; let char_width = font_unit_to_size(glyph_width) * self.ctx.style.font_size().to_pt(); - return Ok((index, char_width)); + Some((index, char_width)) + } else { + None } - - error!("no suitable font for character `{}`", c); } } diff --git a/src/layout/tree.rs b/src/layout/tree.rs deleted file mode 100644 index 86b00f22..00000000 --- a/src/layout/tree.rs +++ /dev/null @@ -1,163 +0,0 @@ -use std::pin::Pin; -use std::future::Future; -use smallvec::smallvec; - -use crate::func::Command; -use crate::syntax::{SyntaxTree, Node, FuncCall}; -use super::*; - - -type RecursiveResult<'a, T> = Pin<Box<dyn Future<Output=LayoutResult<T>> + 'a>>; - -/// Layout a syntax tree into a multibox. -pub async fn layout(tree: &SyntaxTree, ctx: LayoutContext<'_, '_>) -> LayoutResult<MultiLayout> { - let mut layouter = TreeLayouter::new(ctx); - layouter.layout(tree).await?; - layouter.finish() -} - -#[derive(Debug, Clone)] -struct TreeLayouter<'a, 'p> { - ctx: LayoutContext<'a, 'p>, - layouter: LineLayouter, - style: LayoutStyle, -} - -impl<'a, 'p> TreeLayouter<'a, 'p> { - /// Create a new syntax tree layouter. - fn new(ctx: LayoutContext<'a, 'p>) -> TreeLayouter<'a, 'p> { - TreeLayouter { - layouter: LineLayouter::new(LineContext { - spaces: ctx.spaces.clone(), - axes: ctx.axes, - alignment: ctx.alignment, - repeat: ctx.repeat, - debug: ctx.debug, - line_spacing: ctx.style.text.line_spacing(), - }), - style: ctx.style.clone(), - ctx, - } - } - - fn layout<'b>(&'b mut self, tree: &'b SyntaxTree) -> RecursiveResult<'b, ()> { - Box::pin(async move { - for node in &tree.nodes { - match &node.v { - Node::Text(text) => self.layout_text(text).await?, - - Node::Space => self.layout_space(), - Node::Newline => self.layout_paragraph()?, - - Node::ToggleItalic => self.style.text.variant.style.toggle(), - Node::ToggleBolder => { - self.style.text.variant.weight.0 += 300 * - if self.style.text.bolder { -1 } else { 1 }; - self.style.text.bolder = !self.style.text.bolder; - } - Node::ToggleMonospace => { - let list = &mut self.style.text.fallback.list; - match list.get(0).map(|s| s.as_str()) { - Some("monospace") => { list.remove(0); }, - _ => list.insert(0, "monospace".to_string()), - } - } - - Node::Func(func) => self.layout_func(func).await?, - } - } - - Ok(()) - }) - } - - async fn layout_text(&mut self, text: &str) -> LayoutResult<()> { - let layout = layout_text(text, TextContext { - loader: &self.ctx.loader, - style: &self.style.text, - axes: self.ctx.axes, - alignment: self.ctx.alignment, - }).await?; - - self.layouter.add(layout) - } - - fn layout_space(&mut self) { - self.layouter.add_primary_spacing(self.style.text.word_spacing(), WORD_KIND); - } - - fn layout_paragraph(&mut self) -> LayoutResult<()> { - self.layouter.add_secondary_spacing(self.style.text.paragraph_spacing(), PARAGRAPH_KIND) - } - - fn layout_func<'b>(&'b mut self, func: &'b FuncCall) -> RecursiveResult<'b, ()> { - Box::pin(async move { - let commands = func.0.layout(LayoutContext { - style: &self.style, - spaces: self.layouter.remaining(), - nested: true, - debug: false, - .. self.ctx - }).await?; - - for command in commands { - use Command::*; - - match command { - LayoutTree(tree) => self.layout(tree).await?, - - Add(layout) => self.layouter.add(layout)?, - AddMultiple(layouts) => self.layouter.add_multiple(layouts)?, - SpacingFunc(space, kind, axis) => match axis { - Primary => self.layouter.add_primary_spacing(space, kind), - Secondary => self.layouter.add_secondary_spacing(space, kind)?, - } - - FinishLine => self.layouter.finish_line()?, - FinishSpace => self.layouter.finish_space(true)?, - BreakParagraph => self.layout_paragraph()?, - BreakPage => { - if self.ctx.nested { - error!("page break cannot be issued from nested context"); - } - - self.layouter.finish_space(true)? - } - - SetTextStyle(style) => { - self.layouter.set_line_spacing(style.line_spacing()); - self.style.text = style; - } - SetPageStyle(style) => { - if self.ctx.nested { - error!("page style cannot be altered in nested context"); - } - - self.style.page = style; - - let margins = style.margins(); - self.ctx.base = style.dimensions.unpadded(margins); - self.layouter.set_spaces(smallvec![ - LayoutSpace { - dimensions: style.dimensions, - padding: margins, - expansion: LayoutExpansion::new(true, true), - } - ], true); - } - SetAlignment(alignment) => self.ctx.alignment = alignment, - SetAxes(axes) => { - self.layouter.set_axes(axes)?; - self.ctx.axes = axes; - } - } - } - - Ok(()) - }) - } - - fn finish(self) -> LayoutResult<MultiLayout> { - self.layouter.finish() - } -} @@ -27,13 +27,14 @@ use toddle::query::{FontLoader, FontProvider, SharedFontLoader}; use toddle::Error as FontError; use crate::func::Scope; -use crate::layout::{MultiLayout, LayoutResult}; -use crate::syntax::{parse, SyntaxTree, Colorization, ErrorMap, ParseContext, Span}; +use crate::layout::{Layouted, LayoutContext, MultiLayout}; +use crate::syntax::{parse, ParseContext, Parsed, SyntaxModel, Position}; use crate::style::{LayoutStyle, PageStyle, TextStyle}; #[macro_use] mod macros; pub mod export; +pub mod error; #[macro_use] pub mod func; pub mod layout; @@ -51,6 +52,8 @@ pub struct Typesetter<'p> { loader: SharedFontLoader<'p>, /// The base layouting style. style: LayoutStyle, + /// The standard library scope. + scope: Scope, } impl<'p> Typesetter<'p> { @@ -59,6 +62,7 @@ impl<'p> Typesetter<'p> { Typesetter { loader: RefCell::new(FontLoader::new()), style: LayoutStyle::default(), + scope: Scope::with_std(), } } @@ -84,17 +88,16 @@ impl<'p> Typesetter<'p> { } /// Parse source code into a syntax tree. - pub fn parse(&self, src: &str) -> (SyntaxTree, Colorization, ErrorMap) { - let scope = Scope::with_std(); - parse(src, ParseContext { scope: &scope }) + pub fn parse(&self, src: &str) -> Parsed<SyntaxModel> { + parse(Position::ZERO, src, ParseContext { scope: &self.scope }) } /// Layout a syntax tree and return the produced layout. - pub async fn layout(&self, tree: &SyntaxTree) -> LayoutResult<MultiLayout> { + pub async fn layout(&self, model: &SyntaxModel) -> Layouted<MultiLayout> { use crate::layout::prelude::*; let margins = self.style.page.margins(); - Ok(layout( - &tree, + layout( + &model, LayoutContext { loader: &self.loader, style: &self.style, @@ -110,42 +113,12 @@ impl<'p> Typesetter<'p> { nested: false, debug: false, }, - ).await?) + ).await } - /// Process source code directly into a layout. - pub async fn typeset(&self, src: &str) -> TypesetResult<MultiLayout> { - let tree = self.parse(src).0; - let layout = self.layout(&tree).await?; - Ok(layout) + /// Process source code directly into a collection of layouts. + pub async fn typeset(&self, src: &str) -> MultiLayout { + let tree = self.parse(src).output; + self.layout(&tree).await.output } } - -/// The result type for typesetting. -pub type TypesetResult<T> = Result<T, TypesetError>; - -/// The error type for typesetting. -pub struct TypesetError { - pub message: String, - pub span: Option<Span>, -} - -impl TypesetError { - /// Create a new typesetting error. - pub fn with_message(message: impl Into<String>) -> TypesetError { - TypesetError { message: message.into(), span: None } - } -} - -error_type! { - self: TypesetError, - show: f => { - write!(f, "{}", self.message)?; - if let Some(span) = self.span { - write!(f, " at {}", span)?; - } - Ok(()) - }, - from: (err: std::io::Error, TypesetError::with_message(err.to_string())), - from: (err: FontError, TypesetError::with_message(err.to_string())), -} diff --git a/src/syntax/expr.rs b/src/syntax/expr.rs index c4feea74..74deda46 100644 --- a/src/syntax/expr.rs +++ b/src/syntax/expr.rs @@ -1,3 +1,4 @@ +use crate::size::ScaleSize; use super::*; @@ -126,6 +127,7 @@ impl Display for Object { } } +/// A key-value pair in an object. #[derive(Clone, PartialEq)] pub struct Pair { pub key: Spanned<Ident>, @@ -144,7 +146,6 @@ debug_display!(Tuple); debug_display!(Object); debug_display!(Pair); - /// Kinds of expressions. pub trait ExpressionKind: Sized { /// The name of the expression in an `expected <name>` error. diff --git a/src/syntax/func.rs b/src/syntax/func.rs new file mode 100644 index 00000000..5b1ce6e8 --- /dev/null +++ b/src/syntax/func.rs @@ -0,0 +1,125 @@ +use super::*; + + +#[derive(Debug, Clone, PartialEq)] +pub struct FuncHeader { + pub name: Spanned<Ident>, + pub args: FuncArgs, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct FuncArgs { + pub pos: Tuple, + pub key: Object, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum Arg { + Pos(Spanned<Expression>), + Key(Pair), +} + +impl Arg { + /// The span or the value or combined span of key and value. + pub fn span(&self) -> Span { + match self { + Arg::Pos(item) => item.span, + Arg::Key(Pair { key, value }) => Span::merge(key.span, value.span), + } + } +} + +impl FuncArgs { + pub fn new() -> FuncArgs { + FuncArgs { + pos: Tuple::new(), + key: Object::new(), + } + } + + /// Add an argument. + pub fn add(&mut self, arg: Arg) { + match arg { + Arg::Pos(item) => self.add_pos(item), + Arg::Key(pair) => self.add_key_pair(pair), + } + } + + /// Add a positional argument. + pub fn add_pos(&mut self, item: Spanned<Expression>) { + self.pos.add(item); + } + + /// Add a keyword argument. + pub fn add_key(&mut self, key: Spanned<Ident>, value: Spanned<Expression>) { + self.key.add(key, value); + } + + /// Add a keyword argument from an existing pair. + pub fn add_key_pair(&mut self, pair: Pair) { + self.key.add_pair(pair); + } + + // /// Force-extract the first positional argument. + // pub fn get_pos<E: ExpressionKind>(&mut self) -> ParseResult<E> { + // expect(self.get_pos_opt()) + // } + + // /// Extract the first positional argument. + // pub fn get_pos_opt<E: ExpressionKind>(&mut self) -> ParseResult<Option<E>> { + // Ok(if !self.positional.items.is_empty() { + // let spanned = self.positional.items.remove(0); + // Some(E::from_expr(spanned)?) + // } else { + // None + // }) + // } + + // /// Force-extract a keyword argument. + // pub fn get_key<E: ExpressionKind>(&mut self, name: &str) -> ParseResult<E> { + // expect(self.get_key_opt(name)) + // } + + // /// Extract a keyword argument. + // pub fn get_key_opt<E: ExpressionKind>(&mut self, name: &str) -> ParseResult<Option<E>> { + // self.keyword.pairs.iter() + // .position(|p| p.key.v.0 == name) + // .map(|index| { + // let value = self.keyword.pairs.swap_remove(index).value; + // E::from_expr(value) + // }) + // .transpose() + // } + + // /// Iterator over positional arguments. + // pub fn iter_pos(&mut self) -> std::vec::IntoIter<Spanned<Expression>> { + // let tuple = std::mem::replace(&mut self.positional, Tuple::new()); + // tuple.items.into_iter() + // } + + // /// Iterator over all keyword arguments. + // pub fn iter_keys(&mut self) -> std::vec::IntoIter<Pair> { + // let object = std::mem::replace(&mut self.keyword, Object::new()); + // object.pairs.into_iter() + // } + + // /// Clear the argument lists. + // pub fn clear(&mut self) { + // self.positional.items.clear(); + // self.keyword.pairs.clear(); + // } + + // /// Whether both the positional and keyword argument lists are empty. + // pub fn is_empty(&self) -> bool { + // self.positional.items.is_empty() && self.keyword.pairs.is_empty() + // } +} + +// /// Extract the option expression kind from the option or return an error. +// fn expect<E: ExpressionKind>(opt: ParseResult<Option<E>>) -> ParseResult<E> { +// match opt { +// Ok(Some(spanned)) => Ok(spanned), +// Ok(None) => error!("expected {}", E::NAME), +// Err(e) => Err(e), +// } +// } diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs index f644f051..75407f82 100644 --- a/src/syntax/mod.rs +++ b/src/syntax/mod.rs @@ -1,321 +1,161 @@ //! Tokenization and parsing of source code. -use std::fmt::{self, Display, Formatter}; -use unicode_xid::UnicodeXID; +use std::any::Any; +use std::fmt::{self, Debug, Display, Formatter}; +use std::future::Future; +use std::pin::Pin; use serde::Serialize; -use crate::func::LayoutFunc; -use crate::size::{Size, ScaleSize}; - - -pub type ParseResult<T> = crate::TypesetResult<T>; +use crate::error::Error; +use crate::func::{Commands, Command}; +use crate::layout::{Layouted, LayoutContext}; +use crate::size::Size; pub_use_mod!(expr); +pub_use_mod!(func); pub_use_mod!(tokens); pub_use_mod!(parsing); pub_use_mod!(span); - -/// A minimal semantic entity of source code. -#[derive(Debug, Copy, Clone, PartialEq)] -pub enum Token<'s> { - /// One or more whitespace characters. The contained `usize` denotes the - /// number of newlines that were contained in the whitespace. - Whitespace(usize), - - /// A line comment with inner string contents `//<&'s str>\n`. - LineComment(&'s str), - /// A block comment with inner string contents `/*<&'s str>*/`. The comment - /// can contain nested block comments. - BlockComment(&'s str), - /// An erroneous `*/` without an opening block comment. - StarSlash, - - /// A left bracket: `[`. - LeftBracket, - /// A right bracket: `]`. - RightBracket, - - /// A left parenthesis in a function header: `(`. - LeftParen, - /// A right parenthesis in a function header: `)`. - RightParen, - /// A left brace in a function header: `{`. - LeftBrace, - /// A right brace in a function header: `}`. - RightBrace, - - /// A colon in a function header: `:`. - Colon, - /// A comma in a function header: `:`. - Comma, - /// An equals sign in a function header: `=`. - Equals, - - /// An identifier in a function header: `center`. - ExprIdent(&'s str), - /// A quoted string in a function header: `"..."`. - ExprStr(&'s str), - /// A number in a function header: `3.14`. - ExprNumber(f64), - /// A size in a function header: `12pt`. - ExprSize(Size), - /// A boolean in a function header: `true | false`. - ExprBool(bool), - - /// A star in body-text. - Star, - /// An underscore in body-text. - Underscore, - /// A backtick in body-text. - Backtick, - - /// Any other consecutive string. - Text(&'s str), -} - -/// A tree representation of source code. -#[derive(Debug, PartialEq)] -pub struct SyntaxTree { - pub nodes: Vec<Spanned<Node>>, +/// Common syntax types. +pub mod prelude { + pub use super::*; } -impl SyntaxTree { - /// Create an empty syntax tree. - pub fn new() -> SyntaxTree { - SyntaxTree { nodes: vec![] } - } - /// Add a node to the tree. - pub fn add(&mut self, node: Spanned<Node>) { - self.nodes.push(node); - } +pub struct Parsed<T> { + pub output: T, + pub errors: SpanVec<Error>, + pub decorations: SpanVec<Decoration>, } -/// A node in the syntax tree. -#[derive(PartialEq)] -pub enum Node { - /// A number of whitespace characters containing less than two newlines. - Space, - /// Whitespace characters with more than two newlines. - Newline, - /// Plain text. - Text(String), - /// Italics enabled / disabled. - ToggleItalic, - /// Bolder enabled / disabled. - ToggleBolder, - /// Monospace enabled / disabled. - ToggleMonospace, - /// A function invocation. - Func(FuncCall), -} - -impl Display for Node { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - match self { - Node::Space => write!(f, "Space"), - Node::Newline => write!(f, "Newline"), - Node::Text(text) => write!(f, "{:?}", text), - Node::ToggleItalic => write!(f, "ToggleItalic"), - Node::ToggleBolder => write!(f, "ToggleBold"), - Node::ToggleMonospace => write!(f, "ToggleMonospace"), - Node::Func(func) => { - if f.alternate() { - write!(f, "{:#?}", func.0) - } else { - write!(f, "{:?}", func.0) - } - } +impl<T> Parsed<T> { + pub fn map<F, U>(self, f: F) -> Parsed<U> where F: FnOnce(T) -> U { + Parsed { + output: f(self.output), + errors: self.errors, + decorations: self.decorations, } } } -debug_display!(Node); +#[async_trait::async_trait(?Send)] +pub trait Model: Debug + ModelBounds { + async fn layout<'a>( + &'a self, + ctx: LayoutContext<'_, '_> + ) -> Layouted<Commands<'a>>; +} -/// An invocation of a function. -#[derive(Debug)] -pub struct FuncCall(pub Box<dyn LayoutFunc>); +pub type DynFuture<'a, T> = Pin<Box<dyn Future<Output=T> + 'a>>; -impl PartialEq for FuncCall { - fn eq(&self, other: &FuncCall) -> bool { - &self.0 == &other.0 +impl dyn Model { + pub fn downcast<T>(&self) -> Option<&T> where T: Model + 'static { + self.as_any().downcast_ref::<T>() } } -#[derive(Debug, Clone, PartialEq)] -pub struct FuncHeader { - pub name: Spanned<Ident>, - pub args: FuncArgs, +impl PartialEq for dyn Model { + fn eq(&self, other: &dyn Model) -> bool { + self.bound_eq(other) + } } -#[derive(Debug, Clone, PartialEq)] -pub struct FuncArgs { - pub positional: Tuple, - pub keyword: Object, +impl Clone for Box<dyn Model> { + fn clone(&self) -> Self { + self.bound_clone() + } } -#[derive(Debug, Clone, PartialEq)] -pub enum Arg { - Pos(Spanned<Expression>), - Key(Pair), +pub trait ModelBounds { + fn as_any(&self) -> &dyn Any; + fn bound_eq(&self, other: &dyn Model) -> bool; + fn bound_clone(&self) -> Box<dyn Model>; } -impl Arg { - /// The span or the value or combined span of key and value. - pub fn span(&self) -> Span { - match self { - Arg::Pos(spanned) => spanned.span, - Arg::Key(Pair { key, value }) => Span::merge(key.span, value.span), - } +impl<T> ModelBounds for T where T: Model + PartialEq + Clone + 'static { + fn as_any(&self) -> &dyn Any { + self } -} -impl FuncArgs { - pub fn new() -> FuncArgs { - FuncArgs { - positional: Tuple::new(), - keyword: Object::new(), + fn bound_eq(&self, other: &dyn Model) -> bool { + match other.as_any().downcast_ref::<Self>() { + Some(other) => self == other, + None => false, } } - /// Add a positional argument. - pub fn add_pos(&mut self, item: Spanned<Expression>) { - self.positional.add(item); - } - - /// Force-extract the first positional argument. - pub fn get_pos<E: ExpressionKind>(&mut self) -> ParseResult<E> { - expect(self.get_pos_opt()) - } - - /// Extract the first positional argument. - pub fn get_pos_opt<E: ExpressionKind>(&mut self) -> ParseResult<Option<E>> { - Ok(if !self.positional.items.is_empty() { - let spanned = self.positional.items.remove(0); - Some(E::from_expr(spanned)?) - } else { - None - }) - } - - /// Add a keyword argument. - pub fn add_key(&mut self, key: Spanned<Ident>, value: Spanned<Expression>) { - self.keyword.add(key, value); - } - - /// Add a keyword argument from an existing pair. - pub fn add_key_pair(&mut self, pair: Pair) { - self.keyword.add_pair(pair); - } - - /// Force-extract a keyword argument. - pub fn get_key<E: ExpressionKind>(&mut self, name: &str) -> ParseResult<E> { - expect(self.get_key_opt(name)) - } - - /// Extract a keyword argument. - pub fn get_key_opt<E: ExpressionKind>(&mut self, name: &str) -> ParseResult<Option<E>> { - self.keyword.pairs.iter() - .position(|p| p.key.v.0 == name) - .map(|index| { - let value = self.keyword.pairs.swap_remove(index).value; - E::from_expr(value) - }) - .transpose() - } - - /// Iterator over positional arguments. - pub fn iter_pos(&mut self) -> std::vec::IntoIter<Spanned<Expression>> { - let tuple = std::mem::replace(&mut self.positional, Tuple::new()); - tuple.items.into_iter() - } - - /// Iterator over all keyword arguments. - pub fn iter_keys(&mut self) -> std::vec::IntoIter<Pair> { - let object = std::mem::replace(&mut self.keyword, Object::new()); - object.pairs.into_iter() - } - - /// Clear the argument lists. - pub fn clear(&mut self) { - self.positional.items.clear(); - self.keyword.pairs.clear(); - } - - /// Whether both the positional and keyword argument lists are empty. - pub fn is_empty(&self) -> bool { - self.positional.items.is_empty() && self.keyword.pairs.is_empty() - } -} - -/// Extract the option expression kind from the option or return an error. -fn expect<E: ExpressionKind>(opt: ParseResult<Option<E>>) -> ParseResult<E> { - match opt { - Ok(Some(spanned)) => Ok(spanned), - Ok(None) => error!("expected {}", E::NAME), - Err(e) => Err(e), + fn bound_clone(&self) -> Box<dyn Model> { + Box::new(self.clone()) } } -#[derive(Debug, Clone, Eq, PartialEq, Serialize)] -pub struct Colorization { - pub tokens: Vec<Spanned<ColorToken>>, +/// A tree representation of source code. +#[derive(Debug, Clone, PartialEq)] +pub struct SyntaxModel { + pub nodes: SpanVec<Node>, } -impl Colorization { - pub fn new() -> Colorization { - Colorization { tokens: vec![] } - } - - pub fn add(&mut self, token: ColorToken, span: Span) { - self.tokens.push(Spanned { v: token, span }); +impl SyntaxModel { + /// Create an empty syntax model. + pub fn new() -> SyntaxModel { + SyntaxModel { nodes: vec![] } } - pub fn replace_last(&mut self, token: ColorToken) { - self.tokens.last_mut().expect("replace_last: no token").v = token; + /// Add a node to the model. + pub fn add(&mut self, node: Spanned<Node>) { + self.nodes.push(node); } } -/// Entities which can be colored by syntax highlighting. -#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub enum ColorToken { - Comment, - Bracket, - FuncName, - Colon, - Key, - Equals, - Comma, - Paren, - Brace, - ExprIdent, - ExprStr, - ExprNumber, - ExprSize, - ExprBool, - Bold, - Italic, - Monospace, - Invalid, -} - -#[derive(Debug, Clone, Eq, PartialEq, Serialize)] -pub struct ErrorMap { - pub errors: Vec<Spanned<String>>, -} - -impl ErrorMap { - pub fn new() -> ErrorMap { - ErrorMap { errors: vec![] } +#[async_trait::async_trait(?Send)] +impl Model for SyntaxModel { + async fn layout<'a>(&'a self, ctx: LayoutContext<'_, '_>) -> Layouted<Commands<'a>> { + Layouted { + output: vec![Command::LayoutSyntaxModel(self)], + errors: vec![], + } } +} - pub fn add(&mut self, message: impl Into<String>, span: Span) { - self.errors.push(Spanned { v: message.into(), span }); +/// A node in the syntax tree. +#[derive(Debug, Clone)] +pub enum Node { + /// A number of whitespace characters containing less than two newlines. + Space, + /// Whitespace characters with more than two newlines. + Newline, + /// Plain text. + Text(String), + /// Italics enabled / disabled. + ToggleItalic, + /// Bolder enabled / disabled. + ToggleBolder, + /// Monospace enabled / disabled. + ToggleMonospace, + /// A submodel. + Model(Box<dyn Model>), +} + +impl PartialEq for Node { + fn eq(&self, other: &Node) -> bool { + use Node::*; + match (self, other) { + (Space, Space) => true, + (Newline, Newline) => true, + (Text(a), Text(b)) => a == b, + (ToggleItalic, ToggleItalic) => true, + (ToggleBolder, ToggleBolder) => true, + (ToggleMonospace, ToggleMonospace) => true, + (Model(a), Model(b)) => a == b, + _ => false, + } } +} - pub fn add_at(&mut self, message: impl Into<String>, pos: Position) { - self.errors.push(Spanned { v: message.into(), span: Span::at(pos) }) - } +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub enum Decoration { + ValidFuncName, + InvalidFuncName, + ArgumentKey, } diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs index f6d0b629..24bef7ce 100644 --- a/src/syntax/parsing.rs +++ b/src/syntax/parsing.rs @@ -3,9 +3,8 @@ use super::*; use Token::*; -/// Parses source code into a syntax tree given a context. -pub fn parse(src: &str, ctx: ParseContext) -> (SyntaxTree, Colorization, ErrorMap) { - Parser::new(src, ctx).parse() +pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed<SyntaxModel> { + Parser::new(start, src, ctx).parse() } /// The context for parsing. @@ -18,64 +17,68 @@ pub struct ParseContext<'a> { struct Parser<'s> { src: &'s str, ctx: ParseContext<'s>, - colorization: Colorization, - error_map: ErrorMap, tokens: Tokens<'s>, peeked: Option<Option<Spanned<Token<'s>>>>, position: Position, last_position: Position, + errors: SpanVec<Error>, + decorations: SpanVec<Decoration>, } impl<'s> Parser<'s> { - fn new(src: &'s str, ctx: ParseContext<'s>) -> Parser<'s> { + fn new(start: Position, src: &'s str, ctx: ParseContext<'s>) -> Parser<'s> { Parser { src, ctx, - error_map: ErrorMap::new(), - colorization: Colorization::new(), - tokens: Tokens::new(src), + tokens: tokenize(start, src), peeked: None, position: Position::ZERO, last_position: Position::ZERO, + errors: vec![], + decorations: vec![], } } /// The main parsing entrypoint. - fn parse(mut self) -> (SyntaxTree, Colorization, ErrorMap) { - let mut tree = SyntaxTree::new(); - - loop { - if let Some(spanned) = self.eat() { - match spanned.v { - LineComment(_) | BlockComment(_) => {} - - Whitespace(newlines) => { - tree.add(spanned.map_v(if newlines >= 2 { - Node::Newline - } else { - Node::Space - })); - } - - LeftBracket => { - if let Some(func) = self.parse_func() { - tree.add(func); - } - } - - Star => tree.add(spanned.map_v(Node::ToggleBolder)), - Underscore => tree.add(spanned.map_v(Node::ToggleItalic)), - Backtick => tree.add(spanned.map_v(Node::ToggleMonospace)), - Text(text) => tree.add(spanned.map_v(Node::Text(text.to_owned()))), - - _ => self.unexpected(spanned), + fn parse(mut self) -> Parsed<SyntaxModel> { + let mut model = SyntaxModel::new(); + + while let Some(token) = self.eat() { + let mut span = token.span; + let node = match token.v { + LineComment(_) | BlockComment(_) => None, + Whitespace(newlines) => Some(if newlines >= 2 { + Node::Newline + } else { + Node::Space + }), + + LeftBracket => self.parse_func().map(|spanned| { + span = spanned.span; + spanned.v + }), + + Star => Some(Node::ToggleBolder), + Underscore => Some(Node::ToggleItalic), + Backtick => Some(Node::ToggleMonospace), + Text(text) => Some(Node::Text(text.to_owned())), + + _ => { + self.unexpected(token); + None } - } else { - break; + }; + + if let Some(v) = node { + model.add(Spanned { v, span }); } } - (tree, self.colorization, self.error_map) + Parsed { + output: model, + errors: self.errors, + decorations: self.decorations, + } } /// Parses a function including header and body with the cursor starting @@ -90,12 +93,55 @@ impl<'s> Parser<'s> { self.expected_at("closing bracket", self.pos()); } - let call = self.parse_func_call(header)?; + let body = if self.peekv() == Some(LeftBracket) { + self.eat(); + + let start_index = self.tokens.index(); + let start_position = self.tokens.pos(); + + let found = self.tokens.move_to_closing_bracket(); + + let end_index = self.tokens.index(); + let end_position = self.tokens.pos(); + + let body = &self.src[start_index .. end_index]; + + self.position = end_position; + + if found { + let next = self.eat().map(Spanned::value); + debug_assert_eq!(next, Some(RightBracket)); + } else { + self.expected_at("closing bracket", self.pos()); + } + + Some(Spanned::new(body, Span::new(start_position, end_position))) + } else { + None + }; + + let header = header?; + let (parser, decoration) = match self.ctx.scope.get_parser(header.name.v.as_str()) { + Ok(parser) => (parser, Decoration::ValidFuncName), + Err(parser) => { + let error = Error::new(format!("unknown function: `{}`", header.name.v)); + self.errors.push(Spanned::new(error, header.name.span)); + (parser, Decoration::InvalidFuncName) + } + }; + + self.decorations.push(Spanned::new(decoration, header.name.span)); + + let parsed = parser(header, body, self.ctx); + self.errors.extend(offset_spans(parsed.errors, start)); + self.decorations.extend(offset_spans(parsed.decorations, start)); + + let node = Node::Model(parsed.output); let end = self.pos(); let span = Span { start, end }; - Some(Spanned { v: Node::Func(call), span }) + Some(Spanned { v: node, span }) } /// Parses a function header including the closing bracket. @@ -125,7 +171,6 @@ impl<'s> Parser<'s> { match self.peek() { Some(Spanned { v: ExprIdent(ident), span }) => { self.eat(); - self.colorization.replace_last(ColorToken::FuncName); return Some(Spanned { v: Ident(ident.to_string()), span }); } other => self.expected_found_or_at("identifier", other, self.pos()), @@ -144,8 +189,7 @@ impl<'s> Parser<'s> { match self.peekv() { Some(RightBracket) | None => break, _ => match self.parse_arg() { - Some(Arg::Pos(item)) => args.add_pos(item), - Some(Arg::Key(pair)) => args.add_key_pair(pair), + Some(arg) => args.add(arg), None => {} } } @@ -165,11 +209,11 @@ impl<'s> Parser<'s> { let ident = Ident(ident.to_string()); if let Some(Equals) = self.peekv() { - self.colorization.replace_last(ColorToken::Key); - self.eat(); self.skip_whitespace(); + self.decorations.push(Spanned::new(Decoration::ArgumentKey, span)); + self.parse_expr().map(|value| { Arg::Key(Pair { key: Spanned { v: ident, span }, @@ -251,42 +295,6 @@ impl<'s> Parser<'s> { Spanned { v: Expression::Object(Object::new()), span } } - /// Parse the body of a function invocation. - fn parse_func_call(&mut self, header: Option<FuncHeader>) -> Option<FuncCall> { - let body = if self.peekv() == Some(LeftBracket) { - self.eat(); - - let start = self.tokens.index(); - let found = self.tokens.move_to_closing_bracket(); - let end = self.tokens.index(); - - self.last_position = self.position; - self.position = self.tokens.pos(); - - let body = &self.src[start .. end]; - - if found { - let next = self.eat().map(Spanned::value); - debug_assert_eq!(next, Some(RightBracket)); - } else { - self.expected_at("closing bracket", self.pos()); - } - - Some(body) - } else { - None - }; - - let header = header?; - let parser = self.ctx.scope.get_parser(header.name.v.as_str()).or_else(|| { - let message = format!("unknown function: `{}`", header.name.v); - self.error_map.add(message, header.name.span); - None - })?; - - parser(header, body, self.ctx).ok().map(|f| FuncCall(f)) - } - /// Skip all whitespace/comment tokens. fn skip_whitespace(&mut self) { self.eat_until(|t| @@ -296,14 +304,16 @@ impl<'s> Parser<'s> { /// Add an error about an `thing` which was expected but not found at the /// given position. fn expected_at(&mut self, thing: &str, pos: Position) { - self.error_map.add_at(format!("expected {}", thing), pos); + let error = Error::new(format!("expected {}", thing)); + self.errors.push(Spanned::new(error, Span::at(pos))); } /// Add an error about an expected `thing` which was not found, showing /// what was found instead. fn expected_found(&mut self, thing: &str, found: Spanned<Token>) { let message = format!("expected {}, found {}", thing, name(found.v)); - self.error_map.add(message, found.span); + let error = Error::new(message); + self.errors.push(Spanned::new(error, found.span)); } /// Add a found-error if `found` is some and a positional error, otherwise. @@ -321,7 +331,8 @@ impl<'s> Parser<'s> { /// Add an error about an unexpected token `found`. fn unexpected(&mut self, found: Spanned<Token>) { - self.error_map.add(format!("unexpected {}", name(found.v)), found.span); + let error = Error::new(format!("unexpected {}", name(found.v))); + self.errors.push(Spanned::new(error, found.span)); } /// Consume tokens until the function returns true and only consume the last @@ -348,10 +359,6 @@ impl<'s> Parser<'s> { .unwrap_or_else(|| self.tokens.next()); if let Some(token) = token { - if let Some(color) = color(token.v) { - self.colorization.add(color, token.span); - } - self.last_position = self.position; self.position = token.span.end; } @@ -407,23 +414,3 @@ fn name(token: Token) -> &'static str { Text(_) => "invalid identifier", } } - -/// The color token corresponding to a token. -fn color(token: Token) -> Option<ColorToken> { - Some(match token { - LineComment(_) | BlockComment(_) => ColorToken::Comment, - LeftBracket | RightBracket => ColorToken::Bracket, - LeftParen | RightParen => ColorToken::Paren, - LeftBrace | RightBrace => ColorToken::Brace, - Colon => ColorToken::Colon, - Comma => ColorToken::Comma, - Equals => ColorToken::Equals, - ExprIdent(_) => ColorToken::ExprIdent, - ExprStr(_) => ColorToken::ExprStr, - ExprNumber(_) => ColorToken::ExprNumber, - ExprSize(_) => ColorToken::ExprSize, - ExprBool(_) => ColorToken::ExprBool, - StarSlash => ColorToken::Invalid, - _ => return None, - }) -} diff --git a/src/syntax/span.rs b/src/syntax/span.rs index df9a3520..eb39677e 100644 --- a/src/syntax/span.rs +++ b/src/syntax/span.rs @@ -1,6 +1,7 @@ //! Spans map elements to the part of source code they originate from. use std::fmt::{self, Debug, Display, Formatter}; +use std::ops::{Add, AddAssign}; use serde::Serialize; @@ -100,6 +101,30 @@ impl Position { } } +impl Add for Position { + type Output = Position; + + fn add(self, rhs: Position) -> Position { + if rhs.line == 0 { + Position { + line: self.line, + column: self.column + rhs.column + } + } else { + Position { + line: self.line + rhs.line, + column: rhs.column, + } + } + } +} + +impl AddAssign for Position { + fn add_assign(&mut self, other: Position) { + *self = *self + other; + } +} + impl Display for Position { fn fmt(&self, f: &mut Formatter) -> fmt::Result { write!(f, "{}:{}", self.line, self.column) @@ -107,3 +132,18 @@ impl Display for Position { } debug_display!(Position); + +/// A vector of spanned things. +pub type SpanVec<T> = Vec<Spanned<T>>; + +pub fn offset_spans<T>( + vec: SpanVec<T>, + start: Position, +) -> impl Iterator<Item=Spanned<T>> { + vec.into_iter() + .map(move |mut spanned| { + spanned.span.start += start; + spanned.span.end += start; + spanned + }) +} diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs index 69d79965..0588bc6c 100644 --- a/src/syntax/tokens.rs +++ b/src/syntax/tokens.rs @@ -1,21 +1,80 @@ use std::iter::Peekable; use std::str::Chars; +use unicode_xid::UnicodeXID; use super::*; use Token::*; use State::*; +/// A minimal semantic entity of source code. +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum Token<'s> { + /// One or more whitespace characters. The contained `usize` denotes the + /// number of newlines that were contained in the whitespace. + Whitespace(usize), + + /// A line comment with inner string contents `//<&'s str>\n`. + LineComment(&'s str), + /// A block comment with inner string contents `/*<&'s str>*/`. The comment + /// can contain nested block comments. + BlockComment(&'s str), + /// An erroneous `*/` without an opening block comment. + StarSlash, + + /// A left bracket: `[`. + LeftBracket, + /// A right bracket: `]`. + RightBracket, + + /// A left parenthesis in a function header: `(`. + LeftParen, + /// A right parenthesis in a function header: `)`. + RightParen, + /// A left brace in a function header: `{`. + LeftBrace, + /// A right brace in a function header: `}`. + RightBrace, + + /// A colon in a function header: `:`. + Colon, + /// A comma in a function header: `:`. + Comma, + /// An equals sign in a function header: `=`. + Equals, + + /// An identifier in a function header: `center`. + ExprIdent(&'s str), + /// A quoted string in a function header: `"..."`. + ExprStr(&'s str), + /// A number in a function header: `3.14`. + ExprNumber(f64), + /// A size in a function header: `12pt`. + ExprSize(Size), + /// A boolean in a function header: `true | false`. + ExprBool(bool), + + /// A star in body-text. + Star, + /// An underscore in body-text. + Underscore, + /// A backtick in body-text. + Backtick, + + /// Any other consecutive string. + Text(&'s str), +} + /// Decomposes text into a sequence of semantic tokens. -pub fn tokenize(src: &str) -> Tokens { - Tokens::new(src) +pub fn tokenize(start: Position, src: &str) -> Tokens { + Tokens::new(start, src) } /// An iterator over the tokens of a string of source code. pub struct Tokens<'s> { src: &'s str, state: State, - stack: Vec<State>, + stack: Vec<(State, Position)>, iter: Peekable<Chars<'s>>, position: Position, index: usize, @@ -29,13 +88,13 @@ enum State { } impl<'s> Tokens<'s> { - pub fn new(src: &'s str) -> Tokens<'s> { + pub fn new(start: Position, src: &'s str) -> Tokens<'s> { Tokens { src, state: State::Body, stack: vec![], iter: src.chars().peekable(), - position: Position::ZERO, + position: start, index: 0, } } @@ -47,7 +106,7 @@ impl<'s> Tokens<'s> { } /// The line-colunn position in the source at which the last token ends and - /// next token will start. + /// next token will start. This position is pub fn pos(&self) -> Position { self.position } @@ -101,11 +160,13 @@ impl<'s> Iterator for Tokens<'s> { // Functions. '[' => { - if self.state == Header || self.state == Body { - self.stack.push(self.state); - self.state = Header; - } else { - self.state = Body; + match self.state { + Header | Body => { + self.stack.push((self.state, start)); + self.position = Position::new(0, '['.len_utf8()); + self.state = Header; + } + StartBody => self.state = Body, } LeftBracket @@ -114,7 +175,12 @@ impl<'s> Iterator for Tokens<'s> { if self.state == Header && self.peek() == Some('[') { self.state = StartBody; } else { - self.state = self.stack.pop().unwrap_or(Body); + if let Some((state, pos)) = self.stack.pop() { + self.state = state; + self.position = pos + self.position; + } else { + self.state = Body; + } } RightBracket |