summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLaurenz <laurmaedje@gmail.com>2020-01-21 17:09:31 +0100
committerLaurenz <laurmaedje@gmail.com>2020-01-21 17:09:31 +0100
commit78da2bdd5d77d1b8572e5e9da119bfa68127a3fa (patch)
tree020c8c39268690d34226eb7e33e75f86304988d6
parent1c1c994c46f7dc30ee34dbc99b02f2342c4617f3 (diff)
Decoupled function parser 🔗 [WIP]
-rw-r--r--.gitignore1
-rw-r--r--src/bin/main.rs2
-rw-r--r--src/error.rs16
-rw-r--r--src/func/macros.rs22
-rw-r--r--src/func/mod.rs13
-rw-r--r--src/layout/mod.rs94
-rw-r--r--src/layout/model.rs113
-rw-r--r--src/lib.rs3
-rw-r--r--src/library/maps/mod.rs8
-rw-r--r--src/library/mod.rs12
-rw-r--r--src/syntax/expr.rs105
-rw-r--r--src/syntax/func.rs8
-rw-r--r--src/syntax/mod.rs21
-rw-r--r--src/syntax/parsing.rs374
-rw-r--r--src/syntax/span.rs46
-rw-r--r--src/syntax/tokens.rs180
-rw-r--r--tests/layouter/test.typ1
-rw-r--r--tests/parser/tokens.rs123
-rw-r--r--tests/src/layouter.rs59
19 files changed, 595 insertions, 606 deletions
diff --git a/.gitignore b/.gitignore
index cf34a459..1c9ae1a2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,4 @@
**/*.rs.bk
Cargo.lock
tests/cache
+_things
diff --git a/src/bin/main.rs b/src/bin/main.rs
index 231ac487..8cb7c055 100644
--- a/src/bin/main.rs
+++ b/src/bin/main.rs
@@ -41,7 +41,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
let provider = FileSystemFontProvider::from_index("../fonts/index.json").unwrap();
typesetter.add_font_provider(provider);
- let layouts = block_on(typesetter.typeset(&src))?;
+ let layouts = block_on(typesetter.typeset(&src));
let exporter = PdfExporter::new();
let writer = BufWriter::new(File::create(&dest)?);
diff --git a/src/error.rs b/src/error.rs
index ed43818f..55156038 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -1,10 +1,20 @@
-#[derive(Debug, Clone, Eq, PartialEq)]
+use serde::Serialize;
+
+
+#[derive(Debug, Clone, Eq, PartialEq, Serialize)]
pub struct Error {
pub message: String,
+ pub severity: Severity,
}
impl Error {
- pub fn new(message: impl Into<String>) -> Error {
- Error { message: message.into() }
+ pub fn new(message: impl Into<String>, severity: Severity) -> Error {
+ Error { message: message.into(), severity }
}
}
+
+#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize)]
+pub enum Severity {
+ Warning,
+ Error,
+}
diff --git a/src/func/macros.rs b/src/func/macros.rs
index 1e92d735..92b6c16e 100644
--- a/src/func/macros.rs
+++ b/src/func/macros.rs
@@ -22,7 +22,7 @@ macro_rules! function {
// Parse trait.
(@parse($($a:tt)*) parse(default) $($r:tt)*) => {
- function!(@parse($($a)*) parse() { Default::default() } $($r)*);
+ function!(@parse($($a)*) parse(_h, _b, _c, _e, _d, _m) {Default::default() } $($r)*);
};
(@parse($($a:tt)*) parse($h:ident, $b:ident, $c:ident, $e:ident, $d:ident) $($r:tt)* ) => {
function!(@parse($($a)*) parse($h, $b, $c, $e, $d, _metadata) $($r)*);
@@ -40,7 +40,7 @@ macro_rules! function {
fn parse(
#[allow(unused)] mut $header: FuncHeader,
- #[allow(unused)] $body: Option<Spanned<&str>>,
+ #[allow(unused)] $body: Option<(Position, &str)>,
#[allow(unused)] $ctx: ParseContext,
#[allow(unused)] $metadata: Self::Meta,
) -> Parsed<Self> where Self: Sized {
@@ -85,7 +85,7 @@ macro_rules! function {
macro_rules! body {
(opt: $body:expr, $ctx:expr, $errors:expr, $decos:expr) => ({
$body.map(|body| {
- let parsed = $crate::syntax::parse(body.span.start, body.v, $ctx);
+ let parsed = $crate::syntax::parse(body.0, body.1, $ctx);
$errors.extend(parsed.errors);
$decos.extend(parsed.decorations);
parsed.output
@@ -99,13 +99,19 @@ macro_rules! body {
};
}
-/// Construct a spanned error.
+/// Construct an error with an optional span.
#[macro_export]
macro_rules! err {
- ($span:expr, $($args:tt)*) => {
- $crate::syntax::Spanned {
- v: $crate::error::Error::new(format!($($args)*)),
- span: $span,
+ (@$severity:ident: $span:expr; $($args:tt)*) => {
+ $crate::syntax::Spanned { v: err!(@Error: $($args)*), span: $span }
+ };
+
+ (@$severity:ident: $($args:tt)*) => {
+ $crate::error::Error {
+ message: format!($($args)*),
+ severity: $crate::error::Severity::$severity,
}
};
+
+ ($($tts:tt)*) => { err!(@Error: $($tts)*) };
}
diff --git a/src/func/mod.rs b/src/func/mod.rs
index 1ca226c3..e3399903 100644
--- a/src/func/mod.rs
+++ b/src/func/mod.rs
@@ -1,6 +1,5 @@
//! Dynamic typesetting functions.
-use std::any::Any;
use std::collections::HashMap;
use std::fmt::{self, Debug, Formatter};
@@ -12,6 +11,7 @@ mod macros;
/// Useful imports for creating your own functions.
pub mod prelude {
pub use super::{Scope, Parse, Command, Commands};
+ pub use crate::error::Error;
pub use crate::layout::prelude::*;
pub use crate::syntax::prelude::*;
pub use crate::size::{Size, Size2D, SizeBox, ValueBox, ScaleSize, FSize, PSize};
@@ -26,7 +26,7 @@ pub trait Parse {
/// Parse the header and body into this function given a context.
fn parse(
header: FuncHeader,
- body: Option<Spanned<&str>>,
+ body: Option<(Position, &str)>,
ctx: ParseContext,
metadata: Self::Meta,
) -> Parsed<Self> where Self: Sized;
@@ -36,7 +36,7 @@ pub trait Parse {
/// implements [`Model`].
type Parser = dyn Fn(
FuncHeader,
- Option<Spanned<&str>>,
+ Option<(Position, &str)>,
ParseContext,
) -> Parsed<Box<dyn Model>>;
@@ -102,11 +102,16 @@ impl Scope {
}
/// Return the parser with the given name if there is one.
- pub(crate) fn get_parser(&self, name: &str) -> Result<&Parser, &Parser> {
+ pub fn get_parser(&self, name: &str) -> Result<&Parser, &Parser> {
self.parsers.get(name)
.map(|x| &**x)
.ok_or_else(|| &*self.fallback)
}
+
+ /// Return the fallback parser.
+ pub fn get_fallback_parser(&self) -> &Parser {
+ &*self.fallback
+ }
}
impl Debug for Scope {
diff --git a/src/layout/mod.rs b/src/layout/mod.rs
index 75d34409..1cc16a26 100644
--- a/src/layout/mod.rs
+++ b/src/layout/mod.rs
@@ -5,7 +5,7 @@ use smallvec::SmallVec;
use toddle::query::{SharedFontLoader, FontIndex};
use crate::error::Error;
-use crate::syntax::SpanVec;
+use crate::syntax::{SyntaxModel, SpanVec};
use crate::size::{Size, Size2D, SizeBox};
use crate::style::LayoutStyle;
@@ -26,7 +26,7 @@ pub mod prelude {
/// Different kinds of layouters (fully re-exported).
pub mod layouters {
- pub use super::model::layout;
+ pub use super::model::ModelLayouter;
pub use super::line::{LineLayouter, LineContext};
pub use super::stack::{StackLayouter, StackContext};
pub use super::text::{layout_text, TextContext};
@@ -37,20 +37,6 @@ pub use self::layouters::*;
pub use self::prelude::*;
-pub struct Layouted<T> {
- pub output: T,
- pub errors: SpanVec<Error>,
-}
-
-impl<T> Layouted<T> {
- pub fn map<F, U>(self, f: F) -> Layouted<U> where F: FnOnce(T) -> U {
- Layouted {
- output: f(self.output),
- errors: self.errors,
- }
- }
-}
-
/// A collection of layouts.
pub type MultiLayout = Vec<Layout>;
@@ -80,34 +66,6 @@ impl Layout {
}
}
-/// Layout components that can be serialized.
-pub trait Serialize {
- /// Serialize the data structure into an output writable.
- fn serialize<W: Write>(&self, f: &mut W) -> io::Result<()>;
-}
-
-impl Serialize for Layout {
- fn serialize<W: Write>(&self, f: &mut W) -> io::Result<()> {
- writeln!(f, "{:.4} {:.4}", self.dimensions.x.to_pt(), self.dimensions.y.to_pt())?;
- writeln!(f, "{}", self.actions.len())?;
- for action in &self.actions {
- action.serialize(f)?;
- writeln!(f)?;
- }
- Ok(())
- }
-}
-
-impl Serialize for MultiLayout {
- fn serialize<W: Write>(&self, f: &mut W) -> io::Result<()> {
- writeln!(f, "{}", self.len())?;
- for layout in self {
- layout.serialize(f)?;
- }
- Ok(())
- }
-}
-
/// The general context for layouting.
#[derive(Debug, Clone)]
pub struct LayoutContext<'a, 'p> {
@@ -133,6 +91,26 @@ pub struct LayoutContext<'a, 'p> {
pub debug: bool,
}
+pub struct Layouted<T> {
+ pub output: T,
+ pub errors: SpanVec<Error>,
+}
+
+impl<T> Layouted<T> {
+ pub fn map<F, U>(self, f: F) -> Layouted<U> where F: FnOnce(T) -> U {
+ Layouted {
+ output: f(self.output),
+ errors: self.errors,
+ }
+ }
+}
+
+pub async fn layout(model: &SyntaxModel, ctx: LayoutContext<'_, '_>) -> Layouted<MultiLayout> {
+ let mut layouter = ModelLayouter::new(ctx);
+ layouter.layout_syntax_model(model).await;
+ layouter.finish()
+}
+
/// A possibly stack-allocated vector of layout spaces.
pub type LayoutSpaces = SmallVec<[LayoutSpace; 2]>;
@@ -397,3 +375,31 @@ impl LastSpacing {
}
}
}
+
+/// Layout components that can be serialized.
+pub trait Serialize {
+ /// Serialize the data structure into an output writable.
+ fn serialize<W: Write>(&self, f: &mut W) -> io::Result<()>;
+}
+
+impl Serialize for Layout {
+ fn serialize<W: Write>(&self, f: &mut W) -> io::Result<()> {
+ writeln!(f, "{:.4} {:.4}", self.dimensions.x.to_pt(), self.dimensions.y.to_pt())?;
+ writeln!(f, "{}", self.actions.len())?;
+ for action in &self.actions {
+ action.serialize(f)?;
+ writeln!(f)?;
+ }
+ Ok(())
+ }
+}
+
+impl Serialize for MultiLayout {
+ fn serialize<W: Write>(&self, f: &mut W) -> io::Result<()> {
+ writeln!(f, "{}", self.len())?;
+ for layout in self {
+ layout.serialize(f)?;
+ }
+ Ok(())
+ }
+}
diff --git a/src/layout/model.rs b/src/layout/model.rs
index bcec5ceb..73492dd9 100644
--- a/src/layout/model.rs
+++ b/src/layout/model.rs
@@ -1,5 +1,3 @@
-use std::pin::Pin;
-use std::future::Future;
use smallvec::smallvec;
use crate::error::Error;
@@ -9,17 +7,8 @@ use crate::syntax::{SpanVec, Spanned, Span, offset_spans};
use super::*;
-pub async fn layout(
- model: &SyntaxModel,
- ctx: LayoutContext<'_, '_>
-) -> Layouted<MultiLayout> {
- let mut layouter = ModelLayouter::new(ctx);
- layouter.layout_syntax_model(model).await;
- layouter.finish()
-}
-
#[derive(Debug, Clone)]
-struct ModelLayouter<'a, 'p> {
+pub struct ModelLayouter<'a, 'p> {
ctx: LayoutContext<'a, 'p>,
layouter: LineLayouter,
style: LayoutStyle,
@@ -28,7 +17,7 @@ struct ModelLayouter<'a, 'p> {
impl<'a, 'p> ModelLayouter<'a, 'p> {
/// Create a new syntax tree layouter.
- fn new(ctx: LayoutContext<'a, 'p>) -> ModelLayouter<'a, 'p> {
+ pub fn new(ctx: LayoutContext<'a, 'p>) -> ModelLayouter<'a, 'p> {
ModelLayouter {
layouter: LineLayouter::new(LineContext {
spaces: ctx.spaces.clone(),
@@ -44,7 +33,7 @@ impl<'a, 'p> ModelLayouter<'a, 'p> {
}
}
- fn layout<'r>(
+ pub fn layout<'r>(
&'r mut self,
model: Spanned<&'r dyn Model>
) -> DynFuture<'r, ()> { Box::pin(async move {
@@ -60,14 +49,54 @@ impl<'a, 'p> ModelLayouter<'a, 'p> {
self.errors.extend(offset_spans(layouted.errors, model.span.start));
for command in commands {
- self.execute_command(command, model.span);
+ self.execute_command(command, model.span).await;
}
}) }
+ pub fn layout_syntax_model<'r>(
+ &'r mut self,
+ model: &'r SyntaxModel
+ ) -> DynFuture<'r, ()> { Box::pin(async move {
+ use Node::*;
+
+ for node in &model.nodes {
+ match &node.v {
+ Space => self.layout_space(),
+ Newline => self.layout_paragraph(),
+ Text(text) => self.layout_text(text).await,
+
+ ToggleItalic => self.style.text.variant.style.toggle(),
+ ToggleBolder => {
+ let fac = if self.style.text.bolder { -1 } else { 1 };
+ self.style.text.variant.weight.0 += 300 * fac;
+ self.style.text.bolder = !self.style.text.bolder;
+ }
+ ToggleMonospace => {
+ let list = &mut self.style.text.fallback.list;
+ match list.get(0).map(|s| s.as_str()) {
+ Some("monospace") => { list.remove(0); },
+ _ => list.insert(0, "monospace".to_string()),
+ }
+ }
+
+ Node::Model(model) => {
+ self.layout(Spanned::new(model.as_ref(), node.span)).await;
+ }
+ }
+ }
+ }) }
+
+ pub fn finish(self) -> Layouted<MultiLayout> {
+ Layouted {
+ output: self.layouter.finish(),
+ errors: self.errors,
+ }
+ }
+
fn execute_command<'r>(
&'r mut self,
command: Command<'r>,
- model_span: Span,
+ span: Span,
) -> DynFuture<'r, ()> { Box::pin(async move {
use Command::*;
@@ -86,10 +115,8 @@ impl<'a, 'p> ModelLayouter<'a, 'p> {
BreakParagraph => self.layout_paragraph(),
BreakPage => {
if self.ctx.nested {
- self.errors.push(Spanned::new(
- Error::new( "page break cannot be issued from nested context"),
- model_span,
- ));
+ self.errors.push(err!(span;
+ "page break cannot be issued from nested context"));
} else {
self.layouter.finish_space(true)
}
@@ -101,10 +128,8 @@ impl<'a, 'p> ModelLayouter<'a, 'p> {
}
SetPageStyle(style) => {
if self.ctx.nested {
- self.errors.push(Spanned::new(
- Error::new("page style cannot be changed from nested context"),
- model_span,
- ));
+ self.errors.push(err!(span;
+ "page style cannot be changed from nested context"));
} else {
self.style.page = style;
@@ -128,39 +153,6 @@ impl<'a, 'p> ModelLayouter<'a, 'p> {
}
}) }
- fn layout_syntax_model<'r>(
- &'r mut self,
- model: &'r SyntaxModel
- ) -> DynFuture<'r, ()> { Box::pin(async move {
- use Node::*;
-
- for node in &model.nodes {
- match &node.v {
- Space => self.layout_space(),
- Newline => self.layout_paragraph(),
- Text(text) => self.layout_text(text).await,
-
- ToggleItalic => self.style.text.variant.style.toggle(),
- ToggleBolder => {
- let fac = if self.style.text.bolder { -1 } else { 1 };
- self.style.text.variant.weight.0 += 300 * fac;
- self.style.text.bolder = !self.style.text.bolder;
- }
- ToggleMonospace => {
- let list = &mut self.style.text.fallback.list;
- match list.get(0).map(|s| s.as_str()) {
- Some("monospace") => { list.remove(0); },
- _ => list.insert(0, "monospace".to_string()),
- }
- }
-
- Node::Model(model) => {
- self.layout(Spanned::new(model.as_ref(), node.span)).await;
- }
- }
- }
- }) }
-
async fn layout_text(&mut self, text: &str) {
self.layouter.add(layout_text(text, TextContext {
loader: &self.ctx.loader,
@@ -183,11 +175,4 @@ impl<'a, 'p> ModelLayouter<'a, 'p> {
SpacingKind::PARAGRAPH,
);
}
-
- fn finish(self) -> Layouted<MultiLayout> {
- Layouted {
- output: self.layouter.finish(),
- errors: self.errors,
- }
- }
}
diff --git a/src/lib.rs b/src/lib.rs
index 2ca4d55a..3ebe34c2 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -24,10 +24,9 @@ use std::cell::RefCell;
use smallvec::smallvec;
use toddle::query::{FontLoader, FontProvider, SharedFontLoader};
-use toddle::Error as FontError;
use crate::func::Scope;
-use crate::layout::{Layouted, LayoutContext, MultiLayout};
+use crate::layout::{Layouted, MultiLayout};
use crate::syntax::{parse, ParseContext, Parsed, SyntaxModel, Position};
use crate::style::{LayoutStyle, PageStyle, TextStyle};
diff --git a/src/library/maps/mod.rs b/src/library/maps/mod.rs
index a868ce6c..3538def7 100644
--- a/src/library/maps/mod.rs
+++ b/src/library/maps/mod.rs
@@ -21,8 +21,8 @@ macro_rules! key {
impl ExpressionKind for $type {
const NAME: &'static str = $name;
- fn from_expr(expr: Spanned<Expression>) -> ParseResult<Self> {
- if let Expression::Ident(ident) = expr.v {
+ fn from_expr(expr: Spanned<Expr>) -> ParseResult<Self> {
+ if let Expr::Ident(ident) = expr.v {
Self::from_ident(&Spanned::new(ident, expr.span))
} else {
error!("expected {}", Self::NAME);
@@ -55,8 +55,8 @@ impl<T> Into<Option<T>> for DefaultKey<T> {
impl<T> ExpressionKind for DefaultKey<T> where T: ExpressionKind {
const NAME: &'static str = T::NAME;
- fn from_expr(expr: Spanned<Expression>) -> ParseResult<DefaultKey<T>> {
- if let Expression::Ident(ident) = &expr.v {
+ fn from_expr(expr: Spanned<Expr>) -> ParseResult<DefaultKey<T>> {
+ if let Expr::Ident(ident) = &expr.v {
match ident.as_str() {
"default" => return Ok(DefaultKey::None),
_ => {},
diff --git a/src/library/mod.rs b/src/library/mod.rs
index d91f1b35..8d16ccaf 100644
--- a/src/library/mod.rs
+++ b/src/library/mod.rs
@@ -63,8 +63,8 @@ function! {
body: parse!(optional: body, ctx),
list: {
header.args.iter_pos().map(|arg| match arg.v {
- Expression::Str(s) |
- Expression::Ident(Ident(s)) => Ok(s.to_lowercase()),
+ Expr::Str(s) |
+ Expr::Ident(Ident(s)) => Ok(s.to_lowercase()),
_ => error!("expected identifier or string"),
}).collect::<LayoutResult<Vec<_>>>()?
}
@@ -117,8 +117,8 @@ function! {
parse(header, body, ctx) {
FontWeightFunc {
body: parse!(optional: body, ctx),
- weight: match header.args.get_pos::<Expression>()? {
- Expression::Number(weight) => {
+ weight: match header.args.get_pos::<Expr>()? {
+ Expr::Number(weight) => {
let weight = weight.round() as i16;
FontWeight(
if weight < 100 { 100 }
@@ -126,7 +126,7 @@ function! {
else { 900 }
)
}
- Expression::Ident(Ident(s)) => {
+ Expr::Ident(Ident(s)) => {
match FontWeight::from_str(&s) {
Some(weight) => weight,
None => error!("invalid font weight: `{}`", s),
@@ -263,7 +263,7 @@ function! {
SpacingFunc {
axis: AxisKey::Specific(axis),
spacing: FSize::from_expr(
- header.args.get_pos::<Spanned<Expression>>()?
+ header.args.get_pos::<Spanned<Expr>>()?
)?,
}
} else {
diff --git a/src/syntax/expr.rs b/src/syntax/expr.rs
index 74deda46..34a1c6bf 100644
--- a/src/syntax/expr.rs
+++ b/src/syntax/expr.rs
@@ -4,7 +4,7 @@ use super::*;
/// An argument or return value.
#[derive(Clone, PartialEq)]
-pub enum Expression {
+pub enum Expr {
Ident(Ident),
Str(String),
Number(f64),
@@ -14,9 +14,24 @@ pub enum Expression {
Object(Object),
}
-impl Display for Expression {
+impl Expr {
+ pub fn name(&self) -> &'static str {
+ use Expr::*;
+ match self {
+ Ident(_) => "identifier",
+ Str(_) => "string",
+ Number(_) => "number",
+ Size(_) => "size",
+ Bool(_) => "boolean",
+ Tuple(_) => "tuple",
+ Object(_) => "object",
+ }
+ }
+}
+
+impl Display for Expr {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
- use Expression::*;
+ use Expr::*;
match self {
Ident(i) => write!(f, "{}", i),
Str(s) => write!(f, "{:?}", s),
@@ -29,6 +44,8 @@ impl Display for Expression {
}
}
+debug_display!(Expr);
+
/// An identifier.
#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub struct Ident(pub String);
@@ -53,10 +70,15 @@ impl Display for Ident {
}
}
+debug_display!(Ident);
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct StringLike(pub String);
+
/// A sequence of expressions.
#[derive(Clone, PartialEq)]
pub struct Tuple {
- pub items: Vec<Spanned<Expression>>,
+ pub items: Vec<Spanned<Expr>>,
}
impl Tuple {
@@ -64,7 +86,7 @@ impl Tuple {
Tuple { items: vec![] }
}
- pub fn add(&mut self, item: Spanned<Expression>) {
+ pub fn add(&mut self, item: Spanned<Expr>) {
self.items.push(item);
}
}
@@ -86,6 +108,8 @@ impl Display for Tuple {
}
}
+debug_display!(Tuple);
+
/// A key-value collection of identifiers and associated expressions.
#[derive(Clone, PartialEq)]
pub struct Object {
@@ -97,7 +121,7 @@ impl Object {
Object { pairs: vec![] }
}
- pub fn add(&mut self, key: Spanned<Ident>, value: Spanned<Expression>) {
+ pub fn add(&mut self, key: Spanned<Ident>, value: Spanned<Expr>) {
self.pairs.push(Pair { key, value });
}
@@ -127,11 +151,13 @@ impl Display for Object {
}
}
+debug_display!(Object);
+
/// A key-value pair in an object.
#[derive(Clone, PartialEq)]
pub struct Pair {
pub key: Spanned<Ident>,
- pub value: Spanned<Expression>,
+ pub value: Spanned<Expr>,
}
impl Display for Pair {
@@ -140,57 +166,56 @@ impl Display for Pair {
}
}
-debug_display!(Ident);
-debug_display!(Expression);
-debug_display!(Tuple);
-debug_display!(Object);
debug_display!(Pair);
-/// Kinds of expressions.
-pub trait ExpressionKind: Sized {
+pub trait ExprKind: Sized {
/// The name of the expression in an `expected <name>` error.
const NAME: &'static str;
/// Create from expression.
- fn from_expr(expr: Spanned<Expression>) -> ParseResult<Self>;
+ fn from_expr(expr: Spanned<Expr>) -> Result<Self, Error>;
}
+impl<T> ExprKind for Spanned<T> where T: ExprKind {
+ const NAME: &'static str = T::NAME;
+
+ fn from_expr(expr: Spanned<Expr>) -> Result<Self, Error> {
+ let span = expr.span;
+ T::from_expr(expr).map(|v| Spanned { v, span })
+ }
+}
/// Implements the expression kind trait for a type.
macro_rules! kind {
- ($type:ty, $name:expr, $($patterns:tt)*) => {
- impl ExpressionKind for $type {
+ ($type:ty, $name:expr, $($p:pat => $r:expr),* $(,)?) => {
+ impl ExprKind for $type {
const NAME: &'static str = $name;
- fn from_expr(expr: Spanned<Expression>) -> ParseResult<Self> {
+ fn from_expr(expr: Spanned<Expr>) -> Result<Self, Error> {
#[allow(unreachable_patterns)]
Ok(match expr.v {
- $($patterns)*,
- _ => error!("expected {}", Self::NAME),
+ $($p => $r),*,
+ _ => return Err(
+ err!("expected {}, found {}", Self::NAME, expr.v.name())
+ ),
})
}
}
};
}
-kind!(Expression, "expression", e => e);
-kind!(Ident, "identifier", Expression::Ident(ident) => ident);
-kind!(String, "string", Expression::Str(string) => string);
-kind!(f64, "number", Expression::Number(num) => num);
-kind!(bool, "boolean", Expression::Bool(boolean) => boolean);
-kind!(Size, "size", Expression::Size(size) => size);
-kind!(Tuple, "tuple", Expression::Tuple(tuple) => tuple);
-kind!(Object, "object", Expression::Object(object) => object);
-
-kind!(ScaleSize, "number or size",
- Expression::Size(size) => ScaleSize::Absolute(size),
- Expression::Number(scale) => ScaleSize::Scaled(scale as f32)
+kind!(Expr, "expression", e => e);
+kind!(Ident, "identifier", Expr::Ident(i) => i);
+kind!(String, "string", Expr::Str(s) => s);
+kind!(f64, "number", Expr::Number(n) => n);
+kind!(bool, "boolean", Expr::Bool(b) => b);
+kind!(Size, "size", Expr::Size(s) => s);
+kind!(Tuple, "tuple", Expr::Tuple(t) => t);
+kind!(Object, "object", Expr::Object(o) => o);
+kind!(ScaleSize, "number or size",
+ Expr::Size(size) => ScaleSize::Absolute(size),
+ Expr::Number(scale) => ScaleSize::Scaled(scale as f32),
+);
+kind!(StringLike, "identifier or string",
+ Expr::Ident(Ident(s)) => StringLike(s),
+ Expr::Str(s) => StringLike(s),
);
-
-impl<T> ExpressionKind for Spanned<T> where T: ExpressionKind {
- const NAME: &'static str = T::NAME;
-
- fn from_expr(expr: Spanned<Expression>) -> ParseResult<Spanned<T>> {
- let span = expr.span;
- T::from_expr(expr).map(|v| Spanned { v, span })
- }
-}
diff --git a/src/syntax/func.rs b/src/syntax/func.rs
index 5b1ce6e8..abc8c431 100644
--- a/src/syntax/func.rs
+++ b/src/syntax/func.rs
@@ -15,7 +15,7 @@ pub struct FuncArgs {
#[derive(Debug, Clone, PartialEq)]
pub enum Arg {
- Pos(Spanned<Expression>),
+ Pos(Spanned<Expr>),
Key(Pair),
}
@@ -46,12 +46,12 @@ impl FuncArgs {
}
/// Add a positional argument.
- pub fn add_pos(&mut self, item: Spanned<Expression>) {
+ pub fn add_pos(&mut self, item: Spanned<Expr>) {
self.pos.add(item);
}
/// Add a keyword argument.
- pub fn add_key(&mut self, key: Spanned<Ident>, value: Spanned<Expression>) {
+ pub fn add_key(&mut self, key: Spanned<Ident>, value: Spanned<Expr>) {
self.key.add(key, value);
}
@@ -92,7 +92,7 @@ impl FuncArgs {
// }
// /// Iterator over positional arguments.
- // pub fn iter_pos(&mut self) -> std::vec::IntoIter<Spanned<Expression>> {
+ // pub fn iter_pos(&mut self) -> std::vec::IntoIter<Spanned<Expr>> {
// let tuple = std::mem::replace(&mut self.positional, Tuple::new());
// tuple.items.into_iter()
// }
diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs
index 75407f82..a77c764e 100644
--- a/src/syntax/mod.rs
+++ b/src/syntax/mod.rs
@@ -23,22 +23,6 @@ pub mod prelude {
}
-pub struct Parsed<T> {
- pub output: T,
- pub errors: SpanVec<Error>,
- pub decorations: SpanVec<Decoration>,
-}
-
-impl<T> Parsed<T> {
- pub fn map<F, U>(self, f: F) -> Parsed<U> where F: FnOnce(T) -> U {
- Parsed {
- output: f(self.output),
- errors: self.errors,
- decorations: self.decorations,
- }
- }
-}
-
#[async_trait::async_trait(?Send)]
pub trait Model: Debug + ModelBounds {
async fn layout<'a>(
@@ -110,7 +94,7 @@ impl SyntaxModel {
#[async_trait::async_trait(?Send)]
impl Model for SyntaxModel {
- async fn layout<'a>(&'a self, ctx: LayoutContext<'_, '_>) -> Layouted<Commands<'a>> {
+ async fn layout<'a>(&'a self, _: LayoutContext<'_, '_>) -> Layouted<Commands<'a>> {
Layouted {
output: vec![Command::LayoutSyntaxModel(self)],
errors: vec![],
@@ -153,7 +137,8 @@ impl PartialEq for Node {
}
}
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
pub enum Decoration {
ValidFuncName,
InvalidFuncName,
diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs
index 24bef7ce..ed343050 100644
--- a/src/syntax/parsing.rs
+++ b/src/syntax/parsing.rs
@@ -3,10 +3,6 @@ use super::*;
use Token::*;
-pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed<SyntaxModel> {
- Parser::new(start, src, ctx).parse()
-}
-
/// The context for parsing.
#[derive(Debug, Copy, Clone)]
pub struct ParseContext<'a> {
@@ -14,185 +10,169 @@ pub struct ParseContext<'a> {
pub scope: &'a Scope,
}
-struct Parser<'s> {
- src: &'s str,
- ctx: ParseContext<'s>,
- tokens: Tokens<'s>,
- peeked: Option<Option<Spanned<Token<'s>>>>,
- position: Position,
- last_position: Position,
- errors: SpanVec<Error>,
- decorations: SpanVec<Decoration>,
+pub struct Parsed<T> {
+ pub output: T,
+ pub errors: SpanVec<Error>,
+ pub decorations: SpanVec<Decoration>,
}
-impl<'s> Parser<'s> {
- fn new(start: Position, src: &'s str, ctx: ParseContext<'s>) -> Parser<'s> {
- Parser {
- src,
- ctx,
- tokens: tokenize(start, src),
- peeked: None,
- position: Position::ZERO,
- last_position: Position::ZERO,
- errors: vec![],
- decorations: vec![],
- }
- }
-
- /// The main parsing entrypoint.
- fn parse(mut self) -> Parsed<SyntaxModel> {
- let mut model = SyntaxModel::new();
-
- while let Some(token) = self.eat() {
- let mut span = token.span;
- let node = match token.v {
- LineComment(_) | BlockComment(_) => None,
- Whitespace(newlines) => Some(if newlines >= 2 {
- Node::Newline
- } else {
- Node::Space
- }),
-
- LeftBracket => self.parse_func().map(|spanned| {
- span = spanned.span;
- spanned.v
- }),
-
- Star => Some(Node::ToggleBolder),
- Underscore => Some(Node::ToggleItalic),
- Backtick => Some(Node::ToggleMonospace),
- Text(text) => Some(Node::Text(text.to_owned())),
-
- _ => {
- self.unexpected(token);
- None
- }
- };
-
- if let Some(v) = node {
- model.add(Spanned { v, span });
- }
- }
-
+impl<T> Parsed<T> {
+ pub fn map<F, U>(self, f: F) -> Parsed<U> where F: FnOnce(T) -> U {
Parsed {
- output: model,
+ output: f(self.output),
errors: self.errors,
decorations: self.decorations,
}
}
+}
- /// Parses a function including header and body with the cursor starting
- /// right behind the first opening bracket.
- fn parse_func(&mut self) -> Option<Spanned<Node>> {
- let start = self.last_pos();
-
- let header = self.parse_func_header();
- self.eat_until(|t| t == RightBracket, false);
+pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed<SyntaxModel> {
+ let mut model = SyntaxModel::new();
+ let mut errors = Vec::new();
+ let mut decorations = Vec::new();
- if self.eat().map(Spanned::value) != Some(RightBracket) {
- self.expected_at("closing bracket", self.pos());
- }
+ let mut tokens = Tokens::new(start, src, TokenizationMode::Body);
- let body = if self.peekv() == Some(LeftBracket) {
- self.eat();
+ while let Some(token) = tokens.next() {
+ let span = token.span;
- let start_index = self.tokens.index();
- let start_position = self.tokens.pos();
-
- let found = self.tokens.move_to_closing_bracket();
+ let node = match token.v {
+ Space(newlines) => if newlines >= 2 {
+ Node::Newline
+ } else {
+ Node::Space
+ },
- let end_index = self.tokens.index();
- let end_position = self.tokens.pos();
+ Function { header, body, terminated } => {
+ let parsed: Parsed<Node> = FuncParser::new(header, body, ctx).parse();
- let body = &self.src[start_index .. end_index];
+ errors.extend(offset_spans(parsed.errors, span.start));
+ decorations.extend(offset_spans(parsed.decorations, span.start));
- self.position = end_position;
+ if !terminated {
+ errors.push(err!(Span::at(span.end); "expected closing bracket"));
+ }
- if found {
- let next = self.eat().map(Spanned::value);
- debug_assert_eq!(next, Some(RightBracket));
- } else {
- self.expected_at("closing bracket", self.pos());
+ parsed.output
}
- Some(Spanned::new(body, Span::new(start_position, end_position)))
- } else {
- None
- };
+ Star => Node::ToggleBolder,
+ Underscore => Node::ToggleItalic,
+ Backtick => Node::ToggleMonospace,
+ Text(text) => Node::Text(text.to_owned()),
+
+ LineComment(_) | BlockComment(_) => continue,
- let header = header?;
- let (parser, decoration) = match self.ctx.scope.get_parser(header.name.v.as_str()) {
- Ok(parser) => (parser, Decoration::ValidFuncName),
- Err(parser) => {
- let error = Error::new(format!("unknown function: `{}`", header.name.v));
- self.errors.push(Spanned::new(error, header.name.span));
- (parser, Decoration::InvalidFuncName)
+ other => {
+ errors.push(err!(span; "unexpected {}", name(other)));
+ continue;
}
};
- self.decorations.push(Spanned::new(decoration, header.name.span));
+ model.add(Spanned { v: node, span: token.span });
+ }
- let parsed = parser(header, body, self.ctx);
- self.errors.extend(offset_spans(parsed.errors, start));
- self.decorations.extend(offset_spans(parsed.decorations, start));
+ Parsed { output: model, errors, decorations }
+}
- let node = Node::Model(parsed.output);
+struct FuncParser<'s> {
+ ctx: ParseContext<'s>,
+ errors: SpanVec<Error>,
+ decorations: SpanVec<Decoration>,
+ tokens: Tokens<'s>,
+ peeked: Option<Option<Spanned<Token<'s>>>>,
+ body: Option<(Position, &'s str)>,
+}
- let end = self.pos();
- let span = Span { start, end };
+impl<'s> FuncParser<'s> {
+ fn new(
+ header: &'s str,
+ body: Option<(Position, &'s str)>,
+ ctx: ParseContext<'s>
+ ) -> FuncParser<'s> {
+ FuncParser {
+ ctx,
+ errors: vec![],
+ decorations: vec![],
+ tokens: Tokens::new(Position::new(0, 1), header, TokenizationMode::Header),
+ peeked: None,
+ body,
+ }
+ }
+
+ fn parse(mut self) -> Parsed<Node> {
+ let parsed = if let Some(header) = self.parse_func_header() {
+ let name = header.name.v.as_str();
+ let (parser, deco) = match self.ctx.scope.get_parser(name) {
+ Ok(parser) => (parser, Decoration::ValidFuncName),
+ Err(parser) => {
+ self.errors.push(err!(header.name.span; "unknown function"));
+ (parser, Decoration::InvalidFuncName)
+ }
+ };
- Some(Spanned { v: node, span })
+ self.decorations.push(Spanned::new(deco, header.name.span));
+
+ parser(header, self.body, self.ctx)
+ } else {
+ let default = FuncHeader {
+ name: Spanned::new(Ident("".to_string()), Span::ZERO),
+ args: FuncArgs::new(),
+ };
+
+ // Use the fallback function such that the body is still rendered
+ // even if the header is completely unparsable.
+ self.ctx.scope.get_fallback_parser()(default, self.body, self.ctx)
+ };
+
+ self.errors.extend(parsed.errors);
+ self.decorations.extend(parsed.decorations);
+
+ Parsed {
+ output: Node::Model(parsed.output),
+ errors: self.errors,
+ decorations: self.decorations,
+ }
}
- /// Parses a function header including the closing bracket.
fn parse_func_header(&mut self) -> Option<FuncHeader> {
+ let start = self.pos();
self.skip_whitespace();
- let name = self.parse_func_name()?;
- self.skip_whitespace();
- let args = match self.peek() {
- Some(Spanned { v: Colon, .. }) => {
- self.eat();
- self.parse_func_args()
+ let name = match self.eat() {
+ Some(Spanned { v: ExprIdent(ident), span }) => {
+ Spanned { v: Ident(ident.to_string()), span }
}
- Some(Spanned { v: RightBracket, .. }) => FuncArgs::new(),
other => {
- self.expected_at("colon or closing bracket", name.span.end);
- FuncArgs::new()
+ self.expected_found_or_at("identifier", other, start);
+ return None;
}
};
- Some(FuncHeader { name, args })
- }
-
- /// Parses the function name if is the next token. Otherwise, it adds an
- /// error and returns `None`.
- fn parse_func_name(&mut self) -> Option<Spanned<Ident>> {
- match self.peek() {
- Some(Spanned { v: ExprIdent(ident), span }) => {
- self.eat();
- return Some(Spanned { v: Ident(ident.to_string()), span });
+ self.skip_whitespace();
+ let args = match self.eat().map(Spanned::value) {
+ Some(Colon) => self.parse_func_args(),
+ Some(_) => {
+ self.expected_at("colon", name.span.end);
+ FuncArgs::new()
}
- other => self.expected_found_or_at("identifier", other, self.pos()),
- }
+ None => FuncArgs::new(),
+ };
- None
+ Some(FuncHeader { name, args })
}
- /// Parses the function arguments and stops right before the final closing
- /// bracket.
fn parse_func_args(&mut self) -> FuncArgs {
let mut args = FuncArgs::new();
- loop {
- self.skip_whitespace();
- match self.peekv() {
- Some(RightBracket) | None => break,
- _ => match self.parse_arg() {
- Some(arg) => args.add(arg),
- None => {}
- }
+ self.skip_whitespace();
+ while self.peek().is_some() {
+ match self.parse_arg() {
+ Some(arg) => args.add(arg),
+ None => {}
}
+
+ self.skip_whitespace();
}
args
@@ -221,7 +201,7 @@ impl<'s> Parser<'s> {
})
})
} else {
- Some(Arg::Pos(Spanned::new(Expression::Ident(ident), span)))
+ Some(Arg::Pos(Spanned::new(Expr::Ident(ident), span)))
}
} else {
self.parse_expr().map(|expr| Arg::Pos(expr))
@@ -230,7 +210,6 @@ impl<'s> Parser<'s> {
if let Some(arg) = &arg {
self.skip_whitespace();
match self.peekv() {
- Some(RightBracket) => {}
Some(Comma) => { self.eat(); }
Some(_) => self.expected_at("comma", arg.span().end),
_ => {}
@@ -244,19 +223,27 @@ impl<'s> Parser<'s> {
}
/// Parse a atomic or compound (tuple / object) expression.
- fn parse_expr(&mut self) -> Option<Spanned<Expression>> {
+ fn parse_expr(&mut self) -> Option<Spanned<Expr>> {
let first = self.peek()?;
- let mut expr = |v| {
- self.eat();
- Spanned { v, span: first.span }
- };
+ let spanned = |v| Spanned { v, span: first.span };
Some(match first.v {
- ExprIdent(i) => expr(Expression::Ident(Ident(i.to_string()))),
- ExprStr(s) => expr(Expression::Str(s.to_string())),
- ExprNumber(n) => expr(Expression::Number(n)),
- ExprSize(s) => expr(Expression::Size(s)),
- ExprBool(b) => expr(Expression::Bool(b)),
+ ExprIdent(i) => {
+ self.eat();
+ spanned(Expr::Ident(Ident(i.to_string())))
+ }
+ ExprStr { string, terminated } => {
+ if !terminated {
+ self.expected_at("quote", first.span.end);
+ }
+
+ self.eat();
+ spanned(Expr::Str(string.to_string()))
+ }
+ ExprNumber(n) => { self.eat(); spanned(Expr::Number(n)) }
+ ExprSize(s) => { self.eat(); spanned(Expr::Size(s)) }
+ ExprBool(b) => { self.eat(); spanned(Expr::Bool(b)) }
+
LeftParen => self.parse_tuple(),
LeftBrace => self.parse_object(),
_ => return None,
@@ -264,56 +251,48 @@ impl<'s> Parser<'s> {
}
/// Parse a tuple expression.
- fn parse_tuple(&mut self) -> Spanned<Expression> {
+ fn parse_tuple(&mut self) -> Spanned<Expr> {
let start = self.pos();
// TODO: Do the thing.
- self.eat_until(|t| matches!(t, RightParen | RightBracket), false);
- if self.peekv() == Some(RightParen) {
- self.eat();
- }
+ self.eat_until(|t| t == RightParen, true);
let end = self.pos();
let span = Span { start, end };
- Spanned { v: Expression::Tuple(Tuple::new()), span }
+ Spanned { v: Expr::Tuple(Tuple::new()), span }
}
/// Parse an object expression.
- fn parse_object(&mut self) -> Spanned<Expression> {
+ fn parse_object(&mut self) -> Spanned<Expr> {
let start = self.pos();
// TODO: Do the thing.
- self.eat_until(|t| matches!(t, RightBrace | RightBracket), false);
- if self.peekv() == Some(RightBrace) {
- self.eat();
- }
+ self.eat_until(|t| t == RightBrace, true);
let end = self.pos();
let span = Span { start, end };
- Spanned { v: Expression::Object(Object::new()), span }
+ Spanned { v: Expr::Object(Object::new()), span }
}
/// Skip all whitespace/comment tokens.
fn skip_whitespace(&mut self) {
self.eat_until(|t|
- !matches!(t, Whitespace(_) | LineComment(_) | BlockComment(_)), false)
- }
-
- /// Add an error about an `thing` which was expected but not found at the
- /// given position.
- fn expected_at(&mut self, thing: &str, pos: Position) {
- let error = Error::new(format!("expected {}", thing));
- self.errors.push(Spanned::new(error, Span::at(pos)));
+ !matches!(t, Space(_) | LineComment(_) | BlockComment(_)), false)
}
/// Add an error about an expected `thing` which was not found, showing
/// what was found instead.
fn expected_found(&mut self, thing: &str, found: Spanned<Token>) {
- let message = format!("expected {}, found {}", thing, name(found.v));
- let error = Error::new(message);
- self.errors.push(Spanned::new(error, found.span));
+ self.errors.push(err!(found.span;
+ "expected {}, found {}", thing, name(found.v)));
+ }
+
+ /// Add an error about an `thing` which was expected but not found at the
+ /// given position.
+ fn expected_at(&mut self, thing: &str, pos: Position) {
+ self.errors.push(err!(Span::at(pos); "expected {}", thing));
}
/// Add a found-error if `found` is some and a positional error, otherwise.
@@ -329,12 +308,6 @@ impl<'s> Parser<'s> {
}
}
- /// Add an error about an unexpected token `found`.
- fn unexpected(&mut self, found: Spanned<Token>) {
- let error = Error::new(format!("unexpected {}", name(found.v)));
- self.errors.push(Spanned::new(error, found.span));
- }
-
/// Consume tokens until the function returns true and only consume the last
/// token if instructed to.
fn eat_until<F>(&mut self, mut f: F, eat_match: bool)
@@ -351,19 +324,10 @@ impl<'s> Parser<'s> {
}
}
- /// Consume and return the next token, update positions and colorize the
- /// token. All colorable tokens are per default colorized here, to override
- /// a colorization use `Colorization::replace_last`.
+ /// Consume and return the next token.
fn eat(&mut self) -> Option<Spanned<Token<'s>>> {
- let token = self.peeked.take()
- .unwrap_or_else(|| self.tokens.next());
-
- if let Some(token) = token {
- self.last_position = self.position;
- self.position = token.span.end;
- }
-
- token
+ self.peeked.take()
+ .unwrap_or_else(|| self.tokens.next())
}
/// Peek at the next token without consuming it.
@@ -379,23 +343,18 @@ impl<'s> Parser<'s> {
/// The position at the end of the last eat token / start of the peekable
/// token.
fn pos(&self) -> Position {
- self.position
- }
-
- /// The position at the start of the last eaten token.
- fn last_pos(&self) -> Position {
- self.last_position
+ self.peeked.flatten()
+ .map(|s| s.span.start)
+ .unwrap_or_else(|| self.tokens.pos())
}
}
-/// The name of a token in an `expected <...>` error.
+/// The name of a token in an `(un)expected <...>` error.
fn name(token: Token) -> &'static str {
match token {
- Whitespace(_) => "whitespace",
+ Space(_) => "space",
LineComment(_) | BlockComment(_) => "comment",
- StarSlash => "end of block comment",
- LeftBracket => "opening bracket",
- RightBracket => "closing bracket",
+ Function { .. } => "function",
LeftParen => "opening paren",
RightParen => "closing paren",
LeftBrace => "opening brace",
@@ -404,13 +363,16 @@ fn name(token: Token) -> &'static str {
Comma => "comma",
Equals => "equals sign",
ExprIdent(_) => "identifier",
- ExprStr(_) => "string",
+ ExprStr { .. } => "string",
ExprNumber(_) => "number",
ExprSize(_) => "size",
- ExprBool(_) => "bool",
+ ExprBool(_) => "boolean",
Star => "star",
Underscore => "underscore",
Backtick => "backtick",
Text(_) => "invalid identifier",
+ Invalid("]") => "closing bracket",
+ Invalid("*/") => "end of block comment",
+ Invalid(_) => "invalid token",
}
}
diff --git a/src/syntax/span.rs b/src/syntax/span.rs
index eb39677e..e049861f 100644
--- a/src/syntax/span.rs
+++ b/src/syntax/span.rs
@@ -1,7 +1,7 @@
//! Spans map elements to the part of source code they originate from.
use std::fmt::{self, Debug, Display, Formatter};
-use std::ops::{Add, AddAssign};
+use std::ops::{Add, Sub};
use serde::Serialize;
@@ -21,12 +21,13 @@ impl<T> Spanned<T> {
self.v
}
- pub fn map<F, V>(self, f: F) -> Spanned<V> where F: FnOnce(T) -> V {
+ pub fn map<V, F>(self, f: F) -> Spanned<V> where F: FnOnce(T) -> V {
Spanned { v: f(self.v), span: self.span }
}
- pub fn map_v<V>(&self, new_v: V) -> Spanned<V> {
- Spanned { v: new_v, span: self.span }
+ pub fn map_span<F>(mut self, f: F) -> Spanned<T> where F: FnOnce(Span) -> Span {
+ self.span = f(self.span);
+ self
}
}
@@ -74,6 +75,13 @@ impl Span {
pub fn expand(&mut self, other: Span) {
*self = Span::merge(*self, other)
}
+
+ pub fn offset(self, start: Position) -> Span {
+ Span {
+ start: start + self.start,
+ end: start + self.end,
+ }
+ }
}
impl Display for Span {
@@ -119,9 +127,21 @@ impl Add for Position {
}
}
-impl AddAssign for Position {
- fn add_assign(&mut self, other: Position) {
- *self = *self + other;
+impl Sub for Position {
+ type Output = Position;
+
+ fn sub(self, rhs: Position) -> Position {
+ if self.line == rhs.line {
+ Position {
+ line: 0,
+ column: self.column - rhs.column
+ }
+ } else {
+ Position {
+ line: self.line - rhs.line,
+ column: self.column,
+ }
+ }
}
}
@@ -136,14 +156,6 @@ debug_display!(Position);
/// A vector of spanned things.
pub type SpanVec<T> = Vec<Spanned<T>>;
-pub fn offset_spans<T>(
- vec: SpanVec<T>,
- start: Position,
-) -> impl Iterator<Item=Spanned<T>> {
- vec.into_iter()
- .map(move |mut spanned| {
- spanned.span.start += start;
- spanned.span.end += start;
- spanned
- })
+pub fn offset_spans<T>(vec: SpanVec<T>, start: Position) -> impl Iterator<Item=Spanned<T>> {
+ vec.into_iter().map(move |s| s.map_span(|span| span.offset(start)))
}
diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs
index 0588bc6c..6c8e736c 100644
--- a/src/syntax/tokens.rs
+++ b/src/syntax/tokens.rs
@@ -4,7 +4,7 @@ use unicode_xid::UnicodeXID;
use super::*;
use Token::*;
-use State::*;
+use TokenizationMode::*;
/// A minimal semantic entity of source code.
@@ -12,20 +12,20 @@ use State::*;
pub enum Token<'s> {
/// One or more whitespace characters. The contained `usize` denotes the
/// number of newlines that were contained in the whitespace.
- Whitespace(usize),
+ Space(usize),
/// A line comment with inner string contents `//<&'s str>\n`.
LineComment(&'s str),
/// A block comment with inner string contents `/*<&'s str>*/`. The comment
/// can contain nested block comments.
BlockComment(&'s str),
- /// An erroneous `*/` without an opening block comment.
- StarSlash,
- /// A left bracket: `[`.
- LeftBracket,
- /// A right bracket: `]`.
- RightBracket,
+ /// A function invocation `[<header>][<body>]`.
+ Function {
+ header: &'s str,
+ body: Option<(Position, &'s str)>,
+ terminated: bool,
+ },
/// A left parenthesis in a function header: `(`.
LeftParen,
@@ -46,7 +46,7 @@ pub enum Token<'s> {
/// An identifier in a function header: `center`.
ExprIdent(&'s str),
/// A quoted string in a function header: `"..."`.
- ExprStr(&'s str),
+ ExprStr { string: &'s str, terminated: bool },
/// A number in a function header: `3.14`.
ExprNumber(f64),
/// A size in a function header: `12pt`.
@@ -63,36 +63,31 @@ pub enum Token<'s> {
/// Any other consecutive string.
Text(&'s str),
-}
-/// Decomposes text into a sequence of semantic tokens.
-pub fn tokenize(start: Position, src: &str) -> Tokens {
- Tokens::new(start, src)
+ /// Things that are not valid in the context they appeared in.
+ Invalid(&'s str),
}
/// An iterator over the tokens of a string of source code.
pub struct Tokens<'s> {
src: &'s str,
- state: State,
- stack: Vec<(State, Position)>,
+ mode: TokenizationMode,
iter: Peekable<Chars<'s>>,
position: Position,
index: usize,
}
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
-enum State {
+pub enum TokenizationMode {
Header,
- StartBody,
Body,
}
impl<'s> Tokens<'s> {
- pub fn new(start: Position, src: &'s str) -> Tokens<'s> {
+ pub fn new(start: Position, src: &'s str, mode: TokenizationMode) -> Tokens<'s> {
Tokens {
src,
- state: State::Body,
- stack: vec![],
+ mode,
iter: src.chars().peekable(),
position: start,
index: 0,
@@ -110,35 +105,6 @@ impl<'s> Tokens<'s> {
pub fn pos(&self) -> Position {
self.position
}
-
- /// Move through the string until an unbalanced closing bracket is found
- /// without tokenizing the contents.
- ///
- /// Returns whether a closing bracket was found or the end of the string was
- /// reached.
- pub fn move_to_closing_bracket(&mut self) -> bool {
- let mut escaped = false;
- let mut depth = 0;
-
- self.read_string_until(|n| {
- match n {
- '[' if !escaped => depth += 1,
- ']' if !escaped => {
- if depth == 0 {
- return true;
- } else {
- depth -= 1;
- }
- }
- '\\' => escaped = !escaped,
- _ => escaped = false,
- }
-
- false
- }, false, 0, 0);
-
- self.peek() == Some(']')
- }
}
impl<'s> Iterator for Tokens<'s> {
@@ -153,55 +119,31 @@ impl<'s> Iterator for Tokens<'s> {
// Comments.
'/' if self.peek() == Some('/') => self.parse_line_comment(),
'/' if self.peek() == Some('*') => self.parse_block_comment(),
- '*' if self.peek() == Some('/') => { self.eat(); StarSlash }
+ '*' if self.peek() == Some('/') => { self.eat(); Invalid("*/") }
// Whitespace.
c if c.is_whitespace() => self.parse_whitespace(start),
// Functions.
- '[' => {
- match self.state {
- Header | Body => {
- self.stack.push((self.state, start));
- self.position = Position::new(0, '['.len_utf8());
- self.state = Header;
- }
- StartBody => self.state = Body,
- }
-
- LeftBracket
- }
- ']' => {
- if self.state == Header && self.peek() == Some('[') {
- self.state = StartBody;
- } else {
- if let Some((state, pos)) = self.stack.pop() {
- self.state = state;
- self.position = pos + self.position;
- } else {
- self.state = Body;
- }
- }
-
- RightBracket
- }
+ '[' => self.parse_function(start),
+ ']' => Invalid("]"),
// Syntactic elements in function headers.
- '(' if self.state == Header => LeftParen,
- ')' if self.state == Header => RightParen,
- '{' if self.state == Header => LeftBrace,
- '}' if self.state == Header => RightBrace,
- ':' if self.state == Header => Colon,
- ',' if self.state == Header => Comma,
- '=' if self.state == Header => Equals,
+ '(' if self.mode == Header => LeftParen,
+ ')' if self.mode == Header => RightParen,
+ '{' if self.mode == Header => LeftBrace,
+ '}' if self.mode == Header => RightBrace,
+ ':' if self.mode == Header => Colon,
+ ',' if self.mode == Header => Comma,
+ '=' if self.mode == Header => Equals,
// String values.
- '"' if self.state == Header => self.parse_string(),
+ '"' if self.mode == Header => self.parse_string(),
// Style toggles.
- '*' if self.state == Body => Star,
- '_' if self.state == Body => Underscore,
- '`' if self.state == Body => Backtick,
+ '*' if self.mode == Body => Star,
+ '_' if self.mode == Body => Underscore,
+ '`' if self.mode == Body => Backtick,
// An escaped thing.
'\\' => self.parse_escaped(),
@@ -215,9 +157,9 @@ impl<'s> Iterator for Tokens<'s> {
',' | '"' | '/' => true,
_ => false,
}
- }, false, -(c.len_utf8() as isize), 0);
+ }, false, -(c.len_utf8() as isize), 0).0;
- if self.state == Header {
+ if self.mode == Header {
self.parse_expr(text)
} else {
Text(text)
@@ -234,7 +176,7 @@ impl<'s> Iterator for Tokens<'s> {
impl<'s> Tokens<'s> {
fn parse_line_comment(&mut self) -> Token<'s> {
- LineComment(self.read_string_until(is_newline_char, false, 1, 0))
+ LineComment(self.read_string_until(is_newline_char, false, 1, 0).0)
}
fn parse_block_comment(&mut self) -> Token<'s> {
@@ -262,19 +204,60 @@ impl<'s> Tokens<'s> {
}
false
- }, true, 0, -2))
+ }, true, 0, -2).0)
}
fn parse_whitespace(&mut self, start: Position) -> Token<'s> {
self.read_string_until(|n| !n.is_whitespace(), false, 0, 0);
let end = self.pos();
- Whitespace(end.line - start.line)
+ Space(end.line - start.line)
+ }
+
+ fn parse_function(&mut self, start: Position) -> Token<'s> {
+ let (header, terminated) = self.read_function_part();
+ self.eat();
+
+ if self.peek() != Some('[') {
+ return Function { header, body: None, terminated };
+ }
+
+ self.eat();
+
+ let offset = self.pos() - start;
+ let (body, terminated) = self.read_function_part();
+ self.eat();
+
+ Function { header, body: Some((offset, body)), terminated }
+ }
+
+ fn read_function_part(&mut self) -> (&'s str, bool) {
+ let mut escaped = false;
+ let mut in_string = false;
+ let mut depth = 0;
+
+ self.read_string_until(|n| {
+ match n {
+ '"' if !escaped => in_string = !in_string,
+ '[' if !escaped && !in_string => depth += 1,
+ ']' if !escaped && !in_string => {
+ if depth == 0 {
+ return true;
+ } else {
+ depth -= 1;
+ }
+ }
+ '\\' => escaped = !escaped,
+ _ => escaped = false,
+ }
+
+ false
+ }, false, 0, 0)
}
fn parse_string(&mut self) -> Token<'s> {
let mut escaped = false;
- ExprStr(self.read_string_until(|n| {
+ let (string, terminated) = self.read_string_until(|n| {
match n {
'"' if !escaped => return true,
'\\' => escaped = !escaped,
@@ -282,7 +265,8 @@ impl<'s> Tokens<'s> {
}
false
- }, true, 0, -1))
+ }, true, 0, -1);
+ ExprStr { string, terminated }
}
fn parse_escaped(&mut self) -> Token<'s> {
@@ -294,7 +278,7 @@ impl<'s> Tokens<'s> {
}
let c = self.peek().unwrap_or('n');
- if self.state == Body && is_escapable(c) {
+ if self.mode == Body && is_escapable(c) {
let index = self.index();
self.eat();
Text(&self.src[index .. index + c.len_utf8()])
@@ -315,7 +299,7 @@ impl<'s> Tokens<'s> {
} else if is_identifier(text) {
ExprIdent(text)
} else {
- Text(text)
+ Invalid(text)
}
}
@@ -325,7 +309,7 @@ impl<'s> Tokens<'s> {
eat_match: bool,
offset_start: isize,
offset_end: isize,
- ) -> &'s str where F: FnMut(char) -> bool {
+ ) -> (&'s str, bool) where F: FnMut(char) -> bool {
let start = ((self.index() as isize) + offset_start) as usize;
let mut matched = false;
@@ -346,7 +330,7 @@ impl<'s> Tokens<'s> {
end = ((end as isize) + offset_end) as usize;
}
- &self.src[start .. end]
+ (&self.src[start .. end], matched)
}
fn eat(&mut self) -> Option<char> {
diff --git a/tests/layouter/test.typ b/tests/layouter/test.typ
deleted file mode 100644
index b7fd334d..00000000
--- a/tests/layouter/test.typ
+++ /dev/null
@@ -1 +0,0 @@
-[box][hi]
diff --git a/tests/parser/tokens.rs b/tests/parser/tokens.rs
index fb48b32e..66e44ac5 100644
--- a/tests/parser/tokens.rs
+++ b/tests/parser/tokens.rs
@@ -1,80 +1,77 @@
// Whitespace.
t "" => []
-t " " => [W(0)]
-t " " => [W(0)]
-t "\t" => [W(0)]
-t " \t" => [W(0)]
-t "\n" => [W(1)]
-t "\n " => [W(1)]
-t " \n" => [W(1)]
-t " \n " => [W(1)]
-t " \n\t \n " => [W(2)]
-t "\r\n" => [W(1)]
-t " \r\r\n \x0D" => [W(3)]
-t "\n\r" => [W(2)]
+t " " => [S(0)]
+t " " => [S(0)]
+t "\t" => [S(0)]
+t " \t" => [S(0)]
+t "\n" => [S(1)]
+t "\n " => [S(1)]
+t " \n" => [S(1)]
+t " \n " => [S(1)]
+t " \n\t \n " => [S(2)]
+t "\r\n" => [S(1)]
+t " \r\r\n \x0D" => [S(3)]
+t "\n\r" => [S(2)]
// Comments.
-t "a // bc\n " => [T("a"), W(0), LC(" bc"), W(1)]
-t "a //a//b\n " => [T("a"), W(0), LC("a//b"), W(1)]
-t "a //a//b\r\n" => [T("a"), W(0), LC("a//b"), W(1)]
-t "a //a//b\n\nhello" => [T("a"), W(0), LC("a//b"), W(2), T("hello")]
+t "a // bc\n " => [T("a"), S(0), LC(" bc"), S(1)]
+t "a //a//b\n " => [T("a"), S(0), LC("a//b"), S(1)]
+t "a //a//b\r\n" => [T("a"), S(0), LC("a//b"), S(1)]
+t "a //a//b\n\nhello" => [T("a"), S(0), LC("a//b"), S(2), T("hello")]
t "/**/" => [BC("")]
-t "_/*_/*a*/*/" => [U, BC("_/*a*/")]
+t "_/*_/*a*/*/" => [Underscore, BC("_/*a*/")]
t "/*/*/" => [BC("/*/")]
-t "abc*/" => [T("abc"), SS]
+t "abc*/" => [T("abc"), Invalid("*/")]
// Header only tokens.
-t "[" => [LB]
-t "]" => [RB]
-t "[(){}:=,]" => [LB, LP, RP, LBR, RBR, CL, EQ, CM, RB]
-t "[a:b]" => [LB, ID("a"), CL, ID("b"), RB]
-t "[🌓, 🌍,]" => [LB, T("🌓"), CM, W(0), T("🌍"), CM, RB]
-t "[=]" => [LB, EQ, RB]
-t "[,]" => [LB, CM, RB]
-t "a: b" => [T("a"), T(":"), W(0), T("b")]
-t "c=d, " => [T("c"), T("=d"), T(","), W(0)]
-t r#"["hello\"world"]"# => [LB, STR(r#"hello\"world"#), RB]
-t r#"["hi", 12pt]"# => [LB, STR("hi"), CM, W(0), SIZE(Size::pt(12.0)), RB]
-t "\"hi\"" => [T("\"hi"), T("\"")]
-t "[a: true, x=1]" => [LB, ID("a"), CL, W(0), BOOL(true), CM, W(0),
- ID("x"), EQ, NUM(1.0), RB]
-t "[120%]" => [LB, NUM(1.2), RB]
+th "[" => [Func("", None, false)]
+th "]" => [Invalid("]")]
+th "(){}:=," => [LP, RP, LB, RB, Colon, Equals, Comma]
+th "a:b" => [Id("a"), Colon, Id("b")]
+th "=" => [Equals]
+th "," => [Comma]
+th r#""hello\"world""# => [Str(r#"hello\"world"#)]
+th r#""hi", 12pt"# => [Str("hi"), Comma, S(0), Size(12.0)]
+th "\"hi\"" => [T("\"hi"), T("\"")]
+th "a: true, x=1" => [Id("a"), Colon, S(0), Bool(true), Comma, S(0),
+ Id("x"), Equals, Num(1.0)]
+th "120%" => [Num(1.2)]
+th "🌓, 🌍," => [T("🌓"), Comma, S(0), T("🌍"), Comma]
+tb "a: b" => [T("a"), T(":"), S(0), T("b")]
+tb "c=d, " => [T("c"), T("=d"), T(","), S(0)]
// Body only tokens.
-t "_*`" => [U, S, B]
-t "[func]*bold*" => [LB, ID("func"), RB, S, T("bold"), S]
-t "[_*`]" => [LB, T("_"), T("*"), T("`"), RB]
-t "hi_you_ there" => [T("hi"), U, T("you"), U, W(0), T("there")]
+tb "_*`" => [Underscore, Star, Backtick]
+tb "[func]*bold*" => [Func("func", None, true), Star, T("bold"), Star]
+tb "hi_you_ there" => [T("hi"), Underscore, T("you"), Underscore, S(0), T("there")]
+th "_*`" => [Invalid("_"), Invalid("*"), Invalid("`")]
// Nested functions.
-t "[f: [=][*]]" => [LB, ID("f"), CL, W(0), LB, EQ, RB, LB, S, RB, RB]
-t "[_][[,],]," => [LB, T("_"), RB, LB, LB, CM, RB, T(","), RB, T(",")]
-t "[=][=][=]" => [LB, EQ, RB, LB, T("="), RB, LB, EQ, RB]
-t "[=][[=][=][=]]" => [LB, EQ, RB, LB, LB, EQ, RB, LB, T("="), RB, LB, EQ, RB, RB]
+tb "[f: [=][*]]" => [Func("f: [=][*]", None, true)]
+tb "[_][[,],]," => [Func("_", Some("[,],"), true), T(",")]
+tb "[=][=][=]" => [Func("=", Some("="), true), Func("=", None, true)]
+tb "[=][[=][=][=]]" => [Func("=", Some("[=][=][=]")), true]
// Escapes.
-t r"\[" => [T("[")]
-t r"\]" => [T("]")]
-t r"\\" => [T(r"\")]
-t r"\/" => [T("/")]
-t r"\*" => [T("*")]
-t r"\_" => [T("_")]
-t r"\`" => [T("`")]
+tb r"\[" => [T("[")]
+tb r"\]" => [T("]")]
+tb r"\\" => [T(r"\")]
+tb r"\/" => [T("/")]
+tb r"\*" => [T("*")]
+tb r"\_" => [T("_")]
+tb r"\`" => [T("`")]
// Unescapable special symbols.
-t r"\:" => [T(r"\"), T(":")]
-t r"\=" => [T(r"\"), T("=")]
-t r"[\:]" => [LB, T(r"\"), CL, RB]
-t r"[\=]" => [LB, T(r"\"), EQ, RB]
-t r"[\,]" => [LB, T(r"\"), CM, RB]
+th r"\:" => [T(r"\"), T(":")]
+th r"\=" => [T(r"\"), T("=")]
+th r"\:" => [T(r"\"), Colon]
+th r"\=" => [T(r"\"), Equals]
+th r"\," => [T(r"\"), Comma]
-// Spans
-ts "hello" => [(0:0, 0:5, T("hello"))]
-ts "ab\r\nc" => [(0:0, 0:2, T("ab")), (0:2, 1:0, W(1)), (1:0, 1:1, T("c"))]
-ts "[a=10]" => [(0:0, 0:1, LB), (0:1, 0:2, ID("a")), (0:2, 0:3, EQ),
- (0:3, 0:5, NUM(10.0)), (0:5, 0:6, RB)]
-ts r#"[x = "(1)"]*"# => [(0:0, 0:1, LB), (0:1, 0:2, ID("x")), (0:2, 0:3, W(0)),
- (0:3, 0:4, EQ), (0:4, 0:5, W(0)), (0:5, 0:10, STR("(1)")),
- (0:10, 0:11, RB), (0:11, 0:12, S)]
-ts "// ab\r\n\nf" => [(0:0, 0:5, LC(" ab")), (0:5, 2:0, W(2)), (2:0, 2:1, T("f"))]
-ts "/*b*/_" => [(0:0, 0:5, BC("b")), (0:5, 0:6, U)]
+// Spans.
+tbs "hello" => [(0:0, 0:5, T("hello"))]
+tbs "ab\r\nc" => [(0:0, 0:2, T("ab")), (0:2, 1:0, S(1)), (1:0, 1:1, T("c"))]
+tbs "[x = \"(1)\"]*" => [(0:0, 0:11, Func("x = \"(1)\"", None, true)), (0:11, 0:12, Star)]
+tbs "// ab\r\n\nf" => [(0:0, 0:5, LC(" ab")), (0:5, 2:0, S(2)), (2:0, 2:1, T("f"))]
+tbs "/*b*/_" => [(0:0, 0:5, BC("b")), (0:5, 0:6, Underscore)]
+ths "a=10" => [(0:0, 0:1, Id("a")), (0:1, 0:2, Equals), (0:2, 0:4, Num(10.0))]
diff --git a/tests/src/layouter.rs b/tests/src/layouter.rs
index eeca1a1b..95eea204 100644
--- a/tests/src/layouter.rs
+++ b/tests/src/layouter.rs
@@ -10,8 +10,8 @@ use futures_executor::block_on;
use typstc::Typesetter;
use typstc::layout::{MultiLayout, Serialize};
-use typstc::size::{Size, Size2D};
-use typstc::style::PageStyle;
+use typstc::size::{Size, Size2D, ValueBox};
+use typstc::style::{PageStyle, PaperClass};
use typstc::toddle::query::FileSystemFontProvider;
use typstc::export::pdf::PdfExporter;
@@ -62,14 +62,15 @@ fn main() -> DynResult<()> {
Ok(())
}
-/// Create a _PDF_ with a name from the source code.
+/// Create a _PDF_ and render with a name from the source code.
fn test(name: &str, src: &str) -> DynResult<()> {
println!("Testing: {}.", name);
let mut typesetter = Typesetter::new();
typesetter.set_page_style(PageStyle {
+ class: PaperClass::Custom,
dimensions: Size2D::with_all(Size::pt(250.0)),
- .. PageStyle::default()
+ margins: ValueBox::with_all(None),
});
let provider = FileSystemFontProvider::from_index("../fonts/index.json")?;
@@ -120,31 +121,43 @@ fn test(name: &str, src: &str) -> DynResult<()> {
/// Compile the source code with the typesetter.
fn compile(typesetter: &Typesetter, src: &str) -> MultiLayout {
- #[cfg(not(debug_assertions))] {
- use std::time::Instant;
+ #![allow(unused_variables)]
+ use std::time::Instant;
- // Warmup.
+ // Warmup.
+ #[cfg(not(debug_assertions))]
+ let warmup = {
let warmup_start = Instant::now();
block_on(typesetter.typeset(&src));
- let warmup_end = Instant::now();
-
- let start = Instant::now();
- let tree = typesetter.parse(&src).output;
- let mid = Instant::now();
- let layouts = block_on(typesetter.layout(&tree)).output;
- let end = Instant::now();
-
- println!(" - cold start: {:?}", warmup_end - warmup_start);
- println!(" - warmed up: {:?}", end - start);
- println!(" - parsing: {:?}", mid - start);
- println!(" - layouting: {:?}", end - mid);
- println!();
+ Instant::now() - warmup_start
+ };
+
+ let start = Instant::now();
+ let parsed = typesetter.parse(&src);
+ let parse = Instant::now() - start;
+
+ if !parsed.errors.is_empty() {
+ println!("parse errors: {:#?}", parsed.errors);
+ }
- layouts
+ let start_layout = Instant::now();
+ let layouted = block_on(typesetter.layout(&parsed.output));
+ let layout = Instant::now() - start_layout;
+ let total = Instant::now() - start;
+
+ if !layouted.errors.is_empty() {
+ println!("layout errors: {:#?}", layouted.errors);
+ }
+
+ #[cfg(not(debug_assertions))] {
+ println!(" - cold start: {:?}", warmup);
+ println!(" - warmed up: {:?}", total);
+ println!(" - parsing: {:?}", parse);
+ println!(" - layouting: {:?}", layout);
+ println!();
}
- #[cfg(debug_assertions)]
- block_on(typesetter.typeset(&src))
+ layouted.output
}
/// Command line options.