From 95e6b078fecddeaa3d6f2c920b617201b74bf01e Mon Sep 17 00:00:00 2001 From: Laurenz Date: Sun, 19 Jan 2020 21:50:20 +0100 Subject: =?UTF-8?q?Move=20to=20non-fatal=20errors=20=F0=9F=AA=82=20[WIP]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Dynamic models instead of SyntaxTrees - No more ParseResult/LayoutResult - Errors and Decorations which are propagated to parent contexts - Models are finally clonable --- src/syntax/expr.rs | 3 +- src/syntax/func.rs | 125 +++++++++++++++++ src/syntax/mod.rs | 382 +++++++++++++++----------------------------------- src/syntax/parsing.rs | 207 +++++++++++++-------------- src/syntax/span.rs | 40 ++++++ src/syntax/tokens.rs | 90 ++++++++++-- 6 files changed, 453 insertions(+), 394 deletions(-) create mode 100644 src/syntax/func.rs (limited to 'src/syntax') diff --git a/src/syntax/expr.rs b/src/syntax/expr.rs index c4feea74..74deda46 100644 --- a/src/syntax/expr.rs +++ b/src/syntax/expr.rs @@ -1,3 +1,4 @@ +use crate::size::ScaleSize; use super::*; @@ -126,6 +127,7 @@ impl Display for Object { } } +/// A key-value pair in an object. #[derive(Clone, PartialEq)] pub struct Pair { pub key: Spanned, @@ -144,7 +146,6 @@ debug_display!(Tuple); debug_display!(Object); debug_display!(Pair); - /// Kinds of expressions. pub trait ExpressionKind: Sized { /// The name of the expression in an `expected ` error. diff --git a/src/syntax/func.rs b/src/syntax/func.rs new file mode 100644 index 00000000..5b1ce6e8 --- /dev/null +++ b/src/syntax/func.rs @@ -0,0 +1,125 @@ +use super::*; + + +#[derive(Debug, Clone, PartialEq)] +pub struct FuncHeader { + pub name: Spanned, + pub args: FuncArgs, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct FuncArgs { + pub pos: Tuple, + pub key: Object, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum Arg { + Pos(Spanned), + Key(Pair), +} + +impl Arg { + /// The span or the value or combined span of key and value. + pub fn span(&self) -> Span { + match self { + Arg::Pos(item) => item.span, + Arg::Key(Pair { key, value }) => Span::merge(key.span, value.span), + } + } +} + +impl FuncArgs { + pub fn new() -> FuncArgs { + FuncArgs { + pos: Tuple::new(), + key: Object::new(), + } + } + + /// Add an argument. + pub fn add(&mut self, arg: Arg) { + match arg { + Arg::Pos(item) => self.add_pos(item), + Arg::Key(pair) => self.add_key_pair(pair), + } + } + + /// Add a positional argument. + pub fn add_pos(&mut self, item: Spanned) { + self.pos.add(item); + } + + /// Add a keyword argument. + pub fn add_key(&mut self, key: Spanned, value: Spanned) { + self.key.add(key, value); + } + + /// Add a keyword argument from an existing pair. + pub fn add_key_pair(&mut self, pair: Pair) { + self.key.add_pair(pair); + } + + // /// Force-extract the first positional argument. + // pub fn get_pos(&mut self) -> ParseResult { + // expect(self.get_pos_opt()) + // } + + // /// Extract the first positional argument. + // pub fn get_pos_opt(&mut self) -> ParseResult> { + // Ok(if !self.positional.items.is_empty() { + // let spanned = self.positional.items.remove(0); + // Some(E::from_expr(spanned)?) + // } else { + // None + // }) + // } + + // /// Force-extract a keyword argument. + // pub fn get_key(&mut self, name: &str) -> ParseResult { + // expect(self.get_key_opt(name)) + // } + + // /// Extract a keyword argument. + // pub fn get_key_opt(&mut self, name: &str) -> ParseResult> { + // self.keyword.pairs.iter() + // .position(|p| p.key.v.0 == name) + // .map(|index| { + // let value = self.keyword.pairs.swap_remove(index).value; + // E::from_expr(value) + // }) + // .transpose() + // } + + // /// Iterator over positional arguments. + // pub fn iter_pos(&mut self) -> std::vec::IntoIter> { + // let tuple = std::mem::replace(&mut self.positional, Tuple::new()); + // tuple.items.into_iter() + // } + + // /// Iterator over all keyword arguments. + // pub fn iter_keys(&mut self) -> std::vec::IntoIter { + // let object = std::mem::replace(&mut self.keyword, Object::new()); + // object.pairs.into_iter() + // } + + // /// Clear the argument lists. + // pub fn clear(&mut self) { + // self.positional.items.clear(); + // self.keyword.pairs.clear(); + // } + + // /// Whether both the positional and keyword argument lists are empty. + // pub fn is_empty(&self) -> bool { + // self.positional.items.is_empty() && self.keyword.pairs.is_empty() + // } +} + +// /// Extract the option expression kind from the option or return an error. +// fn expect(opt: ParseResult>) -> ParseResult { +// match opt { +// Ok(Some(spanned)) => Ok(spanned), +// Ok(None) => error!("expected {}", E::NAME), +// Err(e) => Err(e), +// } +// } diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs index f644f051..75407f82 100644 --- a/src/syntax/mod.rs +++ b/src/syntax/mod.rs @@ -1,321 +1,161 @@ //! Tokenization and parsing of source code. -use std::fmt::{self, Display, Formatter}; -use unicode_xid::UnicodeXID; +use std::any::Any; +use std::fmt::{self, Debug, Display, Formatter}; +use std::future::Future; +use std::pin::Pin; use serde::Serialize; -use crate::func::LayoutFunc; -use crate::size::{Size, ScaleSize}; - - -pub type ParseResult = crate::TypesetResult; +use crate::error::Error; +use crate::func::{Commands, Command}; +use crate::layout::{Layouted, LayoutContext}; +use crate::size::Size; pub_use_mod!(expr); +pub_use_mod!(func); pub_use_mod!(tokens); pub_use_mod!(parsing); pub_use_mod!(span); - -/// A minimal semantic entity of source code. -#[derive(Debug, Copy, Clone, PartialEq)] -pub enum Token<'s> { - /// One or more whitespace characters. The contained `usize` denotes the - /// number of newlines that were contained in the whitespace. - Whitespace(usize), - - /// A line comment with inner string contents `//<&'s str>\n`. - LineComment(&'s str), - /// A block comment with inner string contents `/*<&'s str>*/`. The comment - /// can contain nested block comments. - BlockComment(&'s str), - /// An erroneous `*/` without an opening block comment. - StarSlash, - - /// A left bracket: `[`. - LeftBracket, - /// A right bracket: `]`. - RightBracket, - - /// A left parenthesis in a function header: `(`. - LeftParen, - /// A right parenthesis in a function header: `)`. - RightParen, - /// A left brace in a function header: `{`. - LeftBrace, - /// A right brace in a function header: `}`. - RightBrace, - - /// A colon in a function header: `:`. - Colon, - /// A comma in a function header: `:`. - Comma, - /// An equals sign in a function header: `=`. - Equals, - - /// An identifier in a function header: `center`. - ExprIdent(&'s str), - /// A quoted string in a function header: `"..."`. - ExprStr(&'s str), - /// A number in a function header: `3.14`. - ExprNumber(f64), - /// A size in a function header: `12pt`. - ExprSize(Size), - /// A boolean in a function header: `true | false`. - ExprBool(bool), - - /// A star in body-text. - Star, - /// An underscore in body-text. - Underscore, - /// A backtick in body-text. - Backtick, - - /// Any other consecutive string. - Text(&'s str), -} - -/// A tree representation of source code. -#[derive(Debug, PartialEq)] -pub struct SyntaxTree { - pub nodes: Vec>, +/// Common syntax types. +pub mod prelude { + pub use super::*; } -impl SyntaxTree { - /// Create an empty syntax tree. - pub fn new() -> SyntaxTree { - SyntaxTree { nodes: vec![] } - } - /// Add a node to the tree. - pub fn add(&mut self, node: Spanned) { - self.nodes.push(node); - } +pub struct Parsed { + pub output: T, + pub errors: SpanVec, + pub decorations: SpanVec, } -/// A node in the syntax tree. -#[derive(PartialEq)] -pub enum Node { - /// A number of whitespace characters containing less than two newlines. - Space, - /// Whitespace characters with more than two newlines. - Newline, - /// Plain text. - Text(String), - /// Italics enabled / disabled. - ToggleItalic, - /// Bolder enabled / disabled. - ToggleBolder, - /// Monospace enabled / disabled. - ToggleMonospace, - /// A function invocation. - Func(FuncCall), -} - -impl Display for Node { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - match self { - Node::Space => write!(f, "Space"), - Node::Newline => write!(f, "Newline"), - Node::Text(text) => write!(f, "{:?}", text), - Node::ToggleItalic => write!(f, "ToggleItalic"), - Node::ToggleBolder => write!(f, "ToggleBold"), - Node::ToggleMonospace => write!(f, "ToggleMonospace"), - Node::Func(func) => { - if f.alternate() { - write!(f, "{:#?}", func.0) - } else { - write!(f, "{:?}", func.0) - } - } +impl Parsed { + pub fn map(self, f: F) -> Parsed where F: FnOnce(T) -> U { + Parsed { + output: f(self.output), + errors: self.errors, + decorations: self.decorations, } } } -debug_display!(Node); +#[async_trait::async_trait(?Send)] +pub trait Model: Debug + ModelBounds { + async fn layout<'a>( + &'a self, + ctx: LayoutContext<'_, '_> + ) -> Layouted>; +} -/// An invocation of a function. -#[derive(Debug)] -pub struct FuncCall(pub Box); +pub type DynFuture<'a, T> = Pin + 'a>>; -impl PartialEq for FuncCall { - fn eq(&self, other: &FuncCall) -> bool { - &self.0 == &other.0 +impl dyn Model { + pub fn downcast(&self) -> Option<&T> where T: Model + 'static { + self.as_any().downcast_ref::() } } -#[derive(Debug, Clone, PartialEq)] -pub struct FuncHeader { - pub name: Spanned, - pub args: FuncArgs, +impl PartialEq for dyn Model { + fn eq(&self, other: &dyn Model) -> bool { + self.bound_eq(other) + } } -#[derive(Debug, Clone, PartialEq)] -pub struct FuncArgs { - pub positional: Tuple, - pub keyword: Object, +impl Clone for Box { + fn clone(&self) -> Self { + self.bound_clone() + } } -#[derive(Debug, Clone, PartialEq)] -pub enum Arg { - Pos(Spanned), - Key(Pair), +pub trait ModelBounds { + fn as_any(&self) -> &dyn Any; + fn bound_eq(&self, other: &dyn Model) -> bool; + fn bound_clone(&self) -> Box; } -impl Arg { - /// The span or the value or combined span of key and value. - pub fn span(&self) -> Span { - match self { - Arg::Pos(spanned) => spanned.span, - Arg::Key(Pair { key, value }) => Span::merge(key.span, value.span), - } +impl ModelBounds for T where T: Model + PartialEq + Clone + 'static { + fn as_any(&self) -> &dyn Any { + self } -} -impl FuncArgs { - pub fn new() -> FuncArgs { - FuncArgs { - positional: Tuple::new(), - keyword: Object::new(), + fn bound_eq(&self, other: &dyn Model) -> bool { + match other.as_any().downcast_ref::() { + Some(other) => self == other, + None => false, } } - /// Add a positional argument. - pub fn add_pos(&mut self, item: Spanned) { - self.positional.add(item); - } - - /// Force-extract the first positional argument. - pub fn get_pos(&mut self) -> ParseResult { - expect(self.get_pos_opt()) - } - - /// Extract the first positional argument. - pub fn get_pos_opt(&mut self) -> ParseResult> { - Ok(if !self.positional.items.is_empty() { - let spanned = self.positional.items.remove(0); - Some(E::from_expr(spanned)?) - } else { - None - }) - } - - /// Add a keyword argument. - pub fn add_key(&mut self, key: Spanned, value: Spanned) { - self.keyword.add(key, value); - } - - /// Add a keyword argument from an existing pair. - pub fn add_key_pair(&mut self, pair: Pair) { - self.keyword.add_pair(pair); - } - - /// Force-extract a keyword argument. - pub fn get_key(&mut self, name: &str) -> ParseResult { - expect(self.get_key_opt(name)) - } - - /// Extract a keyword argument. - pub fn get_key_opt(&mut self, name: &str) -> ParseResult> { - self.keyword.pairs.iter() - .position(|p| p.key.v.0 == name) - .map(|index| { - let value = self.keyword.pairs.swap_remove(index).value; - E::from_expr(value) - }) - .transpose() - } - - /// Iterator over positional arguments. - pub fn iter_pos(&mut self) -> std::vec::IntoIter> { - let tuple = std::mem::replace(&mut self.positional, Tuple::new()); - tuple.items.into_iter() - } - - /// Iterator over all keyword arguments. - pub fn iter_keys(&mut self) -> std::vec::IntoIter { - let object = std::mem::replace(&mut self.keyword, Object::new()); - object.pairs.into_iter() - } - - /// Clear the argument lists. - pub fn clear(&mut self) { - self.positional.items.clear(); - self.keyword.pairs.clear(); - } - - /// Whether both the positional and keyword argument lists are empty. - pub fn is_empty(&self) -> bool { - self.positional.items.is_empty() && self.keyword.pairs.is_empty() - } -} - -/// Extract the option expression kind from the option or return an error. -fn expect(opt: ParseResult>) -> ParseResult { - match opt { - Ok(Some(spanned)) => Ok(spanned), - Ok(None) => error!("expected {}", E::NAME), - Err(e) => Err(e), + fn bound_clone(&self) -> Box { + Box::new(self.clone()) } } -#[derive(Debug, Clone, Eq, PartialEq, Serialize)] -pub struct Colorization { - pub tokens: Vec>, +/// A tree representation of source code. +#[derive(Debug, Clone, PartialEq)] +pub struct SyntaxModel { + pub nodes: SpanVec, } -impl Colorization { - pub fn new() -> Colorization { - Colorization { tokens: vec![] } - } - - pub fn add(&mut self, token: ColorToken, span: Span) { - self.tokens.push(Spanned { v: token, span }); +impl SyntaxModel { + /// Create an empty syntax model. + pub fn new() -> SyntaxModel { + SyntaxModel { nodes: vec![] } } - pub fn replace_last(&mut self, token: ColorToken) { - self.tokens.last_mut().expect("replace_last: no token").v = token; + /// Add a node to the model. + pub fn add(&mut self, node: Spanned) { + self.nodes.push(node); } } -/// Entities which can be colored by syntax highlighting. -#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub enum ColorToken { - Comment, - Bracket, - FuncName, - Colon, - Key, - Equals, - Comma, - Paren, - Brace, - ExprIdent, - ExprStr, - ExprNumber, - ExprSize, - ExprBool, - Bold, - Italic, - Monospace, - Invalid, -} - -#[derive(Debug, Clone, Eq, PartialEq, Serialize)] -pub struct ErrorMap { - pub errors: Vec>, -} - -impl ErrorMap { - pub fn new() -> ErrorMap { - ErrorMap { errors: vec![] } +#[async_trait::async_trait(?Send)] +impl Model for SyntaxModel { + async fn layout<'a>(&'a self, ctx: LayoutContext<'_, '_>) -> Layouted> { + Layouted { + output: vec![Command::LayoutSyntaxModel(self)], + errors: vec![], + } } +} - pub fn add(&mut self, message: impl Into, span: Span) { - self.errors.push(Spanned { v: message.into(), span }); +/// A node in the syntax tree. +#[derive(Debug, Clone)] +pub enum Node { + /// A number of whitespace characters containing less than two newlines. + Space, + /// Whitespace characters with more than two newlines. + Newline, + /// Plain text. + Text(String), + /// Italics enabled / disabled. + ToggleItalic, + /// Bolder enabled / disabled. + ToggleBolder, + /// Monospace enabled / disabled. + ToggleMonospace, + /// A submodel. + Model(Box), +} + +impl PartialEq for Node { + fn eq(&self, other: &Node) -> bool { + use Node::*; + match (self, other) { + (Space, Space) => true, + (Newline, Newline) => true, + (Text(a), Text(b)) => a == b, + (ToggleItalic, ToggleItalic) => true, + (ToggleBolder, ToggleBolder) => true, + (ToggleMonospace, ToggleMonospace) => true, + (Model(a), Model(b)) => a == b, + _ => false, + } } +} - pub fn add_at(&mut self, message: impl Into, pos: Position) { - self.errors.push(Spanned { v: message.into(), span: Span::at(pos) }) - } +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub enum Decoration { + ValidFuncName, + InvalidFuncName, + ArgumentKey, } diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs index f6d0b629..24bef7ce 100644 --- a/src/syntax/parsing.rs +++ b/src/syntax/parsing.rs @@ -3,9 +3,8 @@ use super::*; use Token::*; -/// Parses source code into a syntax tree given a context. -pub fn parse(src: &str, ctx: ParseContext) -> (SyntaxTree, Colorization, ErrorMap) { - Parser::new(src, ctx).parse() +pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed { + Parser::new(start, src, ctx).parse() } /// The context for parsing. @@ -18,64 +17,68 @@ pub struct ParseContext<'a> { struct Parser<'s> { src: &'s str, ctx: ParseContext<'s>, - colorization: Colorization, - error_map: ErrorMap, tokens: Tokens<'s>, peeked: Option>>>, position: Position, last_position: Position, + errors: SpanVec, + decorations: SpanVec, } impl<'s> Parser<'s> { - fn new(src: &'s str, ctx: ParseContext<'s>) -> Parser<'s> { + fn new(start: Position, src: &'s str, ctx: ParseContext<'s>) -> Parser<'s> { Parser { src, ctx, - error_map: ErrorMap::new(), - colorization: Colorization::new(), - tokens: Tokens::new(src), + tokens: tokenize(start, src), peeked: None, position: Position::ZERO, last_position: Position::ZERO, + errors: vec![], + decorations: vec![], } } /// The main parsing entrypoint. - fn parse(mut self) -> (SyntaxTree, Colorization, ErrorMap) { - let mut tree = SyntaxTree::new(); - - loop { - if let Some(spanned) = self.eat() { - match spanned.v { - LineComment(_) | BlockComment(_) => {} - - Whitespace(newlines) => { - tree.add(spanned.map_v(if newlines >= 2 { - Node::Newline - } else { - Node::Space - })); - } - - LeftBracket => { - if let Some(func) = self.parse_func() { - tree.add(func); - } - } - - Star => tree.add(spanned.map_v(Node::ToggleBolder)), - Underscore => tree.add(spanned.map_v(Node::ToggleItalic)), - Backtick => tree.add(spanned.map_v(Node::ToggleMonospace)), - Text(text) => tree.add(spanned.map_v(Node::Text(text.to_owned()))), - - _ => self.unexpected(spanned), + fn parse(mut self) -> Parsed { + let mut model = SyntaxModel::new(); + + while let Some(token) = self.eat() { + let mut span = token.span; + let node = match token.v { + LineComment(_) | BlockComment(_) => None, + Whitespace(newlines) => Some(if newlines >= 2 { + Node::Newline + } else { + Node::Space + }), + + LeftBracket => self.parse_func().map(|spanned| { + span = spanned.span; + spanned.v + }), + + Star => Some(Node::ToggleBolder), + Underscore => Some(Node::ToggleItalic), + Backtick => Some(Node::ToggleMonospace), + Text(text) => Some(Node::Text(text.to_owned())), + + _ => { + self.unexpected(token); + None } - } else { - break; + }; + + if let Some(v) = node { + model.add(Spanned { v, span }); } } - (tree, self.colorization, self.error_map) + Parsed { + output: model, + errors: self.errors, + decorations: self.decorations, + } } /// Parses a function including header and body with the cursor starting @@ -90,12 +93,55 @@ impl<'s> Parser<'s> { self.expected_at("closing bracket", self.pos()); } - let call = self.parse_func_call(header)?; + let body = if self.peekv() == Some(LeftBracket) { + self.eat(); + + let start_index = self.tokens.index(); + let start_position = self.tokens.pos(); + + let found = self.tokens.move_to_closing_bracket(); + + let end_index = self.tokens.index(); + let end_position = self.tokens.pos(); + + let body = &self.src[start_index .. end_index]; + + self.position = end_position; + + if found { + let next = self.eat().map(Spanned::value); + debug_assert_eq!(next, Some(RightBracket)); + } else { + self.expected_at("closing bracket", self.pos()); + } + + Some(Spanned::new(body, Span::new(start_position, end_position))) + } else { + None + }; + + let header = header?; + let (parser, decoration) = match self.ctx.scope.get_parser(header.name.v.as_str()) { + Ok(parser) => (parser, Decoration::ValidFuncName), + Err(parser) => { + let error = Error::new(format!("unknown function: `{}`", header.name.v)); + self.errors.push(Spanned::new(error, header.name.span)); + (parser, Decoration::InvalidFuncName) + } + }; + + self.decorations.push(Spanned::new(decoration, header.name.span)); + + let parsed = parser(header, body, self.ctx); + self.errors.extend(offset_spans(parsed.errors, start)); + self.decorations.extend(offset_spans(parsed.decorations, start)); + + let node = Node::Model(parsed.output); let end = self.pos(); let span = Span { start, end }; - Some(Spanned { v: Node::Func(call), span }) + Some(Spanned { v: node, span }) } /// Parses a function header including the closing bracket. @@ -125,7 +171,6 @@ impl<'s> Parser<'s> { match self.peek() { Some(Spanned { v: ExprIdent(ident), span }) => { self.eat(); - self.colorization.replace_last(ColorToken::FuncName); return Some(Spanned { v: Ident(ident.to_string()), span }); } other => self.expected_found_or_at("identifier", other, self.pos()), @@ -144,8 +189,7 @@ impl<'s> Parser<'s> { match self.peekv() { Some(RightBracket) | None => break, _ => match self.parse_arg() { - Some(Arg::Pos(item)) => args.add_pos(item), - Some(Arg::Key(pair)) => args.add_key_pair(pair), + Some(arg) => args.add(arg), None => {} } } @@ -165,11 +209,11 @@ impl<'s> Parser<'s> { let ident = Ident(ident.to_string()); if let Some(Equals) = self.peekv() { - self.colorization.replace_last(ColorToken::Key); - self.eat(); self.skip_whitespace(); + self.decorations.push(Spanned::new(Decoration::ArgumentKey, span)); + self.parse_expr().map(|value| { Arg::Key(Pair { key: Spanned { v: ident, span }, @@ -251,42 +295,6 @@ impl<'s> Parser<'s> { Spanned { v: Expression::Object(Object::new()), span } } - /// Parse the body of a function invocation. - fn parse_func_call(&mut self, header: Option) -> Option { - let body = if self.peekv() == Some(LeftBracket) { - self.eat(); - - let start = self.tokens.index(); - let found = self.tokens.move_to_closing_bracket(); - let end = self.tokens.index(); - - self.last_position = self.position; - self.position = self.tokens.pos(); - - let body = &self.src[start .. end]; - - if found { - let next = self.eat().map(Spanned::value); - debug_assert_eq!(next, Some(RightBracket)); - } else { - self.expected_at("closing bracket", self.pos()); - } - - Some(body) - } else { - None - }; - - let header = header?; - let parser = self.ctx.scope.get_parser(header.name.v.as_str()).or_else(|| { - let message = format!("unknown function: `{}`", header.name.v); - self.error_map.add(message, header.name.span); - None - })?; - - parser(header, body, self.ctx).ok().map(|f| FuncCall(f)) - } - /// Skip all whitespace/comment tokens. fn skip_whitespace(&mut self) { self.eat_until(|t| @@ -296,14 +304,16 @@ impl<'s> Parser<'s> { /// Add an error about an `thing` which was expected but not found at the /// given position. fn expected_at(&mut self, thing: &str, pos: Position) { - self.error_map.add_at(format!("expected {}", thing), pos); + let error = Error::new(format!("expected {}", thing)); + self.errors.push(Spanned::new(error, Span::at(pos))); } /// Add an error about an expected `thing` which was not found, showing /// what was found instead. fn expected_found(&mut self, thing: &str, found: Spanned) { let message = format!("expected {}, found {}", thing, name(found.v)); - self.error_map.add(message, found.span); + let error = Error::new(message); + self.errors.push(Spanned::new(error, found.span)); } /// Add a found-error if `found` is some and a positional error, otherwise. @@ -321,7 +331,8 @@ impl<'s> Parser<'s> { /// Add an error about an unexpected token `found`. fn unexpected(&mut self, found: Spanned) { - self.error_map.add(format!("unexpected {}", name(found.v)), found.span); + let error = Error::new(format!("unexpected {}", name(found.v))); + self.errors.push(Spanned::new(error, found.span)); } /// Consume tokens until the function returns true and only consume the last @@ -348,10 +359,6 @@ impl<'s> Parser<'s> { .unwrap_or_else(|| self.tokens.next()); if let Some(token) = token { - if let Some(color) = color(token.v) { - self.colorization.add(color, token.span); - } - self.last_position = self.position; self.position = token.span.end; } @@ -407,23 +414,3 @@ fn name(token: Token) -> &'static str { Text(_) => "invalid identifier", } } - -/// The color token corresponding to a token. -fn color(token: Token) -> Option { - Some(match token { - LineComment(_) | BlockComment(_) => ColorToken::Comment, - LeftBracket | RightBracket => ColorToken::Bracket, - LeftParen | RightParen => ColorToken::Paren, - LeftBrace | RightBrace => ColorToken::Brace, - Colon => ColorToken::Colon, - Comma => ColorToken::Comma, - Equals => ColorToken::Equals, - ExprIdent(_) => ColorToken::ExprIdent, - ExprStr(_) => ColorToken::ExprStr, - ExprNumber(_) => ColorToken::ExprNumber, - ExprSize(_) => ColorToken::ExprSize, - ExprBool(_) => ColorToken::ExprBool, - StarSlash => ColorToken::Invalid, - _ => return None, - }) -} diff --git a/src/syntax/span.rs b/src/syntax/span.rs index df9a3520..eb39677e 100644 --- a/src/syntax/span.rs +++ b/src/syntax/span.rs @@ -1,6 +1,7 @@ //! Spans map elements to the part of source code they originate from. use std::fmt::{self, Debug, Display, Formatter}; +use std::ops::{Add, AddAssign}; use serde::Serialize; @@ -100,6 +101,30 @@ impl Position { } } +impl Add for Position { + type Output = Position; + + fn add(self, rhs: Position) -> Position { + if rhs.line == 0 { + Position { + line: self.line, + column: self.column + rhs.column + } + } else { + Position { + line: self.line + rhs.line, + column: rhs.column, + } + } + } +} + +impl AddAssign for Position { + fn add_assign(&mut self, other: Position) { + *self = *self + other; + } +} + impl Display for Position { fn fmt(&self, f: &mut Formatter) -> fmt::Result { write!(f, "{}:{}", self.line, self.column) @@ -107,3 +132,18 @@ impl Display for Position { } debug_display!(Position); + +/// A vector of spanned things. +pub type SpanVec = Vec>; + +pub fn offset_spans( + vec: SpanVec, + start: Position, +) -> impl Iterator> { + vec.into_iter() + .map(move |mut spanned| { + spanned.span.start += start; + spanned.span.end += start; + spanned + }) +} diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs index 69d79965..0588bc6c 100644 --- a/src/syntax/tokens.rs +++ b/src/syntax/tokens.rs @@ -1,21 +1,80 @@ use std::iter::Peekable; use std::str::Chars; +use unicode_xid::UnicodeXID; use super::*; use Token::*; use State::*; +/// A minimal semantic entity of source code. +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum Token<'s> { + /// One or more whitespace characters. The contained `usize` denotes the + /// number of newlines that were contained in the whitespace. + Whitespace(usize), + + /// A line comment with inner string contents `//<&'s str>\n`. + LineComment(&'s str), + /// A block comment with inner string contents `/*<&'s str>*/`. The comment + /// can contain nested block comments. + BlockComment(&'s str), + /// An erroneous `*/` without an opening block comment. + StarSlash, + + /// A left bracket: `[`. + LeftBracket, + /// A right bracket: `]`. + RightBracket, + + /// A left parenthesis in a function header: `(`. + LeftParen, + /// A right parenthesis in a function header: `)`. + RightParen, + /// A left brace in a function header: `{`. + LeftBrace, + /// A right brace in a function header: `}`. + RightBrace, + + /// A colon in a function header: `:`. + Colon, + /// A comma in a function header: `:`. + Comma, + /// An equals sign in a function header: `=`. + Equals, + + /// An identifier in a function header: `center`. + ExprIdent(&'s str), + /// A quoted string in a function header: `"..."`. + ExprStr(&'s str), + /// A number in a function header: `3.14`. + ExprNumber(f64), + /// A size in a function header: `12pt`. + ExprSize(Size), + /// A boolean in a function header: `true | false`. + ExprBool(bool), + + /// A star in body-text. + Star, + /// An underscore in body-text. + Underscore, + /// A backtick in body-text. + Backtick, + + /// Any other consecutive string. + Text(&'s str), +} + /// Decomposes text into a sequence of semantic tokens. -pub fn tokenize(src: &str) -> Tokens { - Tokens::new(src) +pub fn tokenize(start: Position, src: &str) -> Tokens { + Tokens::new(start, src) } /// An iterator over the tokens of a string of source code. pub struct Tokens<'s> { src: &'s str, state: State, - stack: Vec, + stack: Vec<(State, Position)>, iter: Peekable>, position: Position, index: usize, @@ -29,13 +88,13 @@ enum State { } impl<'s> Tokens<'s> { - pub fn new(src: &'s str) -> Tokens<'s> { + pub fn new(start: Position, src: &'s str) -> Tokens<'s> { Tokens { src, state: State::Body, stack: vec![], iter: src.chars().peekable(), - position: Position::ZERO, + position: start, index: 0, } } @@ -47,7 +106,7 @@ impl<'s> Tokens<'s> { } /// The line-colunn position in the source at which the last token ends and - /// next token will start. + /// next token will start. This position is pub fn pos(&self) -> Position { self.position } @@ -101,11 +160,13 @@ impl<'s> Iterator for Tokens<'s> { // Functions. '[' => { - if self.state == Header || self.state == Body { - self.stack.push(self.state); - self.state = Header; - } else { - self.state = Body; + match self.state { + Header | Body => { + self.stack.push((self.state, start)); + self.position = Position::new(0, '['.len_utf8()); + self.state = Header; + } + StartBody => self.state = Body, } LeftBracket @@ -114,7 +175,12 @@ impl<'s> Iterator for Tokens<'s> { if self.state == Header && self.peek() == Some('[') { self.state = StartBody; } else { - self.state = self.stack.pop().unwrap_or(Body); + if let Some((state, pos)) = self.stack.pop() { + self.state = state; + self.position = pos + self.position; + } else { + self.state = Body; + } } RightBracket -- cgit v1.2.3