diff options
| -rw-r--r-- | src/func/macros.rs | 2 | ||||
| -rw-r--r-- | src/layout/tree.rs | 2 | ||||
| -rw-r--r-- | src/lib.rs | 4 | ||||
| -rw-r--r-- | src/syntax/color.rs | 28 | ||||
| -rw-r--r-- | src/syntax/expr.rs | 248 | ||||
| -rw-r--r-- | src/syntax/mod.rs | 311 | ||||
| -rw-r--r-- | src/syntax/parsing.rs | 389 | ||||
| -rw-r--r-- | src/syntax/span.rs | 10 | ||||
| -rw-r--r-- | src/syntax/tokens.rs | 80 |
9 files changed, 723 insertions, 351 deletions
diff --git a/src/func/macros.rs b/src/func/macros.rs index 3a32ec09..9e931ea2 100644 --- a/src/func/macros.rs +++ b/src/func/macros.rs @@ -144,7 +144,7 @@ macro_rules! parse { (optional: $body:expr, $ctx:expr) => ( if let Some(body) = $body { - Some($crate::syntax::parse(body, $ctx)?) + Some($crate::syntax::parse(body, $ctx)) } else { None } diff --git a/src/layout/tree.rs b/src/layout/tree.rs index 238c45fc..86b00f22 100644 --- a/src/layout/tree.rs +++ b/src/layout/tree.rs @@ -49,7 +49,7 @@ impl<'a, 'p> TreeLayouter<'a, 'p> { Node::Space => self.layout_space(), Node::Newline => self.layout_paragraph()?, - Node::ToggleItalics => self.style.text.variant.style.toggle(), + Node::ToggleItalic => self.style.text.variant.style.toggle(), Node::ToggleBolder => { self.style.text.variant.weight.0 += 300 * if self.style.text.bolder { -1 } else { 1 }; @@ -84,7 +84,7 @@ impl<'p> Typesetter<'p> { } /// Parse source code into a syntax tree. - pub fn parse(&self, src: &str) -> ParseResult<SyntaxTree> { + pub fn parse(&self, src: &str) -> SyntaxTree { let scope = Scope::with_std(); parse(src, ParseContext { scope: &scope }) } @@ -115,7 +115,7 @@ impl<'p> Typesetter<'p> { /// Process source code directly into a layout. pub async fn typeset(&self, src: &str) -> TypesetResult<MultiLayout> { - let tree = self.parse(src)?; + let tree = self.parse(src); let layout = self.layout(&tree).await?; Ok(layout) } diff --git a/src/syntax/color.rs b/src/syntax/color.rs new file mode 100644 index 00000000..716cb688 --- /dev/null +++ b/src/syntax/color.rs @@ -0,0 +1,28 @@ +/// Entities which can be colored by syntax highlighting. +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub enum ColorToken { + Comment, + + Bracket, + FuncName, + Colon, + + Key, + Equals, + Comma, + + Paren, + Brace, + + ExprIdent, + ExprString, + ExprNumber, + ExprSize, + ExprBool, + + Bold, + Italic, + Monospace, + + Invalid, +} diff --git a/src/syntax/expr.rs b/src/syntax/expr.rs new file mode 100644 index 00000000..a1680861 --- /dev/null +++ b/src/syntax/expr.rs @@ -0,0 +1,248 @@ +use super::*; + + +/// The arguments passed to a function. +#[derive(Debug, Clone, PartialEq)] +pub struct FuncArgs { + pub pos: Vec<Spanned<PosArg>>, + pub key: Vec<Spanned<KeyArg>>, +} + +impl FuncArgs { + /// Create an empty collection of arguments. + pub fn new() -> FuncArgs { + FuncArgs { + pos: vec![], + key: vec![], + } + } + + /// Add a positional argument. + pub fn add_pos(&mut self, arg: Spanned<PosArg>) { + self.pos.push(arg); + } + + /// Add a keyword argument. + pub fn add_key(&mut self, arg: Spanned<KeyArg>) { + self.key.push(arg); + } + + /// Force-extract the first positional argument. + pub fn get_pos<E: ExpressionKind>(&mut self) -> ParseResult<E> { + expect(self.get_pos_opt()) + } + + /// Extract the first positional argument. + pub fn get_pos_opt<E: ExpressionKind>(&mut self) -> ParseResult<Option<E>> { + Ok(if !self.pos.is_empty() { + let spanned = self.pos.remove(0); + Some(E::from_expr(spanned)?) + } else { + None + }) + } + + /// Iterator over positional arguments. + pub fn pos(&mut self) -> std::vec::IntoIter<Spanned<PosArg>> { + let vec = std::mem::replace(&mut self.pos, vec![]); + vec.into_iter() + } + + /// Force-extract a keyword argument. + pub fn get_key<E: ExpressionKind>(&mut self, name: &str) -> ParseResult<E> { + expect(self.get_key_opt(name)) + } + + /// Extract a keyword argument. + pub fn get_key_opt<E: ExpressionKind>(&mut self, name: &str) -> ParseResult<Option<E>> { + Ok(if let Some(index) = self.key.iter().position(|arg| arg.v.key.v.0 == name) { + let value = self.key.swap_remove(index).v.value; + Some(E::from_expr(value)?) + } else { + None + }) + } + + /// Extract any keyword argument. + pub fn get_key_next(&mut self) -> Option<Spanned<KeyArg>> { + self.key.pop() + } + + /// Iterator over all keyword arguments. + pub fn keys(&mut self) -> std::vec::IntoIter<Spanned<KeyArg>> { + let vec = std::mem::replace(&mut self.key, vec![]); + vec.into_iter() + } + + /// Clear the argument lists. + pub fn clear(&mut self) { + self.pos.clear(); + self.key.clear(); + } + + /// Whether both the positional and keyword argument lists are empty. + pub fn is_empty(&self) -> bool { + self.pos.is_empty() && self.key.is_empty() + } +} + +/// Extract the option expression kind from the option or return an error. +fn expect<E: ExpressionKind>(opt: ParseResult<Option<E>>) -> ParseResult<E> { + match opt { + Ok(Some(spanned)) => Ok(spanned), + Ok(None) => error!("expected {}", E::NAME), + Err(e) => Err(e), + } +} + +/// A positional argument passed to a function. +pub type PosArg = Expression; + +/// A keyword argument passed to a function. +#[derive(Debug, Clone, PartialEq)] +pub struct KeyArg { + pub key: Spanned<Ident>, + pub value: Spanned<Expression>, +} + +/// Either a positional or keyword argument. +#[derive(Debug, Clone, PartialEq)] +pub enum DynArg { + Pos(Spanned<PosArg>), + Key(Spanned<KeyArg>), +} + +/// An argument or return value. +#[derive(Clone, PartialEq)] +pub enum Expression { + Ident(Ident), + Str(String), + Num(f64), + Size(Size), + Bool(bool), +} + +impl Display for Expression { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + use Expression::*; + match self { + Ident(i) => write!(f, "{}", i), + Str(s) => write!(f, "{:?}", s), + Num(n) => write!(f, "{}", n), + Size(s) => write!(f, "{}", s), + Bool(b) => write!(f, "{}", b), + } + } +} + +debug_display!(Expression); + +pub struct Tuple; +pub struct Object; + +/// An identifier. +#[derive(Clone, PartialEq)] +pub struct Ident(pub String); + +impl Ident { + pub fn new<S>(ident: S) -> Option<Ident> where S: AsRef<str> + Into<String> { + if is_identifier(ident.as_ref()) { + Some(Ident(ident.into())) + } else { + None + } + } + + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + +impl Display for Ident { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +debug_display!(Ident); + +/// Whether this word is a valid identifier. +pub fn is_identifier(string: &str) -> bool { + let mut chars = string.chars(); + + match chars.next() { + Some('-') => {} + Some(c) if UnicodeXID::is_xid_start(c) => {} + _ => return false, + } + + while let Some(c) = chars.next() { + match c { + '.' | '-' => {} + c if UnicodeXID::is_xid_continue(c) => {} + _ => return false, + } + } + + true +} + +/// Kinds of expressions. +pub trait ExpressionKind: Sized { + const NAME: &'static str; + + /// Create from expression. + fn from_expr(expr: Spanned<Expression>) -> ParseResult<Self>; +} + +macro_rules! kind { + ($type:ty, $name:expr, $($patterns:tt)*) => { + impl ExpressionKind for $type { + const NAME: &'static str = $name; + + fn from_expr(expr: Spanned<Expression>) -> ParseResult<Self> { + #[allow(unreachable_patterns)] + Ok(match expr.v { + $($patterns)*, + _ => error!("expected {}", Self::NAME), + }) + } + } + }; +} + +kind!(Expression, "expression", e => e); +kind!(Ident, "identifier", Expression::Ident(ident) => ident); +kind!(String, "string", Expression::Str(string) => string); +kind!(f64, "number", Expression::Num(num) => num); +kind!(bool, "boolean", Expression::Bool(boolean) => boolean); +kind!(Size, "size", Expression::Size(size) => size); +kind!(ScaleSize, "number or size", + Expression::Size(size) => ScaleSize::Absolute(size), + Expression::Num(scale) => ScaleSize::Scaled(scale as f32) +); + +impl<T> ExpressionKind for Spanned<T> where T: ExpressionKind { + const NAME: &'static str = T::NAME; + + fn from_expr(expr: Spanned<Expression>) -> ParseResult<Spanned<T>> { + let span = expr.span; + T::from_expr(expr) + .map(|v| Spanned::new(v, span)) + } +} + +impl<T> ExpressionKind for Option<T> where T: ExpressionKind { + const NAME: &'static str = T::NAME; + + fn from_expr(expr: Spanned<Expression>) -> ParseResult<Option<T>> { + if let Expression::Ident(ident) = &expr.v { + match ident.as_str() { + "default" | "none" => return Ok(None), + _ => {}, + } + } + + T::from_expr(expr).map(|v| Some(v)) + } +} diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs index 10a509d2..11b35a06 100644 --- a/src/syntax/mod.rs +++ b/src/syntax/mod.rs @@ -6,312 +6,11 @@ use unicode_xid::UnicodeXID; use crate::func::LayoutFunc; use crate::size::{Size, ScaleSize}; + +pub type ParseResult<T> = crate::TypesetResult<T>; + +pub_use_mod!(color); +pub_use_mod!(expr); pub_use_mod!(tokens); pub_use_mod!(parsing); pub_use_mod!(span); - - -/// A tree representation of source code. -#[derive(Debug, PartialEq)] -pub struct SyntaxTree { - pub nodes: Vec<Spanned<Node>>, -} - -impl SyntaxTree { - /// Create an empty syntax tree. - pub fn new() -> SyntaxTree { - SyntaxTree { nodes: vec![] } - } -} - -/// A node in the syntax tree. -#[derive(Debug, PartialEq)] -pub enum Node { - /// Whitespace. - Space, - /// A line feed. - Newline, - /// Indicates that italics were toggled. - ToggleItalics, - /// Indicates that bolder text was toggled. - ToggleBolder, - /// Indicates that monospace was toggled. - ToggleMonospace, - /// Literal text. - Text(String), - /// A function invocation. - Func(FuncCall), -} - -/// A thing to be syntax highlighted. -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -pub enum ColorToken { - Comment, - Bracket, - FuncName, - Colon, - KeyArg, - Equals, - Comma, - ExprNumber, - ExprSize, - ExprStr, - ExprIdent, - ExprBool, - Bold, - Italic, - Monospace, -} - -/// An invocation of a function. -#[derive(Debug)] -pub struct FuncCall(pub Box<dyn LayoutFunc>); - -impl PartialEq for FuncCall { - fn eq(&self, other: &FuncCall) -> bool { - &self.0 == &other.0 - } -} - -/// The arguments passed to a function. -#[derive(Debug, Clone, PartialEq)] -pub struct FuncArgs { - pub pos: Vec<Spanned<PosArg>>, - pub key: Vec<Spanned<KeyArg>>, -} - -impl FuncArgs { - /// Create an empty collection of arguments. - pub fn new() -> FuncArgs { - FuncArgs { - pos: vec![], - key: vec![], - } - } - - /// Add a positional argument. - pub fn add_pos(&mut self, arg: Spanned<PosArg>) { - self.pos.push(arg); - } - - /// Add a keyword argument. - pub fn add_key(&mut self, arg: Spanned<KeyArg>) { - self.key.push(arg); - } - - /// Force-extract the first positional argument. - pub fn get_pos<E: ExpressionKind>(&mut self) -> ParseResult<E> { - expect(self.get_pos_opt()) - } - - /// Extract the first positional argument. - pub fn get_pos_opt<E: ExpressionKind>(&mut self) -> ParseResult<Option<E>> { - Ok(if !self.pos.is_empty() { - let spanned = self.pos.remove(0); - Some(E::from_expr(spanned)?) - } else { - None - }) - } - - /// Iterator over positional arguments. - pub fn pos(&mut self) -> std::vec::IntoIter<Spanned<PosArg>> { - let vec = std::mem::replace(&mut self.pos, vec![]); - vec.into_iter() - } - - /// Force-extract a keyword argument. - pub fn get_key<E: ExpressionKind>(&mut self, name: &str) -> ParseResult<E> { - expect(self.get_key_opt(name)) - } - - /// Extract a keyword argument. - pub fn get_key_opt<E: ExpressionKind>(&mut self, name: &str) -> ParseResult<Option<E>> { - Ok(if let Some(index) = self.key.iter().position(|arg| arg.v.key.v.0 == name) { - let value = self.key.swap_remove(index).v.value; - Some(E::from_expr(value)?) - } else { - None - }) - } - - /// Extract any keyword argument. - pub fn get_key_next(&mut self) -> Option<Spanned<KeyArg>> { - self.key.pop() - } - - /// Iterator over all keyword arguments. - pub fn keys(&mut self) -> std::vec::IntoIter<Spanned<KeyArg>> { - let vec = std::mem::replace(&mut self.key, vec![]); - vec.into_iter() - } - - /// Clear the argument lists. - pub fn clear(&mut self) { - self.pos.clear(); - self.key.clear(); - } - - /// Whether both the positional and keyword argument lists are empty. - pub fn is_empty(&self) -> bool { - self.pos.is_empty() && self.key.is_empty() - } -} - -/// Extract the option expression kind from the option or return an error. -fn expect<E: ExpressionKind>(opt: ParseResult<Option<E>>) -> ParseResult<E> { - match opt { - Ok(Some(spanned)) => Ok(spanned), - Ok(None) => error!("expected {}", E::NAME), - Err(e) => Err(e), - } -} - -/// A positional argument passed to a function. -pub type PosArg = Expression; - -/// A keyword argument passed to a function. -#[derive(Debug, Clone, PartialEq)] -pub struct KeyArg { - pub key: Spanned<Ident>, - pub value: Spanned<Expression>, -} - -/// Either a positional or keyword argument. -#[derive(Debug, Clone, PartialEq)] -pub enum DynArg { - Pos(Spanned<PosArg>), - Key(Spanned<KeyArg>), -} - -/// An argument or return value. -#[derive(Clone, PartialEq)] -pub enum Expression { - Ident(Ident), - Str(String), - Num(f64), - Size(Size), - Bool(bool), -} - -impl Display for Expression { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - use Expression::*; - match self { - Ident(i) => write!(f, "{}", i), - Str(s) => write!(f, "{:?}", s), - Num(n) => write!(f, "{}", n), - Size(s) => write!(f, "{}", s), - Bool(b) => write!(f, "{}", b), - } - } -} - -debug_display!(Expression); - -/// An identifier. -#[derive(Clone, PartialEq)] -pub struct Ident(pub String); - -impl Ident { - pub fn new<S>(ident: S) -> Option<Ident> where S: AsRef<str> + Into<String> { - if is_identifier(ident.as_ref()) { - Some(Ident(ident.into())) - } else { - None - } - } - - pub fn as_str(&self) -> &str { - self.0.as_str() - } -} - -impl Display for Ident { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -debug_display!(Ident); - -/// Whether this word is a valid identifier. -fn is_identifier(string: &str) -> bool { - let mut chars = string.chars(); - - match chars.next() { - Some('-') => {} - Some(c) if UnicodeXID::is_xid_start(c) => {} - _ => return false, - } - - while let Some(c) = chars.next() { - match c { - '.' | '-' => {} - c if UnicodeXID::is_xid_continue(c) => {} - _ => return false, - } - } - - true -} - -/// Kinds of expressions. -pub trait ExpressionKind: Sized { - const NAME: &'static str; - - /// Create from expression. - fn from_expr(expr: Spanned<Expression>) -> ParseResult<Self>; -} - -macro_rules! kind { - ($type:ty, $name:expr, $($patterns:tt)*) => { - impl ExpressionKind for $type { - const NAME: &'static str = $name; - - fn from_expr(expr: Spanned<Expression>) -> ParseResult<Self> { - #[allow(unreachable_patterns)] - Ok(match expr.v { - $($patterns)*, - _ => error!("expected {}", Self::NAME), - }) - } - } - }; -} - -kind!(Expression, "expression", e => e); -kind!(Ident, "identifier", Expression::Ident(ident) => ident); -kind!(String, "string", Expression::Str(string) => string); -kind!(f64, "number", Expression::Num(num) => num); -kind!(bool, "boolean", Expression::Bool(boolean) => boolean); -kind!(Size, "size", Expression::Size(size) => size); -kind!(ScaleSize, "number or size", - Expression::Size(size) => ScaleSize::Absolute(size), - Expression::Num(scale) => ScaleSize::Scaled(scale as f32) -); - -impl<T> ExpressionKind for Spanned<T> where T: ExpressionKind { - const NAME: &'static str = T::NAME; - - fn from_expr(expr: Spanned<Expression>) -> ParseResult<Spanned<T>> { - let span = expr.span; - T::from_expr(expr) - .map(|v| Spanned::new(v, span)) - } -} - -impl<T> ExpressionKind for Option<T> where T: ExpressionKind { - const NAME: &'static str = T::NAME; - - fn from_expr(expr: Spanned<Expression>) -> ParseResult<Option<T>> { - if let Expression::Ident(ident) = &expr.v { - match ident.as_str() { - "default" | "none" => return Ok(None), - _ => {}, - } - } - - T::from_expr(expr).map(|v| Some(v)) - } -} diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs index 4a50ef96..112c2f65 100644 --- a/src/syntax/parsing.rs +++ b/src/syntax/parsing.rs @@ -1,13 +1,55 @@ +use std::iter::Peekable; + use crate::func::Scope; use super::*; +use Token::*; + + +/// A tree representation of source code. +#[derive(Debug, PartialEq)] +pub struct SyntaxTree { + pub nodes: Vec<Spanned<Node>>, +} + +impl SyntaxTree { + /// Create an empty syntax tree. + pub fn new() -> SyntaxTree { + SyntaxTree { nodes: vec![] } + } +} + +/// A node in the syntax tree. +#[derive(Debug, PartialEq)] +pub enum Node { + /// A number of whitespace characters containing less than two newlines. + Space, + /// Whitespace characters with more than two newlines. + Newline, + /// Plain text. + Text(String), + /// Italics enabled / disabled. + ToggleItalic, + /// Bolder enabled / disabled. + ToggleBolder, + /// Monospace enabled / disabled. + ToggleMonospace, + /// A function invocation. + Func(FuncCall), +} +/// An invocation of a function. +#[derive(Debug)] +pub struct FuncCall(pub Box<dyn LayoutFunc>); -/// The result type for parsing. -pub type ParseResult<T> = crate::TypesetResult<T>; +impl PartialEq for FuncCall { + fn eq(&self, other: &FuncCall) -> bool { + &self.0 == &other.0 + } +} /// Parses source code into a syntax tree given a context. -pub fn parse(src: &str, ctx: ParseContext) -> ParseResult<SyntaxTree> { - unimplemented!() +pub fn parse(src: &str, ctx: ParseContext) -> SyntaxTree { + Parser::new(src, ctx).parse() } /// The context for parsing. @@ -16,3 +58,342 @@ pub struct ParseContext<'a> { /// The scope containing function definitions. pub scope: &'a Scope, } + +struct Parser<'s> { + src: &'s str, + ctx: ParseContext<'s>, + tokens: Peekable<Tokens<'s>>, + errors: Vec<Spanned<String>>, + colored: Vec<Spanned<ColorToken>>, + span: Span, +} + +macro_rules! defer { + ($($tts:tt)*) => ( + unimplemented!() + ); +} + +impl<'s> Parser<'s> { + fn new(src: &'s str, ctx: ParseContext<'s>) -> Parser<'s> { + Parser { + src, + ctx, + tokens: Tokens::new(src).peekable(), + errors: vec![], + colored: vec![], + span: Span::ZERO, + } + } + + fn parse(mut self) -> SyntaxTree { + let mut tree = SyntaxTree::new(); + + loop { + self.skip_whitespace(); + + let start = self.position(); + + let node = match self.next() { + Some(LeftBracket) => self.parse_func().map(|f| Node::Func(f)), + Some(Star) => Some(Node::ToggleBolder), + Some(Underscore) => Some(Node::ToggleItalic), + Some(Backtick) => Some(Node::ToggleMonospace), + Some(Text(text)) => Some(Node::Text(text.to_owned())), + Some(other) => { self.unexpected(other); None }, + None => break, + }; + + if let Some(node) = node { + let end = self.position(); + let span = Span { start, end }; + + tree.nodes.push(Spanned { v: node, span }); + } + } + + tree + } + + fn parse_func(&mut self) -> Option<FuncCall> { + let (name, args) = self.parse_func_header()?; + self.parse_func_call(name, args) + } + + fn parse_func_header(&mut self) -> Option<(Spanned<Ident>, FuncArgs)> { + defer! { self.eat_until(|t| t == RightBracket, true); } + + self.skip_whitespace(); + + let name = self.parse_func_name()?; + + self.skip_whitespace(); + + let args = match self.next() { + Some(Colon) => self.parse_func_args(), + Some(RightBracket) => FuncArgs::new(), + other => { + self.expected("colon or closing bracket", other); + FuncArgs::new() + } + }; + + Some((name, args)) + } + + fn parse_func_call( + &mut self, + name: Spanned<Ident>, + args: FuncArgs, + ) -> Option<FuncCall> { + unimplemented!() + } + + fn parse_func_name(&mut self) -> Option<Spanned<Ident>> { + match self.next() { + Some(ExprIdent(ident)) => { + self.color_span(ColorToken::FuncName, self.span(), true); + Some(Spanned { v: Ident(ident.to_string()), span: self.span() }) + } + other => { + self.expected("identifier", other); + None + } + } + } + + fn parse_func_args(&mut self) -> FuncArgs { + enum State { + Start, + Identifier(Spanned<Ident>), + Assignment(Spanned<Ident>), + Value, + } + + impl State { + fn expected(&self) -> &'static str { + match self { + State::Start => "value or key", + State::Identifier(_) => "comma or assignment", + State::Assignment(_) => "value", + State::Value => "comma", + } + } + } + + let mut args = FuncArgs::new(); + let mut state = State::Start; + + loop { + self.skip_whitespace(); + + /* + let token = self.next(); + match token { + Some(ExprIdent(ident)) => match state { + State::Start => { + state = State::Identifier(Spanned { + v: Ident(ident.to_string()), + span: self.span(), + }); + } + State::Identifier(prev) => { + self.expected(state.expected(), token); + args.add_pos(prev.map(|id| Expression::Ident(id))); + state = State::Identifier(Spanned { + v: Ident(ident.to_string()), + span: self.span(), + }); + } + State::Assignment(key) => { + let span = Span::merge(key.span, self.span()); + args.add_key(Spanned::new(KeyArg { + key, + value: Spanned { + v: Expression::Ident(Ident(ident.to_string())), + span: self.span(), + }, + }, span)); + state = State::Value; + } + State::Value => { + self.expected(state.expected(), token); + state = State::Identifier(Spanned { + v: Ident(ident.to_string()), + span: self.span(), + }); + } + } + + // Handle expressions. + Some(Expr(_)) | Some(LeftParen) | Some(LeftBrace) => { + let expr = match token.unwrap() { + Expr(e) => e, + LeftParen => self.parse_tuple(), + LeftBrace => self.parse_object(), + _ => unreachable!(), + } + } + + // Handle commas after values. + Some(Comma) => match state { + State::Identifier(ident) => { + args.add_pos(ident.map(|id| Expression::Ident(id))); + state = State::Start; + } + State::Value => state = State::Start, + _ => self.expected(state.expected(), token), + } + + // Handle the end of the function header. + Some(RightBracket) => { + match state { + State::Identifier(ident) => { + args.add_pos(ident.map(|id| Expression::Ident(id))); + } + State::Assignment(_) => { + self.expected(state.expected(), token); + } + _ => {} + } + + break; + } + } + */ + } + + args + } + + fn handle_expr(&mut self, expr: Spanned<Expression>) { + + } + + fn parse_tuple(&mut self) -> Spanned<Tuple> { + unimplemented!() + } + + fn parse_object(&mut self) -> Spanned<Object> { + unimplemented!() + } + + fn skip_whitespace(&mut self) { + self.eat_until(|t| match t { + Whitespace(_) | LineComment(_) | BlockComment(_) => false, + _ => true, + }, false) + } + + fn eat_until<F>(&mut self, mut f: F, eat_match: bool) + where F: FnMut(Token<'s>) -> bool { + while let Some(token) = self.tokens.peek() { + if f(token.v) { + if eat_match { + self.next(); + } + break; + } + + self.next(); + } + } + + fn next(&mut self) -> Option<Token<'s>> { + self.tokens.next().map(|spanned| { + self.color_token(&spanned.v, spanned.span); + self.span = spanned.span; + spanned.v + }) + } + + fn span(&self) -> Span { + self.span + } + + fn position(&self) -> Position { + self.span.end + } + + fn unexpected(&mut self, found: Token) { + self.errors.push(Spanned { + v: format!("unexpected {}", name(found)), + span: self.span(), + }); + } + + fn expected(&mut self, thing: &str, found: Option<Token>) { + let message = if let Some(found) = found { + format!("expected {}, found {}", thing, name(found)) + } else { + format!("expected {}", thing) + }; + + self.errors.push(Spanned { + v: message, + span: self.span(), + }); + } + + fn color_token(&mut self, token: &Token<'s>, span: Span) { + let colored = match token { + LineComment(_) | BlockComment(_) => Some(ColorToken::Comment), + StarSlash => Some(ColorToken::Invalid), + LeftBracket | RightBracket => Some(ColorToken::Bracket), + LeftParen | RightParen => Some(ColorToken::Paren), + LeftBrace | RightBrace => Some(ColorToken::Brace), + Colon => Some(ColorToken::Colon), + Comma => Some(ColorToken::Comma), + Equals => Some(ColorToken::Equals), + ExprIdent(_) => Some(ColorToken::ExprIdent), + ExprString(_) => Some(ColorToken::ExprString), + ExprNumber(_) => Some(ColorToken::ExprNumber), + ExprSize(_) => Some(ColorToken::ExprSize), + ExprBool(_) => Some(ColorToken::ExprBool), + _ => None, + }; + + if let Some(color) = colored { + self.colored.push(Spanned { v: color, span }); + } + } + + fn color_span(&mut self, color: ColorToken, span: Span, replace_last: bool) { + let token = Spanned { v: color, span }; + + if replace_last { + if let Some(last) = self.colored.last_mut() { + *last = token; + return; + } + } + + self.colored.push(token); + } +} + +fn name(token: Token) -> &'static str { + match token { + Whitespace(_) => "whitespace", + LineComment(_) | BlockComment(_) => "comment", + StarSlash => "end of block comment", + LeftBracket => "opening bracket", + RightBracket => "closing bracket", + LeftParen => "opening paren", + RightParen => "closing paren", + LeftBrace => "opening brace", + RightBrace => "closing brace", + Colon => "colon", + Comma => "comma", + Equals => "equals sign", + ExprIdent(_) => "identifier", + ExprString(_) => "string", + ExprNumber(_) => "number", + ExprSize(_) => "size", + ExprBool(_) => "bool", + Star => "star", + Underscore => "underscore", + Backtick => "backtick", + Text(_) => "text", + } +} diff --git a/src/syntax/span.rs b/src/syntax/span.rs index 9bf7cafb..10188ed4 100644 --- a/src/syntax/span.rs +++ b/src/syntax/span.rs @@ -19,7 +19,11 @@ impl<T> Spanned<T> { self.v } - pub fn map<V>(&self, new_v: V) -> Spanned<V> { + pub fn map<F, V>(self, f: F) -> Spanned<V> where F: FnOnce(T) -> V { + Spanned { v: f(self.v), span: self.span } + } + + pub fn map_v<V>(&self, new_v: V) -> Spanned<V> { Spanned { v: new_v, span: self.span } } } @@ -40,6 +44,8 @@ pub struct Span { } impl Span { + pub const ZERO: Span = Span { start: Position::ZERO, end: Position::ZERO }; + pub fn new(start: Position, end: Position) -> Span { Span { start, end } } @@ -78,6 +84,8 @@ pub struct Position { } impl Position { + pub const ZERO: Position = Position { line: 0, column: 0 }; + pub fn new(line: usize, column: usize) -> Position { Position { line, column } } diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs index 2e9dd35b..d355b3cc 100644 --- a/src/syntax/tokens.rs +++ b/src/syntax/tokens.rs @@ -6,12 +6,8 @@ use Token::*; use State::*; -pub fn tokenize(src: &str) -> Tokens { - Tokens::new(src) -} - /// A minimal semantic entity of source code. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Copy, Clone, PartialEq)] pub enum Token<'s> { /// One or more whitespace characters. The contained `usize` denotes the /// number of newlines that were contained in the whitespace. @@ -46,8 +42,16 @@ pub enum Token<'s> { /// An equals sign in a function header: `=`. Equals, - /// An expression in a function header. - Expr(Expression), + /// An identifier in a function header: `center`. + ExprIdent(&'s str), + /// A quoted string in a function header: `"..."`. + ExprString(&'s str), + /// A number in a function header: `3.14`. + ExprNumber(f64), + /// A size in a function header: `12pt`. + ExprSize(Size), + /// A boolean in a function header: `true | false`. + ExprBool(bool), /// A star in body-text. Star, @@ -60,6 +64,11 @@ pub enum Token<'s> { Text(&'s str), } +/// Decomposes text into a sequence of semantic tokens. +pub fn tokenize(src: &str) -> Tokens { + Tokens::new(src) +} + /// An iterator over the tokens of a string of source code. pub struct Tokens<'s> { src: &'s str, @@ -138,7 +147,7 @@ impl<'s> Iterator for Tokens<'s> { // Expressions or just strings. c => { - let word = self.read_string_until(|n| { + let text = self.read_string_until(|n| { match n { c if c.is_whitespace() => true, '\\' | '[' | ']' | '*' | '_' | '`' | ':' | '=' | @@ -148,9 +157,9 @@ impl<'s> Iterator for Tokens<'s> { }, false, -(c.len_utf8() as isize), 0); if self.state == Header { - self.parse_expr(word) + self.parse_expr(text) } else { - Text(word) + Text(text) } } }; @@ -169,7 +178,6 @@ impl<'s> Tokens<'s> { fn parse_block_comment(&mut self) -> Token<'s> { enum Last { Slash, Star, Other } - use Last::*; self.eat(); @@ -181,15 +189,15 @@ impl<'s> Tokens<'s> { BlockComment(self.read_string_until(|n| { match n { '/' => match last { - Star if depth == 0 => return true, - Star => depth -= 1, - _ => last = Slash + Last::Star if depth == 0 => return true, + Last::Star => depth -= 1, + _ => last = Last::Slash } '*' => match last { - Slash => depth += 1, - _ => last = Star, + Last::Slash => depth += 1, + _ => last = Last::Star, } - _ => last = Other, + _ => last = Last::Other, } false @@ -205,7 +213,7 @@ impl<'s> Tokens<'s> { fn parse_string(&mut self) -> Token<'s> { let mut escaped = false; - Expr(Expression::Str(self.read_string_until(|n| { + ExprString(self.read_string_until(|n| { if n == '"' && !escaped { return true; } else if n == '\\' { @@ -215,7 +223,7 @@ impl<'s> Tokens<'s> { } false - }, true, 0, -1).to_string())) + }, true, 0, -1)) } fn parse_escaped(&mut self) -> Token<'s> { @@ -236,19 +244,19 @@ impl<'s> Tokens<'s> { } } - fn parse_expr(&mut self, word: &'s str) -> Token<'s> { - if let Ok(b) = word.parse::<bool>() { - Expr(Expression::Bool(b)) - } else if let Ok(num) = word.parse::<f64>() { - Expr(Expression::Num(num)) - } else if let Ok(num) = parse_percentage(word) { - Expr(Expression::Num(num / 100.0)) - } else if let Ok(size) = word.parse::<Size>() { - Expr(Expression::Size(size)) - } else if let Some(ident) = Ident::new(word) { - Expr(Expression::Ident(ident)) + fn parse_expr(&mut self, text: &'s str) -> Token<'s> { + if let Ok(b) = text.parse::<bool>() { + ExprBool(b) + } else if let Ok(num) = text.parse::<f64>() { + ExprNumber(num) + } else if let Some(num) = parse_percentage(text) { + ExprNumber(num / 100.0) + } else if let Ok(size) = text.parse::<Size>() { + ExprSize(size) + } else if is_identifier(text) { + ExprIdent(text) } else { - Text(word) + Text(text) } } @@ -296,11 +304,11 @@ impl<'s> Tokens<'s> { } } -fn parse_percentage(word: &str) -> Result<f64, ()> { - if word.ends_with('%') { - word[.. word.len() - 1].parse::<f64>().map_err(|_| ()) +fn parse_percentage(text: &str) -> Option<f64> { + if text.ends_with('%') { + text[.. text.len() - 1].parse::<f64>().ok() } else { - Err(()) + None } } @@ -325,7 +333,7 @@ impl<'s> Characters<'s> { fn new(src: &'s str) -> Characters<'s> { Characters { iter: src.chars().peekable(), - position: Position::new(0, 0), + position: Position::ZERO, index: 0, } } |