From 0a087cd28bbee5fcdffbb9d49b0ba9f413ad7f92 Mon Sep 17 00:00:00 2001 From: Laurenz Date: Fri, 24 Jan 2020 16:23:57 +0100 Subject: =?UTF-8?q?Reorganize=20modules=20=F0=9F=A7=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/syntax/expr.rs | 22 ++++--- src/syntax/func/keys.rs | 9 +-- src/syntax/func/maps.rs | 9 +-- src/syntax/func/mod.rs | 113 ++++++++----------------------- src/syntax/func/values.rs | 35 +++++----- src/syntax/mod.rs | 119 +++++++++++++++------------------ src/syntax/parsing.rs | 96 +++++++++++---------------- src/syntax/scope.rs | 83 +++++++++++++++++++++++ src/syntax/span.rs | 165 +++++++++++++++++++++++----------------------- src/syntax/tokens.rs | 39 ++++++++++- 10 files changed, 360 insertions(+), 330 deletions(-) create mode 100644 src/syntax/scope.rs (limited to 'src/syntax') diff --git a/src/syntax/expr.rs b/src/syntax/expr.rs index fe24c655..b4c0dfaa 100644 --- a/src/syntax/expr.rs +++ b/src/syntax/expr.rs @@ -1,4 +1,10 @@ -use super::*; +use std::fmt::{self, Display, Formatter}; + +use crate::error::Errors; +use crate::size::Size; +use super::func::{keys::Key, values::Value}; +use super::span::{Span, Spanned}; +use super::tokens::is_identifier; /// An argument or return value. @@ -91,6 +97,13 @@ pub struct Object { pub pairs: Vec, } +/// A key-value pair in an object. +#[derive(Clone, PartialEq)] +pub struct Pair { + pub key: Spanned, + pub value: Spanned, +} + impl Object { pub fn new() -> Object { Object { pairs: vec![] } @@ -162,13 +175,6 @@ impl Object { } } -/// A key-value pair in an object. -#[derive(Clone, PartialEq)] -pub struct Pair { - pub key: Spanned, - pub value: Spanned, -} - impl Display for Expr { fn fmt(&self, f: &mut Formatter) -> fmt::Result { use Expr::*; diff --git a/src/syntax/func/keys.rs b/src/syntax/func/keys.rs index dff97bde..116cd4e6 100644 --- a/src/syntax/func/keys.rs +++ b/src/syntax/func/keys.rs @@ -1,9 +1,10 @@ use crate::layout::prelude::*; +use super::values::AlignmentValue::{self, *}; use super::*; -use AxisKey::*; -use PaddingKey::*; -use AlignmentValue::*; +use self::AxisKey::*; +use self::PaddingKey::*; + pub trait Key { @@ -28,7 +29,7 @@ macro_rules! key { fn parse(key: Spanned<&str>) -> Option { match key.v { $($($p)|* => Some($r)),*, - other => None, + _ => None, } } } diff --git a/src/syntax/func/maps.rs b/src/syntax/func/maps.rs index 452c8ab1..8941024e 100644 --- a/src/syntax/func/maps.rs +++ b/src/syntax/func/maps.rs @@ -1,9 +1,11 @@ //! Deduplicating maps and keys for argument parsing. -use std::collections::HashMap; -use std::hash::Hash; -use crate::layout::{LayoutAxes, SpecificAxis, GenericAxis}; +use crate::error::Errors; +use crate::layout::prelude::*; use crate::size::{PSize, ValueBox}; +use crate::syntax::span::Spanned; +use super::keys::*; +use super::values::*; use super::*; @@ -179,7 +181,6 @@ impl PaddingMap { padding: &mut ValueBox> ) { use PaddingKey::*; - use SpecificAxis::*; let map = self.0.dedup(errors, |key, &val| { (match key { diff --git a/src/syntax/func/mod.rs b/src/syntax/func/mod.rs index b6691ab5..e66b4d6a 100644 --- a/src/syntax/func/mod.rs +++ b/src/syntax/func/mod.rs @@ -1,8 +1,10 @@ -use super::*; +use crate::error::{Error, Errors}; +use super::expr::{Expr, Ident, Tuple, Object, Pair}; +use super::span::{Span, Spanned}; -pub_use_mod!(maps); -pub_use_mod!(keys); -pub_use_mod!(values); +pub mod maps; +pub mod keys; +pub mod values; #[derive(Debug, Clone, PartialEq)] @@ -17,22 +19,6 @@ pub struct FuncArgs { pub key: Object, } -#[derive(Debug, Clone, PartialEq)] -pub enum Arg { - Pos(Spanned), - Key(Pair), -} - -impl Arg { - /// The span or the value or combined span of key and value. - pub fn span(&self) -> Span { - match self { - Arg::Pos(item) => item.span, - Arg::Key(Pair { key, value }) => Span::merge(key.span, value.span), - } - } -} - impl FuncArgs { pub fn new() -> FuncArgs { FuncArgs { @@ -42,10 +28,10 @@ impl FuncArgs { } /// Add an argument. - pub fn add(&mut self, arg: Arg) { + pub fn add(&mut self, arg: FuncArg) { match arg { - Arg::Pos(item) => self.add_pos(item), - Arg::Key(pair) => self.add_key_pair(pair), + FuncArg::Pos(item) => self.add_pos(item), + FuncArg::Key(pair) => self.add_key_pair(pair), } } @@ -64,74 +50,27 @@ impl FuncArgs { self.key.add_pair(pair); } - pub fn into_iter(self) -> impl Iterator { - self.pos.items.into_iter().map(|item| Arg::Pos(item)) - .chain(self.key.pairs.into_iter().map(|pair| Arg::Key(pair))) + pub fn into_iter(self) -> impl Iterator { + self.pos.items.into_iter().map(|item| FuncArg::Pos(item)) + .chain(self.key.pairs.into_iter().map(|pair| FuncArg::Key(pair))) } +} - // /// Force-extract the first positional argument. - // pub fn get_pos(&mut self) -> ParseResult { - // expect(self.get_pos_opt()) - // } - - // /// Extract the first positional argument. - // pub fn get_pos_opt(&mut self) -> ParseResult> { - // Ok(if !self.positional.items.is_empty() { - // let spanned = self.positional.items.remove(0); - // Some(E::from_expr(spanned)?) - // } else { - // None - // }) - // } - - // /// Force-extract a keyword argument. - // pub fn get_key(&mut self, name: &str) -> ParseResult { - // expect(self.get_key_opt(name)) - // } - - // /// Extract a keyword argument. - // pub fn get_key_opt(&mut self, name: &str) -> ParseResult> { - // self.keyword.pairs.iter() - // .position(|p| p.key.v.0 == name) - // .map(|index| { - // let value = self.keyword.pairs.swap_remove(index).value; - // E::from_expr(value) - // }) - // .transpose() - // } - - // /// Iterator over positional arguments. - // pub fn iter_pos(&mut self) -> std::vec::IntoIter> { - // let tuple = std::mem::replace(&mut self.positional, Tuple::new()); - // tuple.items.into_iter() - // } - - // /// Iterator over all keyword arguments. - // pub fn iter_keys(&mut self) -> std::vec::IntoIter { - // let object = std::mem::replace(&mut self.keyword, Object::new()); - // object.pairs.into_iter() - // } - - // /// Clear the argument lists. - // pub fn clear(&mut self) { - // self.positional.items.clear(); - // self.keyword.pairs.clear(); - // } - - // /// Whether both the positional and keyword argument lists are empty. - // pub fn is_empty(&self) -> bool { - // self.positional.items.is_empty() && self.keyword.pairs.is_empty() - // } +#[derive(Debug, Clone, PartialEq)] +pub enum FuncArg { + Pos(Spanned), + Key(Pair), } -// /// Extract the option expression kind from the option or return an error. -// fn expect(opt: ParseResult>) -> ParseResult { -// match opt { -// Ok(Some(spanned)) => Ok(spanned), -// Ok(None) => error!("expected {}", E::NAME), -// Err(e) => Err(e), -// } -// } +impl FuncArg { + /// The span or the value or combined span of key and value. + pub fn span(&self) -> Span { + match self { + FuncArg::Pos(item) => item.span, + FuncArg::Key(Pair { key, value }) => Span::merge(key.span, value.span), + } + } +} pub trait OptionExt: Sized { fn or_missing(self, errors: &mut Errors, span: Span, what: &str) -> Self; diff --git a/src/syntax/func/values.rs b/src/syntax/func/values.rs index b29b9726..a767aef6 100644 --- a/src/syntax/func/values.rs +++ b/src/syntax/func/values.rs @@ -1,12 +1,13 @@ +use std::fmt::{self, Display, Formatter}; use std::marker::PhantomData; use toddle::query::{FontStyle, FontWeight}; use crate::layout::prelude::*; -use crate::size::ScaleSize; +use crate::size::{Size, ScaleSize}; use crate::style::Paper; use super::*; -use AlignmentValue::*; +use self::AlignmentValue::*; pub trait Value { @@ -76,20 +77,6 @@ impl Value for Defaultable { } } -impl Value for Direction { - type Output = Self; - - fn parse(expr: Spanned) -> Result { - Ok(match Ident::parse(expr)?.as_str() { - "left-to-right" | "ltr" | "LTR" => Direction::LeftToRight, - "right-to-left" | "rtl" | "RTL" => Direction::RightToLeft, - "top-to-bottom" | "ttb" | "TTB" => Direction::TopToBottom, - "bottom-to-top" | "btt" | "BTT" => Direction::BottomToTop, - other => return Err(err!("invalid direction")) - }) - } -} - impl Value for FontStyle { type Output = Self; @@ -134,6 +121,20 @@ impl Value for Paper { } } +impl Value for Direction { + type Output = Self; + + fn parse(expr: Spanned) -> Result { + Ok(match Ident::parse(expr)?.as_str() { + "left-to-right" | "ltr" | "LTR" => LeftToRight, + "right-to-left" | "rtl" | "RTL" => RightToLeft, + "top-to-bottom" | "ttb" | "TTB" => TopToBottom, + "bottom-to-top" | "btt" | "BTT" => BottomToTop, + _ => return Err(err!("invalid direction")) + }) + } +} + #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] pub enum AlignmentValue { Align(Alignment), @@ -203,7 +204,7 @@ impl Value for AlignmentValue { "top" => Top, "right" => Right, "bottom" => Bottom, - other => return Err(err!("invalid alignment")) + _ => return Err(err!("invalid alignment")) }) } } diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs index 356535ae..d5afbca6 100644 --- a/src/syntax/mod.rs +++ b/src/syntax/mod.rs @@ -1,77 +1,25 @@ //! Tokenization and parsing of source code. use std::any::Any; -use std::fmt::{self, Debug, Display, Formatter}; -use std::future::Future; -use std::pin::Pin; +use std::fmt::Debug; +use async_trait::async_trait; use serde::Serialize; -use crate::error::{Error, Errors}; -use crate::func::{Commands, Command}; -use crate::layout::{Layouted, LayoutContext}; -use crate::size::Size; +use crate::layout::{LayoutContext, Layouted, Commands, Command}; +use self::span::{Spanned, SpanVec}; -pub_use_mod!(expr); -pub_use_mod!(func); -pub_use_mod!(tokens); -pub_use_mod!(parsing); -pub_use_mod!(span); +pub mod expr; +pub mod func; +pub mod span; -/// Common syntax types. -pub mod prelude { - pub use super::*; -} +pub_use_mod!(scope); +pub_use_mod!(parsing); +pub_use_mod!(tokens); -#[async_trait::async_trait(?Send)] +#[async_trait(?Send)] pub trait Model: Debug + ModelBounds { - async fn layout<'a>( - &'a self, - ctx: LayoutContext<'_, '_> - ) -> Layouted>; -} - -pub type DynFuture<'a, T> = Pin + 'a>>; - -impl dyn Model { - pub fn downcast(&self) -> Option<&T> where T: Model + 'static { - self.as_any().downcast_ref::() - } -} - -impl PartialEq for dyn Model { - fn eq(&self, other: &dyn Model) -> bool { - self.bound_eq(other) - } -} - -impl Clone for Box { - fn clone(&self) -> Self { - self.bound_clone() - } -} - -pub trait ModelBounds { - fn as_any(&self) -> &dyn Any; - fn bound_eq(&self, other: &dyn Model) -> bool; - fn bound_clone(&self) -> Box; -} - -impl ModelBounds for T where T: Model + PartialEq + Clone + 'static { - fn as_any(&self) -> &dyn Any { - self - } - - fn bound_eq(&self, other: &dyn Model) -> bool { - match other.as_any().downcast_ref::() { - Some(other) => self == other, - None => false, - } - } - - fn bound_clone(&self) -> Box { - Box::new(self.clone()) - } + async fn layout<'a>(&'a self, ctx: LayoutContext<'_, '_>) -> Layouted>; } /// A tree representation of source code. @@ -92,7 +40,7 @@ impl SyntaxModel { } } -#[async_trait::async_trait(?Send)] +#[async_trait(?Send)] impl Model for SyntaxModel { async fn layout<'a>(&'a self, _: LayoutContext<'_, '_>) -> Layouted> { Layouted { @@ -144,3 +92,44 @@ pub enum Decoration { InvalidFuncName, ArgumentKey, } + +impl dyn Model { + pub fn downcast(&self) -> Option<&T> where T: Model + 'static { + self.as_any().downcast_ref::() + } +} + +impl PartialEq for dyn Model { + fn eq(&self, other: &dyn Model) -> bool { + self.bound_eq(other) + } +} + +impl Clone for Box { + fn clone(&self) -> Self { + self.bound_clone() + } +} + +pub trait ModelBounds { + fn as_any(&self) -> &dyn Any; + fn bound_eq(&self, other: &dyn Model) -> bool; + fn bound_clone(&self) -> Box; +} + +impl ModelBounds for T where T: Model + PartialEq + Clone + 'static { + fn as_any(&self) -> &dyn Any { + self + } + + fn bound_eq(&self, other: &dyn Model) -> bool { + match other.as_any().downcast_ref::() { + Some(other) => self == other, + None => false, + } + } + + fn bound_clone(&self) -> Box { + Box::new(self.clone()) + } +} diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs index e726a2e0..3e3e827f 100644 --- a/src/syntax/parsing.rs +++ b/src/syntax/parsing.rs @@ -1,6 +1,10 @@ -use crate::func::Scope; +use crate::error::Errors; +use super::expr::*; +use super::func::{FuncHeader, FuncArgs, FuncArg}; +use super::scope::Scope; +use super::span::{Position, Span, Spanned, SpanVec, offset_spans}; +use super::tokens::{Token, Tokens, TokenizationMode}; use super::*; -use Token::*; /// The context for parsing. @@ -37,13 +41,13 @@ pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed if newlines >= 2 { + Token::Space(newlines) => if newlines >= 2 { Node::Newline } else { Node::Space }, - Function { header, body, terminated } => { + Token::Function { header, body, terminated } => { let parsed: Parsed = FuncParser::new(header, body, ctx).parse(); errors.extend(offset_spans(parsed.errors, span.start)); @@ -56,15 +60,15 @@ pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed Node::ToggleBolder, - Underscore => Node::ToggleItalic, - Backtick => Node::ToggleMonospace, - Text(text) => Node::Text(text.to_owned()), + Token::Star => Node::ToggleBolder, + Token::Underscore => Node::ToggleItalic, + Token::Backtick => Node::ToggleMonospace, + Token::Text(text) => Node::Text(text.to_owned()), - LineComment(_) | BlockComment(_) => continue, + Token::LineComment(_) | Token::BlockComment(_) => continue, other => { - errors.push(err!(span; "unexpected {}", name(other))); + errors.push(err!(span; "unexpected {}", other.name())); continue; } }; @@ -140,7 +144,7 @@ impl<'s> FuncParser<'s> { self.skip_whitespace(); let name = match self.eat() { - Some(Spanned { v: ExprIdent(ident), span }) => { + Some(Spanned { v: Token::ExprIdent(ident), span }) => { Spanned { v: Ident(ident.to_string()), span } } other => { @@ -151,7 +155,7 @@ impl<'s> FuncParser<'s> { self.skip_whitespace(); let args = match self.eat().map(Spanned::value) { - Some(Colon) => self.parse_func_args(), + Some(Token::Colon) => self.parse_func_args(), Some(_) => { self.expected_at("colon", name.span.end); FuncArgs::new() @@ -179,38 +183,38 @@ impl<'s> FuncParser<'s> { } /// Parse a positional or keyword argument. - fn parse_arg(&mut self) -> Option { + fn parse_arg(&mut self) -> Option { let first = self.peek()?; let span = first.span; - let arg = if let ExprIdent(ident) = first.v { + let arg = if let Token::ExprIdent(ident) = first.v { self.eat(); self.skip_whitespace(); let ident = Ident(ident.to_string()); - if let Some(Equals) = self.peekv() { + if let Some(Token::Equals) = self.peekv() { self.eat(); self.skip_whitespace(); self.decorations.push(Spanned::new(Decoration::ArgumentKey, span)); self.parse_expr().map(|value| { - Arg::Key(Pair { + FuncArg::Key(Pair { key: Spanned { v: ident, span }, value, }) }) } else { - Some(Arg::Pos(Spanned::new(Expr::Ident(ident), span))) + Some(FuncArg::Pos(Spanned::new(Expr::Ident(ident), span))) } } else { - self.parse_expr().map(|expr| Arg::Pos(expr)) + self.parse_expr().map(|expr| FuncArg::Pos(expr)) }; if let Some(arg) = &arg { self.skip_whitespace(); match self.peekv() { - Some(Comma) => { self.eat(); } + Some(Token::Comma) => { self.eat(); } Some(_) => self.expected_at("comma", arg.span().end), _ => {} } @@ -228,11 +232,11 @@ impl<'s> FuncParser<'s> { let spanned = |v| Spanned { v, span: first.span }; Some(match first.v { - ExprIdent(i) => { + Token::ExprIdent(i) => { self.eat(); spanned(Expr::Ident(Ident(i.to_string()))) } - ExprStr { string, terminated } => { + Token::ExprStr { string, terminated } => { if !terminated { self.expected_at("quote", first.span.end); } @@ -240,12 +244,13 @@ impl<'s> FuncParser<'s> { self.eat(); spanned(Expr::Str(string.to_string())) } - ExprNumber(n) => { self.eat(); spanned(Expr::Number(n)) } - ExprSize(s) => { self.eat(); spanned(Expr::Size(s)) } - ExprBool(b) => { self.eat(); spanned(Expr::Bool(b)) } + Token::ExprNumber(n) => { self.eat(); spanned(Expr::Number(n)) } + Token::ExprSize(s) => { self.eat(); spanned(Expr::Size(s)) } + Token::ExprBool(b) => { self.eat(); spanned(Expr::Bool(b)) } + + Token::LeftParen => self.parse_tuple(), + Token::LeftBrace => self.parse_object(), - LeftParen => self.parse_tuple(), - LeftBrace => self.parse_object(), _ => return None, }) } @@ -255,7 +260,7 @@ impl<'s> FuncParser<'s> { let start = self.pos(); // TODO: Do the thing. - self.eat_until(|t| t == RightParen, true); + self.eat_until(|t| t == Token::RightParen, true); let end = self.pos(); let span = Span { start, end }; @@ -268,7 +273,7 @@ impl<'s> FuncParser<'s> { let start = self.pos(); // TODO: Do the thing. - self.eat_until(|t| t == RightBrace, true); + self.eat_until(|t| t == Token::RightBrace, true); let end = self.pos(); let span = Span { start, end }; @@ -278,15 +283,16 @@ impl<'s> FuncParser<'s> { /// Skip all whitespace/comment tokens. fn skip_whitespace(&mut self) { - self.eat_until(|t| - !matches!(t, Space(_) | LineComment(_) | BlockComment(_)), false) + self.eat_until(|t| !matches!(t, + Token::Space(_) | Token::LineComment(_) | + Token::BlockComment(_)), false) } /// Add an error about an expected `thing` which was not found, showing /// what was found instead. fn expected_found(&mut self, thing: &str, found: Spanned) { self.errors.push(err!(found.span; - "expected {}, found {}", thing, name(found.v))); + "expected {}, found {}", thing, found.v.name())); } /// Add an error about an `thing` which was expected but not found at the @@ -348,31 +354,3 @@ impl<'s> FuncParser<'s> { .unwrap_or_else(|| self.tokens.pos()) } } - -/// The name of a token in an `(un)expected <...>` error. -fn name(token: Token) -> &'static str { - match token { - Space(_) => "space", - LineComment(_) | BlockComment(_) => "comment", - Function { .. } => "function", - LeftParen => "opening paren", - RightParen => "closing paren", - LeftBrace => "opening brace", - RightBrace => "closing brace", - Colon => "colon", - Comma => "comma", - Equals => "equals sign", - ExprIdent(_) => "identifier", - ExprStr { .. } => "string", - ExprNumber(_) => "number", - ExprSize(_) => "size", - ExprBool(_) => "boolean", - Star => "star", - Underscore => "underscore", - Backtick => "backtick", - Text(_) => "invalid identifier", - Invalid("]") => "closing bracket", - Invalid("*/") => "end of block comment", - Invalid(_) => "invalid token", - } -} diff --git a/src/syntax/scope.rs b/src/syntax/scope.rs new file mode 100644 index 00000000..2aae331d --- /dev/null +++ b/src/syntax/scope.rs @@ -0,0 +1,83 @@ +use std::collections::HashMap; +use std::fmt::{self, Debug, Formatter}; + +use crate::func::ParseFunc; +use super::func::FuncHeader; +use super::parsing::{ParseContext, Parsed}; +use super::span::Spanned; +use super::Model; + + +/// A map from identifiers to function parsers. +pub struct Scope { + parsers: HashMap>, + fallback: Box +} + +impl Scope { + /// Create a new empty scope with a fallback parser that is invoked when no + /// match is found. + pub fn new() -> Scope + where F: ParseFunc + Model + 'static { + Scope { + parsers: HashMap::new(), + fallback: parser::(()), + } + } + + /// Create a new scope with the standard functions contained. + pub fn with_std() -> Scope { + crate::library::std() + } + + /// Associate the given name with a type that is parseable into a function. + pub fn add(&mut self, name: &str) + where F: ParseFunc + Model + 'static { + self.add_with_meta::(name, ()); + } + + /// Add a parseable type with additional metadata that is given to the + /// parser (other than the default of `()`). + pub fn add_with_meta(&mut self, name: &str, metadata: ::Meta) + where F: ParseFunc + Model + 'static { + self.parsers.insert( + name.to_owned(), + parser::(metadata), + ); + } + + /// Return the parser with the given name if there is one. + pub fn get_parser(&self, name: &str) -> Result<&Parser, &Parser> { + self.parsers.get(name) + .map(|x| &**x) + .ok_or_else(|| &*self.fallback) + } + + /// Return the fallback parser. + pub fn get_fallback_parser(&self) -> &Parser { + &*self.fallback + } +} + +impl Debug for Scope { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "Scope ")?; + write!(f, "{:?}", self.parsers.keys()) + } +} + +/// A function which parses the source of a function into a model type which +/// implements [`Model`]. +type Parser = dyn Fn( + FuncHeader, + Option>, + ParseContext, +) -> Parsed>; + +fn parser(metadata: ::Meta) -> Box +where F: ParseFunc + Model + 'static { + Box::new(move |h, b, c| { + F::parse(h, b, c, metadata.clone()) + .map(|model| Box::new(model) as Box) + }) +} diff --git a/src/syntax/span.rs b/src/syntax/span.rs index e049861f..f5bd4caf 100644 --- a/src/syntax/span.rs +++ b/src/syntax/span.rs @@ -5,45 +5,56 @@ use std::ops::{Add, Sub}; use serde::Serialize; -/// Annotates a value with the part of the source code it corresponds to. -#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)] -pub struct Spanned { - pub v: T, - pub span: Span, +/// A line-column position in source code. +#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize)] +pub struct Position { + /// The 0-indexed line (inclusive). + pub line: usize, + /// The 0-indexed column (inclusive). + pub column: usize, } -impl Spanned { - pub fn new(v: T, span: Span) -> Spanned { - Spanned { v, span } - } +impl Position { + pub const ZERO: Position = Position { line: 0, column: 0 }; - pub fn value(self) -> T { - self.v + pub fn new(line: usize, column: usize) -> Position { + Position { line, column } } +} - pub fn map(self, f: F) -> Spanned where F: FnOnce(T) -> V { - Spanned { v: f(self.v), span: self.span } - } +impl Add for Position { + type Output = Position; - pub fn map_span(mut self, f: F) -> Spanned where F: FnOnce(Span) -> Span { - self.span = f(self.span); - self + fn add(self, rhs: Position) -> Position { + if rhs.line == 0 { + Position { + line: self.line, + column: self.column + rhs.column + } + } else { + Position { + line: self.line + rhs.line, + column: rhs.column, + } + } } } -impl Display for Spanned where T: std::fmt::Display { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!(f, "({}, {}, ", self.span.start, self.span.end)?; - self.v.fmt(f)?; - write!(f, ")") - } -} +impl Sub for Position { + type Output = Position; -impl Debug for Spanned where T: std::fmt::Debug { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!(f, "({}, {}, ", self.span.start, self.span.end)?; - self.v.fmt(f)?; - write!(f, ")") + fn sub(self, rhs: Position) -> Position { + if self.line == rhs.line { + Position { + line: 0, + column: self.column - rhs.column + } + } else { + Position { + line: self.line - rhs.line, + column: self.column, + } + } } } @@ -84,65 +95,37 @@ impl Span { } } -impl Display for Span { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!(f, "({}, {})", self.start, self.end) - } -} - -debug_display!(Span); - -/// A line-column position in source code. -#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize)] -pub struct Position { - /// The 0-indexed line (inclusive). - pub line: usize, - /// The 0-indexed column (inclusive). - pub column: usize, +/// Annotates a value with the part of the source code it corresponds to. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)] +pub struct Spanned { + pub v: T, + pub span: Span, } -impl Position { - pub const ZERO: Position = Position { line: 0, column: 0 }; +impl Spanned { + pub fn new(v: T, span: Span) -> Spanned { + Spanned { v, span } + } - pub fn new(line: usize, column: usize) -> Position { - Position { line, column } + pub fn value(self) -> T { + self.v } -} -impl Add for Position { - type Output = Position; + pub fn map(self, f: F) -> Spanned where F: FnOnce(T) -> V { + Spanned { v: f(self.v), span: self.span } + } - fn add(self, rhs: Position) -> Position { - if rhs.line == 0 { - Position { - line: self.line, - column: self.column + rhs.column - } - } else { - Position { - line: self.line + rhs.line, - column: rhs.column, - } - } + pub fn map_span(mut self, f: F) -> Spanned where F: FnOnce(Span) -> Span { + self.span = f(self.span); + self } } -impl Sub for Position { - type Output = Position; +/// A vector of spanned things. +pub type SpanVec = Vec>; - fn sub(self, rhs: Position) -> Position { - if self.line == rhs.line { - Position { - line: 0, - column: self.column - rhs.column - } - } else { - Position { - line: self.line - rhs.line, - column: self.column, - } - } - } +pub fn offset_spans(vec: SpanVec, start: Position) -> impl Iterator> { + vec.into_iter().map(move |s| s.map_span(|span| span.offset(start))) } impl Display for Position { @@ -151,11 +134,27 @@ impl Display for Position { } } -debug_display!(Position); +impl Display for Span { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "({}, {})", self.start, self.end) + } +} -/// A vector of spanned things. -pub type SpanVec = Vec>; +impl Display for Spanned where T: std::fmt::Display { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "({}, {}, ", self.span.start, self.span.end)?; + self.v.fmt(f)?; + write!(f, ")") + } +} -pub fn offset_spans(vec: SpanVec, start: Position) -> impl Iterator> { - vec.into_iter().map(move |s| s.map_span(|span| span.offset(start))) +impl Debug for Spanned where T: std::fmt::Debug { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "({}, {}, ", self.span.start, self.span.end)?; + self.v.fmt(f)?; + write!(f, ")") + } } + +debug_display!(Position); +debug_display!(Span); diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs index d0adbf60..0a8e2f17 100644 --- a/src/syntax/tokens.rs +++ b/src/syntax/tokens.rs @@ -2,9 +2,11 @@ use std::iter::Peekable; use std::str::Chars; use unicode_xid::UnicodeXID; -use super::*; -use Token::*; -use TokenizationMode::*; +use crate::size::Size; +use super::span::{Position, Span, Spanned}; + +use self::Token::*; +use self::TokenizationMode::*; /// A minimal semantic entity of source code. @@ -68,6 +70,37 @@ pub enum Token<'s> { Invalid(&'s str), } +impl<'s> Token<'s> { + /// The natural-language name for this token for use in error messages. + pub fn name(self) -> &'static str { + match self { + Space(_) => "space", + LineComment(_) => "line comment", + BlockComment(_) => "block comment", + Function { .. } => "function", + LeftParen => "opening paren", + RightParen => "closing paren", + LeftBrace => "opening brace", + RightBrace => "closing brace", + Colon => "colon", + Comma => "comma", + Equals => "equals sign", + ExprIdent(_) => "identifier", + ExprStr { .. } => "string", + ExprNumber(_) => "number", + ExprSize(_) => "size", + ExprBool(_) => "boolean", + Star => "star", + Underscore => "underscore", + Backtick => "backtick", + Text(_) => "invalid identifier", + Invalid("]") => "closing bracket", + Invalid("*/") => "end of block comment", + Invalid(_) => "invalid token", + } + } +} + /// An iterator over the tokens of a string of source code. pub struct Tokens<'s> { src: &'s str, -- cgit v1.2.3