summaryrefslogtreecommitdiff
path: root/src/syntax
diff options
context:
space:
mode:
Diffstat (limited to 'src/syntax')
-rw-r--r--src/syntax/expr.rs3
-rw-r--r--src/syntax/func.rs125
-rw-r--r--src/syntax/mod.rs382
-rw-r--r--src/syntax/parsing.rs207
-rw-r--r--src/syntax/span.rs40
-rw-r--r--src/syntax/tokens.rs90
6 files changed, 453 insertions, 394 deletions
diff --git a/src/syntax/expr.rs b/src/syntax/expr.rs
index c4feea74..74deda46 100644
--- a/src/syntax/expr.rs
+++ b/src/syntax/expr.rs
@@ -1,3 +1,4 @@
+use crate::size::ScaleSize;
use super::*;
@@ -126,6 +127,7 @@ impl Display for Object {
}
}
+/// A key-value pair in an object.
#[derive(Clone, PartialEq)]
pub struct Pair {
pub key: Spanned<Ident>,
@@ -144,7 +146,6 @@ debug_display!(Tuple);
debug_display!(Object);
debug_display!(Pair);
-
/// Kinds of expressions.
pub trait ExpressionKind: Sized {
/// The name of the expression in an `expected <name>` error.
diff --git a/src/syntax/func.rs b/src/syntax/func.rs
new file mode 100644
index 00000000..5b1ce6e8
--- /dev/null
+++ b/src/syntax/func.rs
@@ -0,0 +1,125 @@
+use super::*;
+
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct FuncHeader {
+ pub name: Spanned<Ident>,
+ pub args: FuncArgs,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct FuncArgs {
+ pub pos: Tuple,
+ pub key: Object,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum Arg {
+ Pos(Spanned<Expression>),
+ Key(Pair),
+}
+
+impl Arg {
+ /// The span or the value or combined span of key and value.
+ pub fn span(&self) -> Span {
+ match self {
+ Arg::Pos(item) => item.span,
+ Arg::Key(Pair { key, value }) => Span::merge(key.span, value.span),
+ }
+ }
+}
+
+impl FuncArgs {
+ pub fn new() -> FuncArgs {
+ FuncArgs {
+ pos: Tuple::new(),
+ key: Object::new(),
+ }
+ }
+
+ /// Add an argument.
+ pub fn add(&mut self, arg: Arg) {
+ match arg {
+ Arg::Pos(item) => self.add_pos(item),
+ Arg::Key(pair) => self.add_key_pair(pair),
+ }
+ }
+
+ /// Add a positional argument.
+ pub fn add_pos(&mut self, item: Spanned<Expression>) {
+ self.pos.add(item);
+ }
+
+ /// Add a keyword argument.
+ pub fn add_key(&mut self, key: Spanned<Ident>, value: Spanned<Expression>) {
+ self.key.add(key, value);
+ }
+
+ /// Add a keyword argument from an existing pair.
+ pub fn add_key_pair(&mut self, pair: Pair) {
+ self.key.add_pair(pair);
+ }
+
+ // /// Force-extract the first positional argument.
+ // pub fn get_pos<E: ExpressionKind>(&mut self) -> ParseResult<E> {
+ // expect(self.get_pos_opt())
+ // }
+
+ // /// Extract the first positional argument.
+ // pub fn get_pos_opt<E: ExpressionKind>(&mut self) -> ParseResult<Option<E>> {
+ // Ok(if !self.positional.items.is_empty() {
+ // let spanned = self.positional.items.remove(0);
+ // Some(E::from_expr(spanned)?)
+ // } else {
+ // None
+ // })
+ // }
+
+ // /// Force-extract a keyword argument.
+ // pub fn get_key<E: ExpressionKind>(&mut self, name: &str) -> ParseResult<E> {
+ // expect(self.get_key_opt(name))
+ // }
+
+ // /// Extract a keyword argument.
+ // pub fn get_key_opt<E: ExpressionKind>(&mut self, name: &str) -> ParseResult<Option<E>> {
+ // self.keyword.pairs.iter()
+ // .position(|p| p.key.v.0 == name)
+ // .map(|index| {
+ // let value = self.keyword.pairs.swap_remove(index).value;
+ // E::from_expr(value)
+ // })
+ // .transpose()
+ // }
+
+ // /// Iterator over positional arguments.
+ // pub fn iter_pos(&mut self) -> std::vec::IntoIter<Spanned<Expression>> {
+ // let tuple = std::mem::replace(&mut self.positional, Tuple::new());
+ // tuple.items.into_iter()
+ // }
+
+ // /// Iterator over all keyword arguments.
+ // pub fn iter_keys(&mut self) -> std::vec::IntoIter<Pair> {
+ // let object = std::mem::replace(&mut self.keyword, Object::new());
+ // object.pairs.into_iter()
+ // }
+
+ // /// Clear the argument lists.
+ // pub fn clear(&mut self) {
+ // self.positional.items.clear();
+ // self.keyword.pairs.clear();
+ // }
+
+ // /// Whether both the positional and keyword argument lists are empty.
+ // pub fn is_empty(&self) -> bool {
+ // self.positional.items.is_empty() && self.keyword.pairs.is_empty()
+ // }
+}
+
+// /// Extract the option expression kind from the option or return an error.
+// fn expect<E: ExpressionKind>(opt: ParseResult<Option<E>>) -> ParseResult<E> {
+// match opt {
+// Ok(Some(spanned)) => Ok(spanned),
+// Ok(None) => error!("expected {}", E::NAME),
+// Err(e) => Err(e),
+// }
+// }
diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs
index f644f051..75407f82 100644
--- a/src/syntax/mod.rs
+++ b/src/syntax/mod.rs
@@ -1,321 +1,161 @@
//! Tokenization and parsing of source code.
-use std::fmt::{self, Display, Formatter};
-use unicode_xid::UnicodeXID;
+use std::any::Any;
+use std::fmt::{self, Debug, Display, Formatter};
+use std::future::Future;
+use std::pin::Pin;
use serde::Serialize;
-use crate::func::LayoutFunc;
-use crate::size::{Size, ScaleSize};
-
-
-pub type ParseResult<T> = crate::TypesetResult<T>;
+use crate::error::Error;
+use crate::func::{Commands, Command};
+use crate::layout::{Layouted, LayoutContext};
+use crate::size::Size;
pub_use_mod!(expr);
+pub_use_mod!(func);
pub_use_mod!(tokens);
pub_use_mod!(parsing);
pub_use_mod!(span);
-
-/// A minimal semantic entity of source code.
-#[derive(Debug, Copy, Clone, PartialEq)]
-pub enum Token<'s> {
- /// One or more whitespace characters. The contained `usize` denotes the
- /// number of newlines that were contained in the whitespace.
- Whitespace(usize),
-
- /// A line comment with inner string contents `//<&'s str>\n`.
- LineComment(&'s str),
- /// A block comment with inner string contents `/*<&'s str>*/`. The comment
- /// can contain nested block comments.
- BlockComment(&'s str),
- /// An erroneous `*/` without an opening block comment.
- StarSlash,
-
- /// A left bracket: `[`.
- LeftBracket,
- /// A right bracket: `]`.
- RightBracket,
-
- /// A left parenthesis in a function header: `(`.
- LeftParen,
- /// A right parenthesis in a function header: `)`.
- RightParen,
- /// A left brace in a function header: `{`.
- LeftBrace,
- /// A right brace in a function header: `}`.
- RightBrace,
-
- /// A colon in a function header: `:`.
- Colon,
- /// A comma in a function header: `:`.
- Comma,
- /// An equals sign in a function header: `=`.
- Equals,
-
- /// An identifier in a function header: `center`.
- ExprIdent(&'s str),
- /// A quoted string in a function header: `"..."`.
- ExprStr(&'s str),
- /// A number in a function header: `3.14`.
- ExprNumber(f64),
- /// A size in a function header: `12pt`.
- ExprSize(Size),
- /// A boolean in a function header: `true | false`.
- ExprBool(bool),
-
- /// A star in body-text.
- Star,
- /// An underscore in body-text.
- Underscore,
- /// A backtick in body-text.
- Backtick,
-
- /// Any other consecutive string.
- Text(&'s str),
-}
-
-/// A tree representation of source code.
-#[derive(Debug, PartialEq)]
-pub struct SyntaxTree {
- pub nodes: Vec<Spanned<Node>>,
+/// Common syntax types.
+pub mod prelude {
+ pub use super::*;
}
-impl SyntaxTree {
- /// Create an empty syntax tree.
- pub fn new() -> SyntaxTree {
- SyntaxTree { nodes: vec![] }
- }
- /// Add a node to the tree.
- pub fn add(&mut self, node: Spanned<Node>) {
- self.nodes.push(node);
- }
+pub struct Parsed<T> {
+ pub output: T,
+ pub errors: SpanVec<Error>,
+ pub decorations: SpanVec<Decoration>,
}
-/// A node in the syntax tree.
-#[derive(PartialEq)]
-pub enum Node {
- /// A number of whitespace characters containing less than two newlines.
- Space,
- /// Whitespace characters with more than two newlines.
- Newline,
- /// Plain text.
- Text(String),
- /// Italics enabled / disabled.
- ToggleItalic,
- /// Bolder enabled / disabled.
- ToggleBolder,
- /// Monospace enabled / disabled.
- ToggleMonospace,
- /// A function invocation.
- Func(FuncCall),
-}
-
-impl Display for Node {
- fn fmt(&self, f: &mut Formatter) -> fmt::Result {
- match self {
- Node::Space => write!(f, "Space"),
- Node::Newline => write!(f, "Newline"),
- Node::Text(text) => write!(f, "{:?}", text),
- Node::ToggleItalic => write!(f, "ToggleItalic"),
- Node::ToggleBolder => write!(f, "ToggleBold"),
- Node::ToggleMonospace => write!(f, "ToggleMonospace"),
- Node::Func(func) => {
- if f.alternate() {
- write!(f, "{:#?}", func.0)
- } else {
- write!(f, "{:?}", func.0)
- }
- }
+impl<T> Parsed<T> {
+ pub fn map<F, U>(self, f: F) -> Parsed<U> where F: FnOnce(T) -> U {
+ Parsed {
+ output: f(self.output),
+ errors: self.errors,
+ decorations: self.decorations,
}
}
}
-debug_display!(Node);
+#[async_trait::async_trait(?Send)]
+pub trait Model: Debug + ModelBounds {
+ async fn layout<'a>(
+ &'a self,
+ ctx: LayoutContext<'_, '_>
+ ) -> Layouted<Commands<'a>>;
+}
-/// An invocation of a function.
-#[derive(Debug)]
-pub struct FuncCall(pub Box<dyn LayoutFunc>);
+pub type DynFuture<'a, T> = Pin<Box<dyn Future<Output=T> + 'a>>;
-impl PartialEq for FuncCall {
- fn eq(&self, other: &FuncCall) -> bool {
- &self.0 == &other.0
+impl dyn Model {
+ pub fn downcast<T>(&self) -> Option<&T> where T: Model + 'static {
+ self.as_any().downcast_ref::<T>()
}
}
-#[derive(Debug, Clone, PartialEq)]
-pub struct FuncHeader {
- pub name: Spanned<Ident>,
- pub args: FuncArgs,
+impl PartialEq for dyn Model {
+ fn eq(&self, other: &dyn Model) -> bool {
+ self.bound_eq(other)
+ }
}
-#[derive(Debug, Clone, PartialEq)]
-pub struct FuncArgs {
- pub positional: Tuple,
- pub keyword: Object,
+impl Clone for Box<dyn Model> {
+ fn clone(&self) -> Self {
+ self.bound_clone()
+ }
}
-#[derive(Debug, Clone, PartialEq)]
-pub enum Arg {
- Pos(Spanned<Expression>),
- Key(Pair),
+pub trait ModelBounds {
+ fn as_any(&self) -> &dyn Any;
+ fn bound_eq(&self, other: &dyn Model) -> bool;
+ fn bound_clone(&self) -> Box<dyn Model>;
}
-impl Arg {
- /// The span or the value or combined span of key and value.
- pub fn span(&self) -> Span {
- match self {
- Arg::Pos(spanned) => spanned.span,
- Arg::Key(Pair { key, value }) => Span::merge(key.span, value.span),
- }
+impl<T> ModelBounds for T where T: Model + PartialEq + Clone + 'static {
+ fn as_any(&self) -> &dyn Any {
+ self
}
-}
-impl FuncArgs {
- pub fn new() -> FuncArgs {
- FuncArgs {
- positional: Tuple::new(),
- keyword: Object::new(),
+ fn bound_eq(&self, other: &dyn Model) -> bool {
+ match other.as_any().downcast_ref::<Self>() {
+ Some(other) => self == other,
+ None => false,
}
}
- /// Add a positional argument.
- pub fn add_pos(&mut self, item: Spanned<Expression>) {
- self.positional.add(item);
- }
-
- /// Force-extract the first positional argument.
- pub fn get_pos<E: ExpressionKind>(&mut self) -> ParseResult<E> {
- expect(self.get_pos_opt())
- }
-
- /// Extract the first positional argument.
- pub fn get_pos_opt<E: ExpressionKind>(&mut self) -> ParseResult<Option<E>> {
- Ok(if !self.positional.items.is_empty() {
- let spanned = self.positional.items.remove(0);
- Some(E::from_expr(spanned)?)
- } else {
- None
- })
- }
-
- /// Add a keyword argument.
- pub fn add_key(&mut self, key: Spanned<Ident>, value: Spanned<Expression>) {
- self.keyword.add(key, value);
- }
-
- /// Add a keyword argument from an existing pair.
- pub fn add_key_pair(&mut self, pair: Pair) {
- self.keyword.add_pair(pair);
- }
-
- /// Force-extract a keyword argument.
- pub fn get_key<E: ExpressionKind>(&mut self, name: &str) -> ParseResult<E> {
- expect(self.get_key_opt(name))
- }
-
- /// Extract a keyword argument.
- pub fn get_key_opt<E: ExpressionKind>(&mut self, name: &str) -> ParseResult<Option<E>> {
- self.keyword.pairs.iter()
- .position(|p| p.key.v.0 == name)
- .map(|index| {
- let value = self.keyword.pairs.swap_remove(index).value;
- E::from_expr(value)
- })
- .transpose()
- }
-
- /// Iterator over positional arguments.
- pub fn iter_pos(&mut self) -> std::vec::IntoIter<Spanned<Expression>> {
- let tuple = std::mem::replace(&mut self.positional, Tuple::new());
- tuple.items.into_iter()
- }
-
- /// Iterator over all keyword arguments.
- pub fn iter_keys(&mut self) -> std::vec::IntoIter<Pair> {
- let object = std::mem::replace(&mut self.keyword, Object::new());
- object.pairs.into_iter()
- }
-
- /// Clear the argument lists.
- pub fn clear(&mut self) {
- self.positional.items.clear();
- self.keyword.pairs.clear();
- }
-
- /// Whether both the positional and keyword argument lists are empty.
- pub fn is_empty(&self) -> bool {
- self.positional.items.is_empty() && self.keyword.pairs.is_empty()
- }
-}
-
-/// Extract the option expression kind from the option or return an error.
-fn expect<E: ExpressionKind>(opt: ParseResult<Option<E>>) -> ParseResult<E> {
- match opt {
- Ok(Some(spanned)) => Ok(spanned),
- Ok(None) => error!("expected {}", E::NAME),
- Err(e) => Err(e),
+ fn bound_clone(&self) -> Box<dyn Model> {
+ Box::new(self.clone())
}
}
-#[derive(Debug, Clone, Eq, PartialEq, Serialize)]
-pub struct Colorization {
- pub tokens: Vec<Spanned<ColorToken>>,
+/// A tree representation of source code.
+#[derive(Debug, Clone, PartialEq)]
+pub struct SyntaxModel {
+ pub nodes: SpanVec<Node>,
}
-impl Colorization {
- pub fn new() -> Colorization {
- Colorization { tokens: vec![] }
- }
-
- pub fn add(&mut self, token: ColorToken, span: Span) {
- self.tokens.push(Spanned { v: token, span });
+impl SyntaxModel {
+ /// Create an empty syntax model.
+ pub fn new() -> SyntaxModel {
+ SyntaxModel { nodes: vec![] }
}
- pub fn replace_last(&mut self, token: ColorToken) {
- self.tokens.last_mut().expect("replace_last: no token").v = token;
+ /// Add a node to the model.
+ pub fn add(&mut self, node: Spanned<Node>) {
+ self.nodes.push(node);
}
}
-/// Entities which can be colored by syntax highlighting.
-#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize)]
-#[serde(rename_all = "camelCase")]
-pub enum ColorToken {
- Comment,
- Bracket,
- FuncName,
- Colon,
- Key,
- Equals,
- Comma,
- Paren,
- Brace,
- ExprIdent,
- ExprStr,
- ExprNumber,
- ExprSize,
- ExprBool,
- Bold,
- Italic,
- Monospace,
- Invalid,
-}
-
-#[derive(Debug, Clone, Eq, PartialEq, Serialize)]
-pub struct ErrorMap {
- pub errors: Vec<Spanned<String>>,
-}
-
-impl ErrorMap {
- pub fn new() -> ErrorMap {
- ErrorMap { errors: vec![] }
+#[async_trait::async_trait(?Send)]
+impl Model for SyntaxModel {
+ async fn layout<'a>(&'a self, ctx: LayoutContext<'_, '_>) -> Layouted<Commands<'a>> {
+ Layouted {
+ output: vec![Command::LayoutSyntaxModel(self)],
+ errors: vec![],
+ }
}
+}
- pub fn add(&mut self, message: impl Into<String>, span: Span) {
- self.errors.push(Spanned { v: message.into(), span });
+/// A node in the syntax tree.
+#[derive(Debug, Clone)]
+pub enum Node {
+ /// A number of whitespace characters containing less than two newlines.
+ Space,
+ /// Whitespace characters with more than two newlines.
+ Newline,
+ /// Plain text.
+ Text(String),
+ /// Italics enabled / disabled.
+ ToggleItalic,
+ /// Bolder enabled / disabled.
+ ToggleBolder,
+ /// Monospace enabled / disabled.
+ ToggleMonospace,
+ /// A submodel.
+ Model(Box<dyn Model>),
+}
+
+impl PartialEq for Node {
+ fn eq(&self, other: &Node) -> bool {
+ use Node::*;
+ match (self, other) {
+ (Space, Space) => true,
+ (Newline, Newline) => true,
+ (Text(a), Text(b)) => a == b,
+ (ToggleItalic, ToggleItalic) => true,
+ (ToggleBolder, ToggleBolder) => true,
+ (ToggleMonospace, ToggleMonospace) => true,
+ (Model(a), Model(b)) => a == b,
+ _ => false,
+ }
}
+}
- pub fn add_at(&mut self, message: impl Into<String>, pos: Position) {
- self.errors.push(Spanned { v: message.into(), span: Span::at(pos) })
- }
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+pub enum Decoration {
+ ValidFuncName,
+ InvalidFuncName,
+ ArgumentKey,
}
diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs
index f6d0b629..24bef7ce 100644
--- a/src/syntax/parsing.rs
+++ b/src/syntax/parsing.rs
@@ -3,9 +3,8 @@ use super::*;
use Token::*;
-/// Parses source code into a syntax tree given a context.
-pub fn parse(src: &str, ctx: ParseContext) -> (SyntaxTree, Colorization, ErrorMap) {
- Parser::new(src, ctx).parse()
+pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed<SyntaxModel> {
+ Parser::new(start, src, ctx).parse()
}
/// The context for parsing.
@@ -18,64 +17,68 @@ pub struct ParseContext<'a> {
struct Parser<'s> {
src: &'s str,
ctx: ParseContext<'s>,
- colorization: Colorization,
- error_map: ErrorMap,
tokens: Tokens<'s>,
peeked: Option<Option<Spanned<Token<'s>>>>,
position: Position,
last_position: Position,
+ errors: SpanVec<Error>,
+ decorations: SpanVec<Decoration>,
}
impl<'s> Parser<'s> {
- fn new(src: &'s str, ctx: ParseContext<'s>) -> Parser<'s> {
+ fn new(start: Position, src: &'s str, ctx: ParseContext<'s>) -> Parser<'s> {
Parser {
src,
ctx,
- error_map: ErrorMap::new(),
- colorization: Colorization::new(),
- tokens: Tokens::new(src),
+ tokens: tokenize(start, src),
peeked: None,
position: Position::ZERO,
last_position: Position::ZERO,
+ errors: vec![],
+ decorations: vec![],
}
}
/// The main parsing entrypoint.
- fn parse(mut self) -> (SyntaxTree, Colorization, ErrorMap) {
- let mut tree = SyntaxTree::new();
-
- loop {
- if let Some(spanned) = self.eat() {
- match spanned.v {
- LineComment(_) | BlockComment(_) => {}
-
- Whitespace(newlines) => {
- tree.add(spanned.map_v(if newlines >= 2 {
- Node::Newline
- } else {
- Node::Space
- }));
- }
-
- LeftBracket => {
- if let Some(func) = self.parse_func() {
- tree.add(func);
- }
- }
-
- Star => tree.add(spanned.map_v(Node::ToggleBolder)),
- Underscore => tree.add(spanned.map_v(Node::ToggleItalic)),
- Backtick => tree.add(spanned.map_v(Node::ToggleMonospace)),
- Text(text) => tree.add(spanned.map_v(Node::Text(text.to_owned()))),
-
- _ => self.unexpected(spanned),
+ fn parse(mut self) -> Parsed<SyntaxModel> {
+ let mut model = SyntaxModel::new();
+
+ while let Some(token) = self.eat() {
+ let mut span = token.span;
+ let node = match token.v {
+ LineComment(_) | BlockComment(_) => None,
+ Whitespace(newlines) => Some(if newlines >= 2 {
+ Node::Newline
+ } else {
+ Node::Space
+ }),
+
+ LeftBracket => self.parse_func().map(|spanned| {
+ span = spanned.span;
+ spanned.v
+ }),
+
+ Star => Some(Node::ToggleBolder),
+ Underscore => Some(Node::ToggleItalic),
+ Backtick => Some(Node::ToggleMonospace),
+ Text(text) => Some(Node::Text(text.to_owned())),
+
+ _ => {
+ self.unexpected(token);
+ None
}
- } else {
- break;
+ };
+
+ if let Some(v) = node {
+ model.add(Spanned { v, span });
}
}
- (tree, self.colorization, self.error_map)
+ Parsed {
+ output: model,
+ errors: self.errors,
+ decorations: self.decorations,
+ }
}
/// Parses a function including header and body with the cursor starting
@@ -90,12 +93,55 @@ impl<'s> Parser<'s> {
self.expected_at("closing bracket", self.pos());
}
- let call = self.parse_func_call(header)?;
+ let body = if self.peekv() == Some(LeftBracket) {
+ self.eat();
+
+ let start_index = self.tokens.index();
+ let start_position = self.tokens.pos();
+
+ let found = self.tokens.move_to_closing_bracket();
+
+ let end_index = self.tokens.index();
+ let end_position = self.tokens.pos();
+
+ let body = &self.src[start_index .. end_index];
+
+ self.position = end_position;
+
+ if found {
+ let next = self.eat().map(Spanned::value);
+ debug_assert_eq!(next, Some(RightBracket));
+ } else {
+ self.expected_at("closing bracket", self.pos());
+ }
+
+ Some(Spanned::new(body, Span::new(start_position, end_position)))
+ } else {
+ None
+ };
+
+ let header = header?;
+ let (parser, decoration) = match self.ctx.scope.get_parser(header.name.v.as_str()) {
+ Ok(parser) => (parser, Decoration::ValidFuncName),
+ Err(parser) => {
+ let error = Error::new(format!("unknown function: `{}`", header.name.v));
+ self.errors.push(Spanned::new(error, header.name.span));
+ (parser, Decoration::InvalidFuncName)
+ }
+ };
+
+ self.decorations.push(Spanned::new(decoration, header.name.span));
+
+ let parsed = parser(header, body, self.ctx);
+ self.errors.extend(offset_spans(parsed.errors, start));
+ self.decorations.extend(offset_spans(parsed.decorations, start));
+
+ let node = Node::Model(parsed.output);
let end = self.pos();
let span = Span { start, end };
- Some(Spanned { v: Node::Func(call), span })
+ Some(Spanned { v: node, span })
}
/// Parses a function header including the closing bracket.
@@ -125,7 +171,6 @@ impl<'s> Parser<'s> {
match self.peek() {
Some(Spanned { v: ExprIdent(ident), span }) => {
self.eat();
- self.colorization.replace_last(ColorToken::FuncName);
return Some(Spanned { v: Ident(ident.to_string()), span });
}
other => self.expected_found_or_at("identifier", other, self.pos()),
@@ -144,8 +189,7 @@ impl<'s> Parser<'s> {
match self.peekv() {
Some(RightBracket) | None => break,
_ => match self.parse_arg() {
- Some(Arg::Pos(item)) => args.add_pos(item),
- Some(Arg::Key(pair)) => args.add_key_pair(pair),
+ Some(arg) => args.add(arg),
None => {}
}
}
@@ -165,11 +209,11 @@ impl<'s> Parser<'s> {
let ident = Ident(ident.to_string());
if let Some(Equals) = self.peekv() {
- self.colorization.replace_last(ColorToken::Key);
-
self.eat();
self.skip_whitespace();
+ self.decorations.push(Spanned::new(Decoration::ArgumentKey, span));
+
self.parse_expr().map(|value| {
Arg::Key(Pair {
key: Spanned { v: ident, span },
@@ -251,42 +295,6 @@ impl<'s> Parser<'s> {
Spanned { v: Expression::Object(Object::new()), span }
}
- /// Parse the body of a function invocation.
- fn parse_func_call(&mut self, header: Option<FuncHeader>) -> Option<FuncCall> {
- let body = if self.peekv() == Some(LeftBracket) {
- self.eat();
-
- let start = self.tokens.index();
- let found = self.tokens.move_to_closing_bracket();
- let end = self.tokens.index();
-
- self.last_position = self.position;
- self.position = self.tokens.pos();
-
- let body = &self.src[start .. end];
-
- if found {
- let next = self.eat().map(Spanned::value);
- debug_assert_eq!(next, Some(RightBracket));
- } else {
- self.expected_at("closing bracket", self.pos());
- }
-
- Some(body)
- } else {
- None
- };
-
- let header = header?;
- let parser = self.ctx.scope.get_parser(header.name.v.as_str()).or_else(|| {
- let message = format!("unknown function: `{}`", header.name.v);
- self.error_map.add(message, header.name.span);
- None
- })?;
-
- parser(header, body, self.ctx).ok().map(|f| FuncCall(f))
- }
-
/// Skip all whitespace/comment tokens.
fn skip_whitespace(&mut self) {
self.eat_until(|t|
@@ -296,14 +304,16 @@ impl<'s> Parser<'s> {
/// Add an error about an `thing` which was expected but not found at the
/// given position.
fn expected_at(&mut self, thing: &str, pos: Position) {
- self.error_map.add_at(format!("expected {}", thing), pos);
+ let error = Error::new(format!("expected {}", thing));
+ self.errors.push(Spanned::new(error, Span::at(pos)));
}
/// Add an error about an expected `thing` which was not found, showing
/// what was found instead.
fn expected_found(&mut self, thing: &str, found: Spanned<Token>) {
let message = format!("expected {}, found {}", thing, name(found.v));
- self.error_map.add(message, found.span);
+ let error = Error::new(message);
+ self.errors.push(Spanned::new(error, found.span));
}
/// Add a found-error if `found` is some and a positional error, otherwise.
@@ -321,7 +331,8 @@ impl<'s> Parser<'s> {
/// Add an error about an unexpected token `found`.
fn unexpected(&mut self, found: Spanned<Token>) {
- self.error_map.add(format!("unexpected {}", name(found.v)), found.span);
+ let error = Error::new(format!("unexpected {}", name(found.v)));
+ self.errors.push(Spanned::new(error, found.span));
}
/// Consume tokens until the function returns true and only consume the last
@@ -348,10 +359,6 @@ impl<'s> Parser<'s> {
.unwrap_or_else(|| self.tokens.next());
if let Some(token) = token {
- if let Some(color) = color(token.v) {
- self.colorization.add(color, token.span);
- }
-
self.last_position = self.position;
self.position = token.span.end;
}
@@ -407,23 +414,3 @@ fn name(token: Token) -> &'static str {
Text(_) => "invalid identifier",
}
}
-
-/// The color token corresponding to a token.
-fn color(token: Token) -> Option<ColorToken> {
- Some(match token {
- LineComment(_) | BlockComment(_) => ColorToken::Comment,
- LeftBracket | RightBracket => ColorToken::Bracket,
- LeftParen | RightParen => ColorToken::Paren,
- LeftBrace | RightBrace => ColorToken::Brace,
- Colon => ColorToken::Colon,
- Comma => ColorToken::Comma,
- Equals => ColorToken::Equals,
- ExprIdent(_) => ColorToken::ExprIdent,
- ExprStr(_) => ColorToken::ExprStr,
- ExprNumber(_) => ColorToken::ExprNumber,
- ExprSize(_) => ColorToken::ExprSize,
- ExprBool(_) => ColorToken::ExprBool,
- StarSlash => ColorToken::Invalid,
- _ => return None,
- })
-}
diff --git a/src/syntax/span.rs b/src/syntax/span.rs
index df9a3520..eb39677e 100644
--- a/src/syntax/span.rs
+++ b/src/syntax/span.rs
@@ -1,6 +1,7 @@
//! Spans map elements to the part of source code they originate from.
use std::fmt::{self, Debug, Display, Formatter};
+use std::ops::{Add, AddAssign};
use serde::Serialize;
@@ -100,6 +101,30 @@ impl Position {
}
}
+impl Add for Position {
+ type Output = Position;
+
+ fn add(self, rhs: Position) -> Position {
+ if rhs.line == 0 {
+ Position {
+ line: self.line,
+ column: self.column + rhs.column
+ }
+ } else {
+ Position {
+ line: self.line + rhs.line,
+ column: rhs.column,
+ }
+ }
+ }
+}
+
+impl AddAssign for Position {
+ fn add_assign(&mut self, other: Position) {
+ *self = *self + other;
+ }
+}
+
impl Display for Position {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
write!(f, "{}:{}", self.line, self.column)
@@ -107,3 +132,18 @@ impl Display for Position {
}
debug_display!(Position);
+
+/// A vector of spanned things.
+pub type SpanVec<T> = Vec<Spanned<T>>;
+
+pub fn offset_spans<T>(
+ vec: SpanVec<T>,
+ start: Position,
+) -> impl Iterator<Item=Spanned<T>> {
+ vec.into_iter()
+ .map(move |mut spanned| {
+ spanned.span.start += start;
+ spanned.span.end += start;
+ spanned
+ })
+}
diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs
index 69d79965..0588bc6c 100644
--- a/src/syntax/tokens.rs
+++ b/src/syntax/tokens.rs
@@ -1,21 +1,80 @@
use std::iter::Peekable;
use std::str::Chars;
+use unicode_xid::UnicodeXID;
use super::*;
use Token::*;
use State::*;
+/// A minimal semantic entity of source code.
+#[derive(Debug, Copy, Clone, PartialEq)]
+pub enum Token<'s> {
+ /// One or more whitespace characters. The contained `usize` denotes the
+ /// number of newlines that were contained in the whitespace.
+ Whitespace(usize),
+
+ /// A line comment with inner string contents `//<&'s str>\n`.
+ LineComment(&'s str),
+ /// A block comment with inner string contents `/*<&'s str>*/`. The comment
+ /// can contain nested block comments.
+ BlockComment(&'s str),
+ /// An erroneous `*/` without an opening block comment.
+ StarSlash,
+
+ /// A left bracket: `[`.
+ LeftBracket,
+ /// A right bracket: `]`.
+ RightBracket,
+
+ /// A left parenthesis in a function header: `(`.
+ LeftParen,
+ /// A right parenthesis in a function header: `)`.
+ RightParen,
+ /// A left brace in a function header: `{`.
+ LeftBrace,
+ /// A right brace in a function header: `}`.
+ RightBrace,
+
+ /// A colon in a function header: `:`.
+ Colon,
+ /// A comma in a function header: `:`.
+ Comma,
+ /// An equals sign in a function header: `=`.
+ Equals,
+
+ /// An identifier in a function header: `center`.
+ ExprIdent(&'s str),
+ /// A quoted string in a function header: `"..."`.
+ ExprStr(&'s str),
+ /// A number in a function header: `3.14`.
+ ExprNumber(f64),
+ /// A size in a function header: `12pt`.
+ ExprSize(Size),
+ /// A boolean in a function header: `true | false`.
+ ExprBool(bool),
+
+ /// A star in body-text.
+ Star,
+ /// An underscore in body-text.
+ Underscore,
+ /// A backtick in body-text.
+ Backtick,
+
+ /// Any other consecutive string.
+ Text(&'s str),
+}
+
/// Decomposes text into a sequence of semantic tokens.
-pub fn tokenize(src: &str) -> Tokens {
- Tokens::new(src)
+pub fn tokenize(start: Position, src: &str) -> Tokens {
+ Tokens::new(start, src)
}
/// An iterator over the tokens of a string of source code.
pub struct Tokens<'s> {
src: &'s str,
state: State,
- stack: Vec<State>,
+ stack: Vec<(State, Position)>,
iter: Peekable<Chars<'s>>,
position: Position,
index: usize,
@@ -29,13 +88,13 @@ enum State {
}
impl<'s> Tokens<'s> {
- pub fn new(src: &'s str) -> Tokens<'s> {
+ pub fn new(start: Position, src: &'s str) -> Tokens<'s> {
Tokens {
src,
state: State::Body,
stack: vec![],
iter: src.chars().peekable(),
- position: Position::ZERO,
+ position: start,
index: 0,
}
}
@@ -47,7 +106,7 @@ impl<'s> Tokens<'s> {
}
/// The line-colunn position in the source at which the last token ends and
- /// next token will start.
+ /// next token will start. This position is
pub fn pos(&self) -> Position {
self.position
}
@@ -101,11 +160,13 @@ impl<'s> Iterator for Tokens<'s> {
// Functions.
'[' => {
- if self.state == Header || self.state == Body {
- self.stack.push(self.state);
- self.state = Header;
- } else {
- self.state = Body;
+ match self.state {
+ Header | Body => {
+ self.stack.push((self.state, start));
+ self.position = Position::new(0, '['.len_utf8());
+ self.state = Header;
+ }
+ StartBody => self.state = Body,
}
LeftBracket
@@ -114,7 +175,12 @@ impl<'s> Iterator for Tokens<'s> {
if self.state == Header && self.peek() == Some('[') {
self.state = StartBody;
} else {
- self.state = self.stack.pop().unwrap_or(Body);
+ if let Some((state, pos)) = self.stack.pop() {
+ self.state = state;
+ self.position = pos + self.position;
+ } else {
+ self.state = Body;
+ }
}
RightBracket
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-15 22:57:13 +0000