summaryrefslogtreecommitdiff
path: root/src/syntax
diff options
context:
space:
mode:
Diffstat (limited to 'src/syntax')
-rw-r--r--src/syntax/expr.rs105
-rw-r--r--src/syntax/func.rs8
-rw-r--r--src/syntax/mod.rs21
-rw-r--r--src/syntax/parsing.rs374
-rw-r--r--src/syntax/span.rs46
-rw-r--r--src/syntax/tokens.rs180
6 files changed, 351 insertions, 383 deletions
diff --git a/src/syntax/expr.rs b/src/syntax/expr.rs
index 74deda46..34a1c6bf 100644
--- a/src/syntax/expr.rs
+++ b/src/syntax/expr.rs
@@ -4,7 +4,7 @@ use super::*;
/// An argument or return value.
#[derive(Clone, PartialEq)]
-pub enum Expression {
+pub enum Expr {
Ident(Ident),
Str(String),
Number(f64),
@@ -14,9 +14,24 @@ pub enum Expression {
Object(Object),
}
-impl Display for Expression {
+impl Expr {
+ pub fn name(&self) -> &'static str {
+ use Expr::*;
+ match self {
+ Ident(_) => "identifier",
+ Str(_) => "string",
+ Number(_) => "number",
+ Size(_) => "size",
+ Bool(_) => "boolean",
+ Tuple(_) => "tuple",
+ Object(_) => "object",
+ }
+ }
+}
+
+impl Display for Expr {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
- use Expression::*;
+ use Expr::*;
match self {
Ident(i) => write!(f, "{}", i),
Str(s) => write!(f, "{:?}", s),
@@ -29,6 +44,8 @@ impl Display for Expression {
}
}
+debug_display!(Expr);
+
/// An identifier.
#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub struct Ident(pub String);
@@ -53,10 +70,15 @@ impl Display for Ident {
}
}
+debug_display!(Ident);
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct StringLike(pub String);
+
/// A sequence of expressions.
#[derive(Clone, PartialEq)]
pub struct Tuple {
- pub items: Vec<Spanned<Expression>>,
+ pub items: Vec<Spanned<Expr>>,
}
impl Tuple {
@@ -64,7 +86,7 @@ impl Tuple {
Tuple { items: vec![] }
}
- pub fn add(&mut self, item: Spanned<Expression>) {
+ pub fn add(&mut self, item: Spanned<Expr>) {
self.items.push(item);
}
}
@@ -86,6 +108,8 @@ impl Display for Tuple {
}
}
+debug_display!(Tuple);
+
/// A key-value collection of identifiers and associated expressions.
#[derive(Clone, PartialEq)]
pub struct Object {
@@ -97,7 +121,7 @@ impl Object {
Object { pairs: vec![] }
}
- pub fn add(&mut self, key: Spanned<Ident>, value: Spanned<Expression>) {
+ pub fn add(&mut self, key: Spanned<Ident>, value: Spanned<Expr>) {
self.pairs.push(Pair { key, value });
}
@@ -127,11 +151,13 @@ impl Display for Object {
}
}
+debug_display!(Object);
+
/// A key-value pair in an object.
#[derive(Clone, PartialEq)]
pub struct Pair {
pub key: Spanned<Ident>,
- pub value: Spanned<Expression>,
+ pub value: Spanned<Expr>,
}
impl Display for Pair {
@@ -140,57 +166,56 @@ impl Display for Pair {
}
}
-debug_display!(Ident);
-debug_display!(Expression);
-debug_display!(Tuple);
-debug_display!(Object);
debug_display!(Pair);
-/// Kinds of expressions.
-pub trait ExpressionKind: Sized {
+pub trait ExprKind: Sized {
/// The name of the expression in an `expected <name>` error.
const NAME: &'static str;
/// Create from expression.
- fn from_expr(expr: Spanned<Expression>) -> ParseResult<Self>;
+ fn from_expr(expr: Spanned<Expr>) -> Result<Self, Error>;
}
+impl<T> ExprKind for Spanned<T> where T: ExprKind {
+ const NAME: &'static str = T::NAME;
+
+ fn from_expr(expr: Spanned<Expr>) -> Result<Self, Error> {
+ let span = expr.span;
+ T::from_expr(expr).map(|v| Spanned { v, span })
+ }
+}
/// Implements the expression kind trait for a type.
macro_rules! kind {
- ($type:ty, $name:expr, $($patterns:tt)*) => {
- impl ExpressionKind for $type {
+ ($type:ty, $name:expr, $($p:pat => $r:expr),* $(,)?) => {
+ impl ExprKind for $type {
const NAME: &'static str = $name;
- fn from_expr(expr: Spanned<Expression>) -> ParseResult<Self> {
+ fn from_expr(expr: Spanned<Expr>) -> Result<Self, Error> {
#[allow(unreachable_patterns)]
Ok(match expr.v {
- $($patterns)*,
- _ => error!("expected {}", Self::NAME),
+ $($p => $r),*,
+ _ => return Err(
+ err!("expected {}, found {}", Self::NAME, expr.v.name())
+ ),
})
}
}
};
}
-kind!(Expression, "expression", e => e);
-kind!(Ident, "identifier", Expression::Ident(ident) => ident);
-kind!(String, "string", Expression::Str(string) => string);
-kind!(f64, "number", Expression::Number(num) => num);
-kind!(bool, "boolean", Expression::Bool(boolean) => boolean);
-kind!(Size, "size", Expression::Size(size) => size);
-kind!(Tuple, "tuple", Expression::Tuple(tuple) => tuple);
-kind!(Object, "object", Expression::Object(object) => object);
-
-kind!(ScaleSize, "number or size",
- Expression::Size(size) => ScaleSize::Absolute(size),
- Expression::Number(scale) => ScaleSize::Scaled(scale as f32)
+kind!(Expr, "expression", e => e);
+kind!(Ident, "identifier", Expr::Ident(i) => i);
+kind!(String, "string", Expr::Str(s) => s);
+kind!(f64, "number", Expr::Number(n) => n);
+kind!(bool, "boolean", Expr::Bool(b) => b);
+kind!(Size, "size", Expr::Size(s) => s);
+kind!(Tuple, "tuple", Expr::Tuple(t) => t);
+kind!(Object, "object", Expr::Object(o) => o);
+kind!(ScaleSize, "number or size",
+ Expr::Size(size) => ScaleSize::Absolute(size),
+ Expr::Number(scale) => ScaleSize::Scaled(scale as f32),
+);
+kind!(StringLike, "identifier or string",
+ Expr::Ident(Ident(s)) => StringLike(s),
+ Expr::Str(s) => StringLike(s),
);
-
-impl<T> ExpressionKind for Spanned<T> where T: ExpressionKind {
- const NAME: &'static str = T::NAME;
-
- fn from_expr(expr: Spanned<Expression>) -> ParseResult<Spanned<T>> {
- let span = expr.span;
- T::from_expr(expr).map(|v| Spanned { v, span })
- }
-}
diff --git a/src/syntax/func.rs b/src/syntax/func.rs
index 5b1ce6e8..abc8c431 100644
--- a/src/syntax/func.rs
+++ b/src/syntax/func.rs
@@ -15,7 +15,7 @@ pub struct FuncArgs {
#[derive(Debug, Clone, PartialEq)]
pub enum Arg {
- Pos(Spanned<Expression>),
+ Pos(Spanned<Expr>),
Key(Pair),
}
@@ -46,12 +46,12 @@ impl FuncArgs {
}
/// Add a positional argument.
- pub fn add_pos(&mut self, item: Spanned<Expression>) {
+ pub fn add_pos(&mut self, item: Spanned<Expr>) {
self.pos.add(item);
}
/// Add a keyword argument.
- pub fn add_key(&mut self, key: Spanned<Ident>, value: Spanned<Expression>) {
+ pub fn add_key(&mut self, key: Spanned<Ident>, value: Spanned<Expr>) {
self.key.add(key, value);
}
@@ -92,7 +92,7 @@ impl FuncArgs {
// }
// /// Iterator over positional arguments.
- // pub fn iter_pos(&mut self) -> std::vec::IntoIter<Spanned<Expression>> {
+ // pub fn iter_pos(&mut self) -> std::vec::IntoIter<Spanned<Expr>> {
// let tuple = std::mem::replace(&mut self.positional, Tuple::new());
// tuple.items.into_iter()
// }
diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs
index 75407f82..a77c764e 100644
--- a/src/syntax/mod.rs
+++ b/src/syntax/mod.rs
@@ -23,22 +23,6 @@ pub mod prelude {
}
-pub struct Parsed<T> {
- pub output: T,
- pub errors: SpanVec<Error>,
- pub decorations: SpanVec<Decoration>,
-}
-
-impl<T> Parsed<T> {
- pub fn map<F, U>(self, f: F) -> Parsed<U> where F: FnOnce(T) -> U {
- Parsed {
- output: f(self.output),
- errors: self.errors,
- decorations: self.decorations,
- }
- }
-}
-
#[async_trait::async_trait(?Send)]
pub trait Model: Debug + ModelBounds {
async fn layout<'a>(
@@ -110,7 +94,7 @@ impl SyntaxModel {
#[async_trait::async_trait(?Send)]
impl Model for SyntaxModel {
- async fn layout<'a>(&'a self, ctx: LayoutContext<'_, '_>) -> Layouted<Commands<'a>> {
+ async fn layout<'a>(&'a self, _: LayoutContext<'_, '_>) -> Layouted<Commands<'a>> {
Layouted {
output: vec![Command::LayoutSyntaxModel(self)],
errors: vec![],
@@ -153,7 +137,8 @@ impl PartialEq for Node {
}
}
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
pub enum Decoration {
ValidFuncName,
InvalidFuncName,
diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs
index 24bef7ce..ed343050 100644
--- a/src/syntax/parsing.rs
+++ b/src/syntax/parsing.rs
@@ -3,10 +3,6 @@ use super::*;
use Token::*;
-pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed<SyntaxModel> {
- Parser::new(start, src, ctx).parse()
-}
-
/// The context for parsing.
#[derive(Debug, Copy, Clone)]
pub struct ParseContext<'a> {
@@ -14,185 +10,169 @@ pub struct ParseContext<'a> {
pub scope: &'a Scope,
}
-struct Parser<'s> {
- src: &'s str,
- ctx: ParseContext<'s>,
- tokens: Tokens<'s>,
- peeked: Option<Option<Spanned<Token<'s>>>>,
- position: Position,
- last_position: Position,
- errors: SpanVec<Error>,
- decorations: SpanVec<Decoration>,
+pub struct Parsed<T> {
+ pub output: T,
+ pub errors: SpanVec<Error>,
+ pub decorations: SpanVec<Decoration>,
}
-impl<'s> Parser<'s> {
- fn new(start: Position, src: &'s str, ctx: ParseContext<'s>) -> Parser<'s> {
- Parser {
- src,
- ctx,
- tokens: tokenize(start, src),
- peeked: None,
- position: Position::ZERO,
- last_position: Position::ZERO,
- errors: vec![],
- decorations: vec![],
- }
- }
-
- /// The main parsing entrypoint.
- fn parse(mut self) -> Parsed<SyntaxModel> {
- let mut model = SyntaxModel::new();
-
- while let Some(token) = self.eat() {
- let mut span = token.span;
- let node = match token.v {
- LineComment(_) | BlockComment(_) => None,
- Whitespace(newlines) => Some(if newlines >= 2 {
- Node::Newline
- } else {
- Node::Space
- }),
-
- LeftBracket => self.parse_func().map(|spanned| {
- span = spanned.span;
- spanned.v
- }),
-
- Star => Some(Node::ToggleBolder),
- Underscore => Some(Node::ToggleItalic),
- Backtick => Some(Node::ToggleMonospace),
- Text(text) => Some(Node::Text(text.to_owned())),
-
- _ => {
- self.unexpected(token);
- None
- }
- };
-
- if let Some(v) = node {
- model.add(Spanned { v, span });
- }
- }
-
+impl<T> Parsed<T> {
+ pub fn map<F, U>(self, f: F) -> Parsed<U> where F: FnOnce(T) -> U {
Parsed {
- output: model,
+ output: f(self.output),
errors: self.errors,
decorations: self.decorations,
}
}
+}
- /// Parses a function including header and body with the cursor starting
- /// right behind the first opening bracket.
- fn parse_func(&mut self) -> Option<Spanned<Node>> {
- let start = self.last_pos();
-
- let header = self.parse_func_header();
- self.eat_until(|t| t == RightBracket, false);
+pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed<SyntaxModel> {
+ let mut model = SyntaxModel::new();
+ let mut errors = Vec::new();
+ let mut decorations = Vec::new();
- if self.eat().map(Spanned::value) != Some(RightBracket) {
- self.expected_at("closing bracket", self.pos());
- }
+ let mut tokens = Tokens::new(start, src, TokenizationMode::Body);
- let body = if self.peekv() == Some(LeftBracket) {
- self.eat();
+ while let Some(token) = tokens.next() {
+ let span = token.span;
- let start_index = self.tokens.index();
- let start_position = self.tokens.pos();
-
- let found = self.tokens.move_to_closing_bracket();
+ let node = match token.v {
+ Space(newlines) => if newlines >= 2 {
+ Node::Newline
+ } else {
+ Node::Space
+ },
- let end_index = self.tokens.index();
- let end_position = self.tokens.pos();
+ Function { header, body, terminated } => {
+ let parsed: Parsed<Node> = FuncParser::new(header, body, ctx).parse();
- let body = &self.src[start_index .. end_index];
+ errors.extend(offset_spans(parsed.errors, span.start));
+ decorations.extend(offset_spans(parsed.decorations, span.start));
- self.position = end_position;
+ if !terminated {
+ errors.push(err!(Span::at(span.end); "expected closing bracket"));
+ }
- if found {
- let next = self.eat().map(Spanned::value);
- debug_assert_eq!(next, Some(RightBracket));
- } else {
- self.expected_at("closing bracket", self.pos());
+ parsed.output
}
- Some(Spanned::new(body, Span::new(start_position, end_position)))
- } else {
- None
- };
+ Star => Node::ToggleBolder,
+ Underscore => Node::ToggleItalic,
+ Backtick => Node::ToggleMonospace,
+ Text(text) => Node::Text(text.to_owned()),
+
+ LineComment(_) | BlockComment(_) => continue,
- let header = header?;
- let (parser, decoration) = match self.ctx.scope.get_parser(header.name.v.as_str()) {
- Ok(parser) => (parser, Decoration::ValidFuncName),
- Err(parser) => {
- let error = Error::new(format!("unknown function: `{}`", header.name.v));
- self.errors.push(Spanned::new(error, header.name.span));
- (parser, Decoration::InvalidFuncName)
+ other => {
+ errors.push(err!(span; "unexpected {}", name(other)));
+ continue;
}
};
- self.decorations.push(Spanned::new(decoration, header.name.span));
+ model.add(Spanned { v: node, span: token.span });
+ }
- let parsed = parser(header, body, self.ctx);
- self.errors.extend(offset_spans(parsed.errors, start));
- self.decorations.extend(offset_spans(parsed.decorations, start));
+ Parsed { output: model, errors, decorations }
+}
- let node = Node::Model(parsed.output);
+struct FuncParser<'s> {
+ ctx: ParseContext<'s>,
+ errors: SpanVec<Error>,
+ decorations: SpanVec<Decoration>,
+ tokens: Tokens<'s>,
+ peeked: Option<Option<Spanned<Token<'s>>>>,
+ body: Option<(Position, &'s str)>,
+}
- let end = self.pos();
- let span = Span { start, end };
+impl<'s> FuncParser<'s> {
+ fn new(
+ header: &'s str,
+ body: Option<(Position, &'s str)>,
+ ctx: ParseContext<'s>
+ ) -> FuncParser<'s> {
+ FuncParser {
+ ctx,
+ errors: vec![],
+ decorations: vec![],
+ tokens: Tokens::new(Position::new(0, 1), header, TokenizationMode::Header),
+ peeked: None,
+ body,
+ }
+ }
+
+ fn parse(mut self) -> Parsed<Node> {
+ let parsed = if let Some(header) = self.parse_func_header() {
+ let name = header.name.v.as_str();
+ let (parser, deco) = match self.ctx.scope.get_parser(name) {
+ Ok(parser) => (parser, Decoration::ValidFuncName),
+ Err(parser) => {
+ self.errors.push(err!(header.name.span; "unknown function"));
+ (parser, Decoration::InvalidFuncName)
+ }
+ };
- Some(Spanned { v: node, span })
+ self.decorations.push(Spanned::new(deco, header.name.span));
+
+ parser(header, self.body, self.ctx)
+ } else {
+ let default = FuncHeader {
+ name: Spanned::new(Ident("".to_string()), Span::ZERO),
+ args: FuncArgs::new(),
+ };
+
+ // Use the fallback function such that the body is still rendered
+ // even if the header is completely unparsable.
+ self.ctx.scope.get_fallback_parser()(default, self.body, self.ctx)
+ };
+
+ self.errors.extend(parsed.errors);
+ self.decorations.extend(parsed.decorations);
+
+ Parsed {
+ output: Node::Model(parsed.output),
+ errors: self.errors,
+ decorations: self.decorations,
+ }
}
- /// Parses a function header including the closing bracket.
fn parse_func_header(&mut self) -> Option<FuncHeader> {
+ let start = self.pos();
self.skip_whitespace();
- let name = self.parse_func_name()?;
- self.skip_whitespace();
- let args = match self.peek() {
- Some(Spanned { v: Colon, .. }) => {
- self.eat();
- self.parse_func_args()
+ let name = match self.eat() {
+ Some(Spanned { v: ExprIdent(ident), span }) => {
+ Spanned { v: Ident(ident.to_string()), span }
}
- Some(Spanned { v: RightBracket, .. }) => FuncArgs::new(),
other => {
- self.expected_at("colon or closing bracket", name.span.end);
- FuncArgs::new()
+ self.expected_found_or_at("identifier", other, start);
+ return None;
}
};
- Some(FuncHeader { name, args })
- }
-
- /// Parses the function name if is the next token. Otherwise, it adds an
- /// error and returns `None`.
- fn parse_func_name(&mut self) -> Option<Spanned<Ident>> {
- match self.peek() {
- Some(Spanned { v: ExprIdent(ident), span }) => {
- self.eat();
- return Some(Spanned { v: Ident(ident.to_string()), span });
+ self.skip_whitespace();
+ let args = match self.eat().map(Spanned::value) {
+ Some(Colon) => self.parse_func_args(),
+ Some(_) => {
+ self.expected_at("colon", name.span.end);
+ FuncArgs::new()
}
- other => self.expected_found_or_at("identifier", other, self.pos()),
- }
+ None => FuncArgs::new(),
+ };
- None
+ Some(FuncHeader { name, args })
}
- /// Parses the function arguments and stops right before the final closing
- /// bracket.
fn parse_func_args(&mut self) -> FuncArgs {
let mut args = FuncArgs::new();
- loop {
- self.skip_whitespace();
- match self.peekv() {
- Some(RightBracket) | None => break,
- _ => match self.parse_arg() {
- Some(arg) => args.add(arg),
- None => {}
- }
+ self.skip_whitespace();
+ while self.peek().is_some() {
+ match self.parse_arg() {
+ Some(arg) => args.add(arg),
+ None => {}
}
+
+ self.skip_whitespace();
}
args
@@ -221,7 +201,7 @@ impl<'s> Parser<'s> {
})
})
} else {
- Some(Arg::Pos(Spanned::new(Expression::Ident(ident), span)))
+ Some(Arg::Pos(Spanned::new(Expr::Ident(ident), span)))
}
} else {
self.parse_expr().map(|expr| Arg::Pos(expr))
@@ -230,7 +210,6 @@ impl<'s> Parser<'s> {
if let Some(arg) = &arg {
self.skip_whitespace();
match self.peekv() {
- Some(RightBracket) => {}
Some(Comma) => { self.eat(); }
Some(_) => self.expected_at("comma", arg.span().end),
_ => {}
@@ -244,19 +223,27 @@ impl<'s> Parser<'s> {
}
/// Parse a atomic or compound (tuple / object) expression.
- fn parse_expr(&mut self) -> Option<Spanned<Expression>> {
+ fn parse_expr(&mut self) -> Option<Spanned<Expr>> {
let first = self.peek()?;
- let mut expr = |v| {
- self.eat();
- Spanned { v, span: first.span }
- };
+ let spanned = |v| Spanned { v, span: first.span };
Some(match first.v {
- ExprIdent(i) => expr(Expression::Ident(Ident(i.to_string()))),
- ExprStr(s) => expr(Expression::Str(s.to_string())),
- ExprNumber(n) => expr(Expression::Number(n)),
- ExprSize(s) => expr(Expression::Size(s)),
- ExprBool(b) => expr(Expression::Bool(b)),
+ ExprIdent(i) => {
+ self.eat();
+ spanned(Expr::Ident(Ident(i.to_string())))
+ }
+ ExprStr { string, terminated } => {
+ if !terminated {
+ self.expected_at("quote", first.span.end);
+ }
+
+ self.eat();
+ spanned(Expr::Str(string.to_string()))
+ }
+ ExprNumber(n) => { self.eat(); spanned(Expr::Number(n)) }
+ ExprSize(s) => { self.eat(); spanned(Expr::Size(s)) }
+ ExprBool(b) => { self.eat(); spanned(Expr::Bool(b)) }
+
LeftParen => self.parse_tuple(),
LeftBrace => self.parse_object(),
_ => return None,
@@ -264,56 +251,48 @@ impl<'s> Parser<'s> {
}
/// Parse a tuple expression.
- fn parse_tuple(&mut self) -> Spanned<Expression> {
+ fn parse_tuple(&mut self) -> Spanned<Expr> {
let start = self.pos();
// TODO: Do the thing.
- self.eat_until(|t| matches!(t, RightParen | RightBracket), false);
- if self.peekv() == Some(RightParen) {
- self.eat();
- }
+ self.eat_until(|t| t == RightParen, true);
let end = self.pos();
let span = Span { start, end };
- Spanned { v: Expression::Tuple(Tuple::new()), span }
+ Spanned { v: Expr::Tuple(Tuple::new()), span }
}
/// Parse an object expression.
- fn parse_object(&mut self) -> Spanned<Expression> {
+ fn parse_object(&mut self) -> Spanned<Expr> {
let start = self.pos();
// TODO: Do the thing.
- self.eat_until(|t| matches!(t, RightBrace | RightBracket), false);
- if self.peekv() == Some(RightBrace) {
- self.eat();
- }
+ self.eat_until(|t| t == RightBrace, true);
let end = self.pos();
let span = Span { start, end };
- Spanned { v: Expression::Object(Object::new()), span }
+ Spanned { v: Expr::Object(Object::new()), span }
}
/// Skip all whitespace/comment tokens.
fn skip_whitespace(&mut self) {
self.eat_until(|t|
- !matches!(t, Whitespace(_) | LineComment(_) | BlockComment(_)), false)
- }
-
- /// Add an error about an `thing` which was expected but not found at the
- /// given position.
- fn expected_at(&mut self, thing: &str, pos: Position) {
- let error = Error::new(format!("expected {}", thing));
- self.errors.push(Spanned::new(error, Span::at(pos)));
+ !matches!(t, Space(_) | LineComment(_) | BlockComment(_)), false)
}
/// Add an error about an expected `thing` which was not found, showing
/// what was found instead.
fn expected_found(&mut self, thing: &str, found: Spanned<Token>) {
- let message = format!("expected {}, found {}", thing, name(found.v));
- let error = Error::new(message);
- self.errors.push(Spanned::new(error, found.span));
+ self.errors.push(err!(found.span;
+ "expected {}, found {}", thing, name(found.v)));
+ }
+
+ /// Add an error about an `thing` which was expected but not found at the
+ /// given position.
+ fn expected_at(&mut self, thing: &str, pos: Position) {
+ self.errors.push(err!(Span::at(pos); "expected {}", thing));
}
/// Add a found-error if `found` is some and a positional error, otherwise.
@@ -329,12 +308,6 @@ impl<'s> Parser<'s> {
}
}
- /// Add an error about an unexpected token `found`.
- fn unexpected(&mut self, found: Spanned<Token>) {
- let error = Error::new(format!("unexpected {}", name(found.v)));
- self.errors.push(Spanned::new(error, found.span));
- }
-
/// Consume tokens until the function returns true and only consume the last
/// token if instructed to.
fn eat_until<F>(&mut self, mut f: F, eat_match: bool)
@@ -351,19 +324,10 @@ impl<'s> Parser<'s> {
}
}
- /// Consume and return the next token, update positions and colorize the
- /// token. All colorable tokens are per default colorized here, to override
- /// a colorization use `Colorization::replace_last`.
+ /// Consume and return the next token.
fn eat(&mut self) -> Option<Spanned<Token<'s>>> {
- let token = self.peeked.take()
- .unwrap_or_else(|| self.tokens.next());
-
- if let Some(token) = token {
- self.last_position = self.position;
- self.position = token.span.end;
- }
-
- token
+ self.peeked.take()
+ .unwrap_or_else(|| self.tokens.next())
}
/// Peek at the next token without consuming it.
@@ -379,23 +343,18 @@ impl<'s> Parser<'s> {
/// The position at the end of the last eat token / start of the peekable
/// token.
fn pos(&self) -> Position {
- self.position
- }
-
- /// The position at the start of the last eaten token.
- fn last_pos(&self) -> Position {
- self.last_position
+ self.peeked.flatten()
+ .map(|s| s.span.start)
+ .unwrap_or_else(|| self.tokens.pos())
}
}
-/// The name of a token in an `expected <...>` error.
+/// The name of a token in an `(un)expected <...>` error.
fn name(token: Token) -> &'static str {
match token {
- Whitespace(_) => "whitespace",
+ Space(_) => "space",
LineComment(_) | BlockComment(_) => "comment",
- StarSlash => "end of block comment",
- LeftBracket => "opening bracket",
- RightBracket => "closing bracket",
+ Function { .. } => "function",
LeftParen => "opening paren",
RightParen => "closing paren",
LeftBrace => "opening brace",
@@ -404,13 +363,16 @@ fn name(token: Token) -> &'static str {
Comma => "comma",
Equals => "equals sign",
ExprIdent(_) => "identifier",
- ExprStr(_) => "string",
+ ExprStr { .. } => "string",
ExprNumber(_) => "number",
ExprSize(_) => "size",
- ExprBool(_) => "bool",
+ ExprBool(_) => "boolean",
Star => "star",
Underscore => "underscore",
Backtick => "backtick",
Text(_) => "invalid identifier",
+ Invalid("]") => "closing bracket",
+ Invalid("*/") => "end of block comment",
+ Invalid(_) => "invalid token",
}
}
diff --git a/src/syntax/span.rs b/src/syntax/span.rs
index eb39677e..e049861f 100644
--- a/src/syntax/span.rs
+++ b/src/syntax/span.rs
@@ -1,7 +1,7 @@
//! Spans map elements to the part of source code they originate from.
use std::fmt::{self, Debug, Display, Formatter};
-use std::ops::{Add, AddAssign};
+use std::ops::{Add, Sub};
use serde::Serialize;
@@ -21,12 +21,13 @@ impl<T> Spanned<T> {
self.v
}
- pub fn map<F, V>(self, f: F) -> Spanned<V> where F: FnOnce(T) -> V {
+ pub fn map<V, F>(self, f: F) -> Spanned<V> where F: FnOnce(T) -> V {
Spanned { v: f(self.v), span: self.span }
}
- pub fn map_v<V>(&self, new_v: V) -> Spanned<V> {
- Spanned { v: new_v, span: self.span }
+ pub fn map_span<F>(mut self, f: F) -> Spanned<T> where F: FnOnce(Span) -> Span {
+ self.span = f(self.span);
+ self
}
}
@@ -74,6 +75,13 @@ impl Span {
pub fn expand(&mut self, other: Span) {
*self = Span::merge(*self, other)
}
+
+ pub fn offset(self, start: Position) -> Span {
+ Span {
+ start: start + self.start,
+ end: start + self.end,
+ }
+ }
}
impl Display for Span {
@@ -119,9 +127,21 @@ impl Add for Position {
}
}
-impl AddAssign for Position {
- fn add_assign(&mut self, other: Position) {
- *self = *self + other;
+impl Sub for Position {
+ type Output = Position;
+
+ fn sub(self, rhs: Position) -> Position {
+ if self.line == rhs.line {
+ Position {
+ line: 0,
+ column: self.column - rhs.column
+ }
+ } else {
+ Position {
+ line: self.line - rhs.line,
+ column: self.column,
+ }
+ }
}
}
@@ -136,14 +156,6 @@ debug_display!(Position);
/// A vector of spanned things.
pub type SpanVec<T> = Vec<Spanned<T>>;
-pub fn offset_spans<T>(
- vec: SpanVec<T>,
- start: Position,
-) -> impl Iterator<Item=Spanned<T>> {
- vec.into_iter()
- .map(move |mut spanned| {
- spanned.span.start += start;
- spanned.span.end += start;
- spanned
- })
+pub fn offset_spans<T>(vec: SpanVec<T>, start: Position) -> impl Iterator<Item=Spanned<T>> {
+ vec.into_iter().map(move |s| s.map_span(|span| span.offset(start)))
}
diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs
index 0588bc6c..6c8e736c 100644
--- a/src/syntax/tokens.rs
+++ b/src/syntax/tokens.rs
@@ -4,7 +4,7 @@ use unicode_xid::UnicodeXID;
use super::*;
use Token::*;
-use State::*;
+use TokenizationMode::*;
/// A minimal semantic entity of source code.
@@ -12,20 +12,20 @@ use State::*;
pub enum Token<'s> {
/// One or more whitespace characters. The contained `usize` denotes the
/// number of newlines that were contained in the whitespace.
- Whitespace(usize),
+ Space(usize),
/// A line comment with inner string contents `//<&'s str>\n`.
LineComment(&'s str),
/// A block comment with inner string contents `/*<&'s str>*/`. The comment
/// can contain nested block comments.
BlockComment(&'s str),
- /// An erroneous `*/` without an opening block comment.
- StarSlash,
- /// A left bracket: `[`.
- LeftBracket,
- /// A right bracket: `]`.
- RightBracket,
+ /// A function invocation `[<header>][<body>]`.
+ Function {
+ header: &'s str,
+ body: Option<(Position, &'s str)>,
+ terminated: bool,
+ },
/// A left parenthesis in a function header: `(`.
LeftParen,
@@ -46,7 +46,7 @@ pub enum Token<'s> {
/// An identifier in a function header: `center`.
ExprIdent(&'s str),
/// A quoted string in a function header: `"..."`.
- ExprStr(&'s str),
+ ExprStr { string: &'s str, terminated: bool },
/// A number in a function header: `3.14`.
ExprNumber(f64),
/// A size in a function header: `12pt`.
@@ -63,36 +63,31 @@ pub enum Token<'s> {
/// Any other consecutive string.
Text(&'s str),
-}
-/// Decomposes text into a sequence of semantic tokens.
-pub fn tokenize(start: Position, src: &str) -> Tokens {
- Tokens::new(start, src)
+ /// Things that are not valid in the context they appeared in.
+ Invalid(&'s str),
}
/// An iterator over the tokens of a string of source code.
pub struct Tokens<'s> {
src: &'s str,
- state: State,
- stack: Vec<(State, Position)>,
+ mode: TokenizationMode,
iter: Peekable<Chars<'s>>,
position: Position,
index: usize,
}
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
-enum State {
+pub enum TokenizationMode {
Header,
- StartBody,
Body,
}
impl<'s> Tokens<'s> {
- pub fn new(start: Position, src: &'s str) -> Tokens<'s> {
+ pub fn new(start: Position, src: &'s str, mode: TokenizationMode) -> Tokens<'s> {
Tokens {
src,
- state: State::Body,
- stack: vec![],
+ mode,
iter: src.chars().peekable(),
position: start,
index: 0,
@@ -110,35 +105,6 @@ impl<'s> Tokens<'s> {
pub fn pos(&self) -> Position {
self.position
}
-
- /// Move through the string until an unbalanced closing bracket is found
- /// without tokenizing the contents.
- ///
- /// Returns whether a closing bracket was found or the end of the string was
- /// reached.
- pub fn move_to_closing_bracket(&mut self) -> bool {
- let mut escaped = false;
- let mut depth = 0;
-
- self.read_string_until(|n| {
- match n {
- '[' if !escaped => depth += 1,
- ']' if !escaped => {
- if depth == 0 {
- return true;
- } else {
- depth -= 1;
- }
- }
- '\\' => escaped = !escaped,
- _ => escaped = false,
- }
-
- false
- }, false, 0, 0);
-
- self.peek() == Some(']')
- }
}
impl<'s> Iterator for Tokens<'s> {
@@ -153,55 +119,31 @@ impl<'s> Iterator for Tokens<'s> {
// Comments.
'/' if self.peek() == Some('/') => self.parse_line_comment(),
'/' if self.peek() == Some('*') => self.parse_block_comment(),
- '*' if self.peek() == Some('/') => { self.eat(); StarSlash }
+ '*' if self.peek() == Some('/') => { self.eat(); Invalid("*/") }
// Whitespace.
c if c.is_whitespace() => self.parse_whitespace(start),
// Functions.
- '[' => {
- match self.state {
- Header | Body => {
- self.stack.push((self.state, start));
- self.position = Position::new(0, '['.len_utf8());
- self.state = Header;
- }
- StartBody => self.state = Body,
- }
-
- LeftBracket
- }
- ']' => {
- if self.state == Header && self.peek() == Some('[') {
- self.state = StartBody;
- } else {
- if let Some((state, pos)) = self.stack.pop() {
- self.state = state;
- self.position = pos + self.position;
- } else {
- self.state = Body;
- }
- }
-
- RightBracket
- }
+ '[' => self.parse_function(start),
+ ']' => Invalid("]"),
// Syntactic elements in function headers.
- '(' if self.state == Header => LeftParen,
- ')' if self.state == Header => RightParen,
- '{' if self.state == Header => LeftBrace,
- '}' if self.state == Header => RightBrace,
- ':' if self.state == Header => Colon,
- ',' if self.state == Header => Comma,
- '=' if self.state == Header => Equals,
+ '(' if self.mode == Header => LeftParen,
+ ')' if self.mode == Header => RightParen,
+ '{' if self.mode == Header => LeftBrace,
+ '}' if self.mode == Header => RightBrace,
+ ':' if self.mode == Header => Colon,
+ ',' if self.mode == Header => Comma,
+ '=' if self.mode == Header => Equals,
// String values.
- '"' if self.state == Header => self.parse_string(),
+ '"' if self.mode == Header => self.parse_string(),
// Style toggles.
- '*' if self.state == Body => Star,
- '_' if self.state == Body => Underscore,
- '`' if self.state == Body => Backtick,
+ '*' if self.mode == Body => Star,
+ '_' if self.mode == Body => Underscore,
+ '`' if self.mode == Body => Backtick,
// An escaped thing.
'\\' => self.parse_escaped(),
@@ -215,9 +157,9 @@ impl<'s> Iterator for Tokens<'s> {
',' | '"' | '/' => true,
_ => false,
}
- }, false, -(c.len_utf8() as isize), 0);
+ }, false, -(c.len_utf8() as isize), 0).0;
- if self.state == Header {
+ if self.mode == Header {
self.parse_expr(text)
} else {
Text(text)
@@ -234,7 +176,7 @@ impl<'s> Iterator for Tokens<'s> {
impl<'s> Tokens<'s> {
fn parse_line_comment(&mut self) -> Token<'s> {
- LineComment(self.read_string_until(is_newline_char, false, 1, 0))
+ LineComment(self.read_string_until(is_newline_char, false, 1, 0).0)
}
fn parse_block_comment(&mut self) -> Token<'s> {
@@ -262,19 +204,60 @@ impl<'s> Tokens<'s> {
}
false
- }, true, 0, -2))
+ }, true, 0, -2).0)
}
fn parse_whitespace(&mut self, start: Position) -> Token<'s> {
self.read_string_until(|n| !n.is_whitespace(), false, 0, 0);
let end = self.pos();
- Whitespace(end.line - start.line)
+ Space(end.line - start.line)
+ }
+
+ fn parse_function(&mut self, start: Position) -> Token<'s> {
+ let (header, terminated) = self.read_function_part();
+ self.eat();
+
+ if self.peek() != Some('[') {
+ return Function { header, body: None, terminated };
+ }
+
+ self.eat();
+
+ let offset = self.pos() - start;
+ let (body, terminated) = self.read_function_part();
+ self.eat();
+
+ Function { header, body: Some((offset, body)), terminated }
+ }
+
+ fn read_function_part(&mut self) -> (&'s str, bool) {
+ let mut escaped = false;
+ let mut in_string = false;
+ let mut depth = 0;
+
+ self.read_string_until(|n| {
+ match n {
+ '"' if !escaped => in_string = !in_string,
+ '[' if !escaped && !in_string => depth += 1,
+ ']' if !escaped && !in_string => {
+ if depth == 0 {
+ return true;
+ } else {
+ depth -= 1;
+ }
+ }
+ '\\' => escaped = !escaped,
+ _ => escaped = false,
+ }
+
+ false
+ }, false, 0, 0)
}
fn parse_string(&mut self) -> Token<'s> {
let mut escaped = false;
- ExprStr(self.read_string_until(|n| {
+ let (string, terminated) = self.read_string_until(|n| {
match n {
'"' if !escaped => return true,
'\\' => escaped = !escaped,
@@ -282,7 +265,8 @@ impl<'s> Tokens<'s> {
}
false
- }, true, 0, -1))
+ }, true, 0, -1);
+ ExprStr { string, terminated }
}
fn parse_escaped(&mut self) -> Token<'s> {
@@ -294,7 +278,7 @@ impl<'s> Tokens<'s> {
}
let c = self.peek().unwrap_or('n');
- if self.state == Body && is_escapable(c) {
+ if self.mode == Body && is_escapable(c) {
let index = self.index();
self.eat();
Text(&self.src[index .. index + c.len_utf8()])
@@ -315,7 +299,7 @@ impl<'s> Tokens<'s> {
} else if is_identifier(text) {
ExprIdent(text)
} else {
- Text(text)
+ Invalid(text)
}
}
@@ -325,7 +309,7 @@ impl<'s> Tokens<'s> {
eat_match: bool,
offset_start: isize,
offset_end: isize,
- ) -> &'s str where F: FnMut(char) -> bool {
+ ) -> (&'s str, bool) where F: FnMut(char) -> bool {
let start = ((self.index() as isize) + offset_start) as usize;
let mut matched = false;
@@ -346,7 +330,7 @@ impl<'s> Tokens<'s> {
end = ((end as isize) + offset_end) as usize;
}
- &self.src[start .. end]
+ (&self.src[start .. end], matched)
}
fn eat(&mut self) -> Option<char> {
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-15 17:32:56 +0000