summaryrefslogtreecommitdiff
path: root/src/parse
diff options
context:
space:
mode:
authorLaurenz <laurmaedje@gmail.com>2021-01-30 12:09:26 +0100
committerLaurenz <laurmaedje@gmail.com>2021-01-30 12:09:26 +0100
commit89eb8bae49edb71d9a9279a2210bb1ccaf4dd707 (patch)
tree160b1a2ff41b5bba8a58f882df9d10c9eb036cf2 /src/parse
parentac24075469f171fe83a976b9a97b9b1ea078a7e3 (diff)
New syntax 💎
- Everything everywhere! - Blocks with curly braces: {} - Templates with brackets: [] - Function templates with hashtag: `#[f]` - Headings with equals sign: `= Introduction`
Diffstat (limited to 'src/parse')
-rw-r--r--src/parse/mod.rs114
-rw-r--r--src/parse/parser.rs12
-rw-r--r--src/parse/tokens.rs89
3 files changed, 115 insertions, 100 deletions
diff --git a/src/parse/mod.rs b/src/parse/mod.rs
index ff75a563..3fc7d483 100644
--- a/src/parse/mod.rs
+++ b/src/parse/mod.rs
@@ -45,38 +45,33 @@ fn tree(p: &mut Parser) -> Tree {
fn node(p: &mut Parser, at_start: &mut bool) -> Option<Node> {
let token = p.peek()?;
let node = match token {
- // Bracket call.
- Token::LeftBracket => {
- return Some(Node::Expr(bracket_call(p)?));
+ // Whitespace.
+ Token::Space(newlines) => {
+ *at_start |= newlines > 0;
+ if newlines < 2 { Node::Space } else { Node::Parbreak }
}
- // Code block.
- Token::LeftBrace => {
- return Some(Node::Expr(block(p, false)?));
- }
+ // Text.
+ Token::Text(text) => Node::Text(text.into()),
// Markup.
Token::Star => Node::Strong,
Token::Underscore => Node::Emph,
- Token::Tilde => Node::Text("\u{00A0}".into()),
- Token::Hash => {
+ Token::Eq => {
if *at_start {
return Some(Node::Heading(heading(p)));
} else {
Node::Text(p.get(p.peek_span()).into())
}
}
+ Token::Tilde => Node::Text("\u{00A0}".into()),
Token::Backslash => Node::Linebreak,
- Token::Space(newlines) => {
- *at_start |= newlines > 0;
- if newlines < 2 { Node::Space } else { Node::Parbreak }
- }
- Token::Text(text) => Node::Text(text.into()),
Token::Raw(t) => Node::Raw(raw(p, t)),
Token::UnicodeEscape(t) => Node::Text(unicode_escape(p, t)),
// Keywords.
Token::Let | Token::If | Token::For => {
+ *at_start = false;
let stmt = token == Token::Let;
let group = if stmt { Group::Stmt } else { Group::Expr };
@@ -92,6 +87,24 @@ fn node(p: &mut Parser, at_start: &mut bool) -> Option<Node> {
return expr.map(Node::Expr);
}
+ // Block.
+ Token::LeftBrace => {
+ *at_start = false;
+ return Some(Node::Expr(block(p, false)?));
+ }
+
+ // Template.
+ Token::LeftBracket => {
+ *at_start = false;
+ return Some(Node::Expr(template(p)));
+ }
+
+ // Function template.
+ Token::HashBracket => {
+ *at_start = false;
+ return Some(Node::Expr(bracket_call(p)?));
+ }
+
// Comments.
Token::LineComment(_) | Token::BlockComment(_) => {
p.eat();
@@ -99,6 +112,7 @@ fn node(p: &mut Parser, at_start: &mut bool) -> Option<Node> {
}
_ => {
+ *at_start = false;
p.unexpected();
return None;
}
@@ -109,12 +123,12 @@ fn node(p: &mut Parser, at_start: &mut bool) -> Option<Node> {
/// Parse a heading.
fn heading(p: &mut Parser) -> NodeHeading {
- // Count hashtags.
+ // Count depth.
let mut level = p.span(|p| {
- p.assert(Token::Hash);
+ p.assert(&[Token::Eq]);
let mut level = 0u8;
- while p.eat_if(Token::Hash) {
+ while p.eat_if(Token::Eq) {
level = level.saturating_add(1);
}
level
@@ -278,22 +292,18 @@ fn expr_with(p: &mut Parser, min_prec: usize) -> Option<Expr> {
/// Parse a primary expression.
fn primary(p: &mut Parser) -> Option<Expr> {
let expr = match p.peek() {
- // Template.
- Some(Token::LeftBracket) => {
- return Some(template(p));
- }
-
- // Nested block.
- Some(Token::LeftBrace) => {
- return block(p, true);
- }
-
- // Dictionary or just a parenthesized expression.
- Some(Token::LeftParen) => {
- return Some(parenthesized(p));
- }
+ // Basic values.
+ Some(Token::None) => Expr::None,
+ Some(Token::Bool(b)) => Expr::Bool(b),
+ Some(Token::Int(i)) => Expr::Int(i),
+ Some(Token::Float(f)) => Expr::Float(f),
+ Some(Token::Length(val, unit)) => Expr::Length(val, unit),
+ Some(Token::Angle(val, unit)) => Expr::Angle(val, unit),
+ Some(Token::Percent(p)) => Expr::Percent(p),
+ Some(Token::Color(color)) => Expr::Color(color),
+ Some(Token::Str(token)) => Expr::Str(string(p, token)),
- // Function or just ident.
+ // Function or identifier.
Some(Token::Ident(id)) => {
p.eat();
let ident = Ident(id.into());
@@ -305,23 +315,33 @@ fn primary(p: &mut Parser) -> Option<Expr> {
}
}
- // Basic values.
- Some(Token::None) => Expr::None,
- Some(Token::Bool(b)) => Expr::Bool(b),
- Some(Token::Int(i)) => Expr::Int(i),
- Some(Token::Float(f)) => Expr::Float(f),
- Some(Token::Length(val, unit)) => Expr::Length(val, unit),
- Some(Token::Angle(val, unit)) => Expr::Angle(val, unit),
- Some(Token::Percent(p)) => Expr::Percent(p),
- Some(Token::Color(color)) => Expr::Color(color),
- Some(Token::Str(token)) => Expr::Str(string(p, token)),
-
// Keywords.
Some(Token::Let) => return expr_let(p),
Some(Token::If) => return expr_if(p),
Some(Token::For) => return expr_for(p),
- // No value.
+ // Block.
+ Some(Token::LeftBrace) => {
+ return block(p, true);
+ }
+
+ // Template.
+ Some(Token::LeftBracket) => {
+ return Some(template(p));
+ }
+
+ // Function template.
+ Some(Token::HashBracket) => {
+ let call = p.span_if(bracket_call)?.map(Node::Expr);
+ return Some(Expr::Template(vec![call]));
+ }
+
+ // Array, dictionary or parenthesized expression.
+ Some(Token::LeftParen) => {
+ return Some(parenthesized(p));
+ }
+
+ // Nothing.
_ => {
p.expected("expression");
return None;
@@ -380,7 +400,7 @@ fn string(p: &mut Parser, token: TokenStr) -> String {
/// Parse a let expression.
fn expr_let(p: &mut Parser) -> Option<Expr> {
- p.assert(Token::Let);
+ p.assert(&[Token::Let]);
let mut expr_let = None;
if let Some(pat) = p.span_if(ident) {
@@ -397,7 +417,7 @@ fn expr_let(p: &mut Parser) -> Option<Expr> {
/// Parse an if expresion.
fn expr_if(p: &mut Parser) -> Option<Expr> {
- p.assert(Token::If);
+ p.assert(&[Token::If]);
let mut expr_if = None;
if let Some(condition) = p.span_if(expr) {
@@ -420,7 +440,7 @@ fn expr_if(p: &mut Parser) -> Option<Expr> {
/// Parse a for expression.
fn expr_for(p: &mut Parser) -> Option<Expr> {
- p.assert(Token::For);
+ p.assert(&[Token::For]);
let mut expr_for = None;
if let Some(pat) = p.span_if(for_pattern) {
diff --git a/src/parse/parser.rs b/src/parse/parser.rs
index 906d9e62..2ca8eb10 100644
--- a/src/parse/parser.rs
+++ b/src/parse/parser.rs
@@ -105,9 +105,9 @@ impl<'s> Parser<'s> {
self.repeek();
match group {
- Group::Paren => self.assert(Token::LeftParen),
- Group::Bracket => self.assert(Token::LeftBracket),
- Group::Brace => self.assert(Token::LeftBrace),
+ Group::Paren => self.assert(&[Token::LeftParen]),
+ Group::Bracket => self.assert(&[Token::HashBracket, Token::LeftBracket]),
+ Group::Brace => self.assert(&[Token::LeftBrace]),
Group::Subheader => {}
Group::Stmt => {}
Group::Expr => {}
@@ -210,10 +210,10 @@ impl<'s> Parser<'s> {
eaten
}
- /// Consume the next token, debug-asserting that it is the given one.
- pub fn assert(&mut self, t: Token) {
+ /// Consume the next token, debug-asserting that it is one of the given ones.
+ pub fn assert(&mut self, ts: &[Token]) {
let next = self.eat();
- debug_assert_eq!(next, Some(t));
+ debug_assert!(next.map_or(false, |n| ts.contains(&n)));
}
/// Skip whitespace and comment tokens.
diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs
index 056bbbbb..4f5d8ab4 100644
--- a/src/parse/tokens.rs
+++ b/src/parse/tokens.rs
@@ -67,12 +67,15 @@ impl<'s> Iterator for Tokens<'s> {
loop {
// Common elements.
return Some(match c {
- // Functions, blocks and terminators.
+ // Blocks and templates.
'[' => Token::LeftBracket,
']' => Token::RightBracket,
'{' => Token::LeftBrace,
'}' => Token::RightBrace,
+ // Keywords, function templates, colors.
+ '#' => self.hash(start),
+
// Whitespace.
c if c.is_whitespace() => self.whitespace(c),
@@ -90,8 +93,8 @@ impl<'s> Iterator for Tokens<'s> {
// Markup.
'*' => Token::Star,
'_' => Token::Underscore,
+ '=' => Token::Eq,
'~' => Token::Tilde,
- '#' => self.hash(start),
'`' => self.raw(),
'$' => self.math(),
'\\' => self.backslash(),
@@ -140,8 +143,7 @@ impl<'s> Iterator for Tokens<'s> {
self.number(start, c)
}
- // Hex values and strings.
- '#' => self.hex(start),
+ // Strings.
'"' => self.string(),
_ => Token::Invalid(self.s.eaten_from(start)),
@@ -151,6 +153,27 @@ impl<'s> Iterator for Tokens<'s> {
}
impl<'s> Tokens<'s> {
+ fn hash(&mut self, start: usize) -> Token<'s> {
+ if self.s.eat_if('[') {
+ return Token::HashBracket;
+ }
+
+ self.s.eat_while(is_id_continue);
+ let read = self.s.eaten_from(start);
+
+ if let Some(keyword) = keyword(read) {
+ return keyword;
+ }
+
+ if self.mode == TokenMode::Code {
+ if let Ok(color) = RgbaColor::from_str(read) {
+ return Token::Color(color);
+ }
+ }
+
+ Token::Invalid(read)
+ }
+
fn whitespace(&mut self, first: char) -> Token<'s> {
// Fast path for just a single space
if first == ' ' && !self.s.check(|c| c.is_whitespace()) {
@@ -182,10 +205,10 @@ impl<'s> Tokens<'s> {
c if c.is_whitespace() => true,
// Comments.
'/' if self.s.check(|c| c == '/' || c == '*') => true,
- // Parenthesis.
- '[' | ']' | '{' | '}' => true,
+ // Parenthesis and hashtag.
+ '[' | ']' | '{' | '}' | '#' => true,
// Markup.
- '*' | '_' | '#' | '~' | '`' | '$' => true,
+ '*' | '_' | '=' | '~' | '`' | '$' => true,
// Escaping.
'\\' => true,
_ => false,
@@ -198,21 +221,6 @@ impl<'s> Tokens<'s> {
Token::Text(self.s.eaten_from(start))
}
- fn hash(&mut self, start: usize) -> Token<'s> {
- if self.s.check(is_id_start) {
- self.s.eat();
- self.s.eat_while(is_id_continue);
- let read = self.s.eaten_from(start);
- if let Some(keyword) = keyword(read) {
- keyword
- } else {
- Token::Invalid(read)
- }
- } else {
- Token::Hash
- }
- }
-
fn raw(&mut self) -> Token<'s> {
let mut backticks = 1;
while self.s.eat_if('`') {
@@ -276,10 +284,10 @@ impl<'s> Tokens<'s> {
match c {
// Backslash and comments.
'\\' | '/' |
- // Parenthesis.
- '[' | ']' | '{' | '}' |
+ // Parenthesis and hashtag.
+ '[' | ']' | '{' | '}' | '#' |
// Markup.
- '*' | '_' | '#' | '~' | '`' | '$' => {
+ '*' | '_' | '=' | '~' | '`' | '$' => {
let start = self.s.index();
self.s.eat_assert(c);
Token::Text(&self.s.eaten_from(start))
@@ -367,18 +375,6 @@ impl<'s> Tokens<'s> {
}
}
- fn hex(&mut self, start: usize) -> Token<'s> {
- self.s.eat_while(is_id_continue);
- let read = self.s.eaten_from(start);
- if let Some(keyword) = keyword(read) {
- keyword
- } else if let Ok(color) = RgbaColor::from_str(read) {
- Token::Color(color)
- } else {
- Token::Invalid(read)
- }
- }
-
fn string(&mut self) -> Token<'s> {
let mut escaped = false;
Token::Str(TokenStr {
@@ -596,8 +592,8 @@ mod tests {
// Test markup tokens.
t!(Markup[" a1"]: "*" => Star);
t!(Markup: "_" => Underscore);
- t!(Markup[""]: "###" => Hash, Hash, Hash);
- t!(Markup["a1/"]: "# " => Hash, Space(0));
+ t!(Markup[""]: "===" => Eq, Eq, Eq);
+ t!(Markup["a1/"]: "= " => Eq, Space(0));
t!(Markup: "~" => Tilde);
t!(Markup[" "]: r"\" => Backslash);
}
@@ -655,10 +651,9 @@ mod tests {
];
for &(s, t) in &both {
- t!(Code[" "]: format!("#{}", s) => t);
- t!(Markup[" "]: format!("#{}", s) => t);
- t!(Markup[" "]: format!("#{0}#{0}", s) => t, t);
- t!(Markup[" /"]: format!("# {}", s) => Hash, Space(0), Text(s));
+ t!(Both[" "]: format!("#{}", s) => t);
+ t!(Both[" "]: format!("#{0}#{0}", s) => t, t);
+ t!(Markup[" /"]: format!("# {}", s) => Token::Invalid("#"), Space(0), Text(s));
}
let code = [
@@ -713,7 +708,7 @@ mod tests {
// Test code symbols in text.
t!(Markup[" /"]: "a():\"b" => Text("a():\"b"));
- t!(Markup[" /"]: ";:,=|/+-" => Text(";:,=|/+-"));
+ t!(Markup[" /"]: ";:,|/+-" => Text(";:,|/+-"));
// Test text ends.
t!(Markup[""]: "hello " => Text("hello"), Space(0));
@@ -765,17 +760,17 @@ mod tests {
t!(Markup: r"\}" => Text("}"));
t!(Markup: r"\*" => Text("*"));
t!(Markup: r"\_" => Text("_"));
- t!(Markup: r"\#" => Text("#"));
+ t!(Markup: r"\=" => Text("="));
t!(Markup: r"\~" => Text("~"));
t!(Markup: r"\`" => Text("`"));
t!(Markup: r"\$" => Text("$"));
+ t!(Markup: r"\#" => Text("#"));
// Test unescapable symbols.
t!(Markup[" /"]: r"\a" => Text(r"\"), Text("a"));
t!(Markup[" /"]: r"\u" => Text(r"\"), Text("u"));
t!(Markup[" /"]: r"\1" => Text(r"\"), Text("1"));
t!(Markup[" /"]: r"\:" => Text(r"\"), Text(":"));
- t!(Markup[" /"]: r"\=" => Text(r"\"), Text("="));
t!(Markup[" /"]: r#"\""# => Text(r"\"), Text("\""));
// Test basic unicode escapes.
@@ -947,7 +942,7 @@ mod tests {
t!(Code: "1%%" => Percent(1.0), Invalid("%"));
// Test invalid keyword.
- t!(Markup[" /"]: "#-" => Hash, Text("-"));
+ t!(Markup[" /"]: "#-" => Invalid("#-"));
t!(Markup[" /"]: "#do" => Invalid("#do"));
t!(Code[" /"]: r"#letter" => Invalid(r"#letter"));
}