From 89eb8bae49edb71d9a9279a2210bb1ccaf4dd707 Mon Sep 17 00:00:00 2001 From: Laurenz Date: Sat, 30 Jan 2021 12:09:26 +0100 Subject: =?UTF-8?q?New=20syntax=20=F0=9F=92=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Everything everywhere! - Blocks with curly braces: {} - Templates with brackets: [] - Function templates with hashtag: `#[f]` - Headings with equals sign: `= Introduction` --- src/parse/mod.rs | 114 ++++++++++++++++++++++++++++++---------------------- src/parse/parser.rs | 12 +++--- src/parse/tokens.rs | 89 +++++++++++++++++++--------------------- 3 files changed, 115 insertions(+), 100 deletions(-) (limited to 'src/parse') diff --git a/src/parse/mod.rs b/src/parse/mod.rs index ff75a563..3fc7d483 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -45,38 +45,33 @@ fn tree(p: &mut Parser) -> Tree { fn node(p: &mut Parser, at_start: &mut bool) -> Option { let token = p.peek()?; let node = match token { - // Bracket call. - Token::LeftBracket => { - return Some(Node::Expr(bracket_call(p)?)); + // Whitespace. + Token::Space(newlines) => { + *at_start |= newlines > 0; + if newlines < 2 { Node::Space } else { Node::Parbreak } } - // Code block. - Token::LeftBrace => { - return Some(Node::Expr(block(p, false)?)); - } + // Text. + Token::Text(text) => Node::Text(text.into()), // Markup. Token::Star => Node::Strong, Token::Underscore => Node::Emph, - Token::Tilde => Node::Text("\u{00A0}".into()), - Token::Hash => { + Token::Eq => { if *at_start { return Some(Node::Heading(heading(p))); } else { Node::Text(p.get(p.peek_span()).into()) } } + Token::Tilde => Node::Text("\u{00A0}".into()), Token::Backslash => Node::Linebreak, - Token::Space(newlines) => { - *at_start |= newlines > 0; - if newlines < 2 { Node::Space } else { Node::Parbreak } - } - Token::Text(text) => Node::Text(text.into()), Token::Raw(t) => Node::Raw(raw(p, t)), Token::UnicodeEscape(t) => Node::Text(unicode_escape(p, t)), // Keywords. Token::Let | Token::If | Token::For => { + *at_start = false; let stmt = token == Token::Let; let group = if stmt { Group::Stmt } else { Group::Expr }; @@ -92,6 +87,24 @@ fn node(p: &mut Parser, at_start: &mut bool) -> Option { return expr.map(Node::Expr); } + // Block. + Token::LeftBrace => { + *at_start = false; + return Some(Node::Expr(block(p, false)?)); + } + + // Template. + Token::LeftBracket => { + *at_start = false; + return Some(Node::Expr(template(p))); + } + + // Function template. + Token::HashBracket => { + *at_start = false; + return Some(Node::Expr(bracket_call(p)?)); + } + // Comments. Token::LineComment(_) | Token::BlockComment(_) => { p.eat(); @@ -99,6 +112,7 @@ fn node(p: &mut Parser, at_start: &mut bool) -> Option { } _ => { + *at_start = false; p.unexpected(); return None; } @@ -109,12 +123,12 @@ fn node(p: &mut Parser, at_start: &mut bool) -> Option { /// Parse a heading. fn heading(p: &mut Parser) -> NodeHeading { - // Count hashtags. + // Count depth. let mut level = p.span(|p| { - p.assert(Token::Hash); + p.assert(&[Token::Eq]); let mut level = 0u8; - while p.eat_if(Token::Hash) { + while p.eat_if(Token::Eq) { level = level.saturating_add(1); } level @@ -278,22 +292,18 @@ fn expr_with(p: &mut Parser, min_prec: usize) -> Option { /// Parse a primary expression. fn primary(p: &mut Parser) -> Option { let expr = match p.peek() { - // Template. - Some(Token::LeftBracket) => { - return Some(template(p)); - } - - // Nested block. - Some(Token::LeftBrace) => { - return block(p, true); - } - - // Dictionary or just a parenthesized expression. - Some(Token::LeftParen) => { - return Some(parenthesized(p)); - } + // Basic values. + Some(Token::None) => Expr::None, + Some(Token::Bool(b)) => Expr::Bool(b), + Some(Token::Int(i)) => Expr::Int(i), + Some(Token::Float(f)) => Expr::Float(f), + Some(Token::Length(val, unit)) => Expr::Length(val, unit), + Some(Token::Angle(val, unit)) => Expr::Angle(val, unit), + Some(Token::Percent(p)) => Expr::Percent(p), + Some(Token::Color(color)) => Expr::Color(color), + Some(Token::Str(token)) => Expr::Str(string(p, token)), - // Function or just ident. + // Function or identifier. Some(Token::Ident(id)) => { p.eat(); let ident = Ident(id.into()); @@ -305,23 +315,33 @@ fn primary(p: &mut Parser) -> Option { } } - // Basic values. - Some(Token::None) => Expr::None, - Some(Token::Bool(b)) => Expr::Bool(b), - Some(Token::Int(i)) => Expr::Int(i), - Some(Token::Float(f)) => Expr::Float(f), - Some(Token::Length(val, unit)) => Expr::Length(val, unit), - Some(Token::Angle(val, unit)) => Expr::Angle(val, unit), - Some(Token::Percent(p)) => Expr::Percent(p), - Some(Token::Color(color)) => Expr::Color(color), - Some(Token::Str(token)) => Expr::Str(string(p, token)), - // Keywords. Some(Token::Let) => return expr_let(p), Some(Token::If) => return expr_if(p), Some(Token::For) => return expr_for(p), - // No value. + // Block. + Some(Token::LeftBrace) => { + return block(p, true); + } + + // Template. + Some(Token::LeftBracket) => { + return Some(template(p)); + } + + // Function template. + Some(Token::HashBracket) => { + let call = p.span_if(bracket_call)?.map(Node::Expr); + return Some(Expr::Template(vec![call])); + } + + // Array, dictionary or parenthesized expression. + Some(Token::LeftParen) => { + return Some(parenthesized(p)); + } + + // Nothing. _ => { p.expected("expression"); return None; @@ -380,7 +400,7 @@ fn string(p: &mut Parser, token: TokenStr) -> String { /// Parse a let expression. fn expr_let(p: &mut Parser) -> Option { - p.assert(Token::Let); + p.assert(&[Token::Let]); let mut expr_let = None; if let Some(pat) = p.span_if(ident) { @@ -397,7 +417,7 @@ fn expr_let(p: &mut Parser) -> Option { /// Parse an if expresion. fn expr_if(p: &mut Parser) -> Option { - p.assert(Token::If); + p.assert(&[Token::If]); let mut expr_if = None; if let Some(condition) = p.span_if(expr) { @@ -420,7 +440,7 @@ fn expr_if(p: &mut Parser) -> Option { /// Parse a for expression. fn expr_for(p: &mut Parser) -> Option { - p.assert(Token::For); + p.assert(&[Token::For]); let mut expr_for = None; if let Some(pat) = p.span_if(for_pattern) { diff --git a/src/parse/parser.rs b/src/parse/parser.rs index 906d9e62..2ca8eb10 100644 --- a/src/parse/parser.rs +++ b/src/parse/parser.rs @@ -105,9 +105,9 @@ impl<'s> Parser<'s> { self.repeek(); match group { - Group::Paren => self.assert(Token::LeftParen), - Group::Bracket => self.assert(Token::LeftBracket), - Group::Brace => self.assert(Token::LeftBrace), + Group::Paren => self.assert(&[Token::LeftParen]), + Group::Bracket => self.assert(&[Token::HashBracket, Token::LeftBracket]), + Group::Brace => self.assert(&[Token::LeftBrace]), Group::Subheader => {} Group::Stmt => {} Group::Expr => {} @@ -210,10 +210,10 @@ impl<'s> Parser<'s> { eaten } - /// Consume the next token, debug-asserting that it is the given one. - pub fn assert(&mut self, t: Token) { + /// Consume the next token, debug-asserting that it is one of the given ones. + pub fn assert(&mut self, ts: &[Token]) { let next = self.eat(); - debug_assert_eq!(next, Some(t)); + debug_assert!(next.map_or(false, |n| ts.contains(&n))); } /// Skip whitespace and comment tokens. diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs index 056bbbbb..4f5d8ab4 100644 --- a/src/parse/tokens.rs +++ b/src/parse/tokens.rs @@ -67,12 +67,15 @@ impl<'s> Iterator for Tokens<'s> { loop { // Common elements. return Some(match c { - // Functions, blocks and terminators. + // Blocks and templates. '[' => Token::LeftBracket, ']' => Token::RightBracket, '{' => Token::LeftBrace, '}' => Token::RightBrace, + // Keywords, function templates, colors. + '#' => self.hash(start), + // Whitespace. c if c.is_whitespace() => self.whitespace(c), @@ -90,8 +93,8 @@ impl<'s> Iterator for Tokens<'s> { // Markup. '*' => Token::Star, '_' => Token::Underscore, + '=' => Token::Eq, '~' => Token::Tilde, - '#' => self.hash(start), '`' => self.raw(), '$' => self.math(), '\\' => self.backslash(), @@ -140,8 +143,7 @@ impl<'s> Iterator for Tokens<'s> { self.number(start, c) } - // Hex values and strings. - '#' => self.hex(start), + // Strings. '"' => self.string(), _ => Token::Invalid(self.s.eaten_from(start)), @@ -151,6 +153,27 @@ impl<'s> Iterator for Tokens<'s> { } impl<'s> Tokens<'s> { + fn hash(&mut self, start: usize) -> Token<'s> { + if self.s.eat_if('[') { + return Token::HashBracket; + } + + self.s.eat_while(is_id_continue); + let read = self.s.eaten_from(start); + + if let Some(keyword) = keyword(read) { + return keyword; + } + + if self.mode == TokenMode::Code { + if let Ok(color) = RgbaColor::from_str(read) { + return Token::Color(color); + } + } + + Token::Invalid(read) + } + fn whitespace(&mut self, first: char) -> Token<'s> { // Fast path for just a single space if first == ' ' && !self.s.check(|c| c.is_whitespace()) { @@ -182,10 +205,10 @@ impl<'s> Tokens<'s> { c if c.is_whitespace() => true, // Comments. '/' if self.s.check(|c| c == '/' || c == '*') => true, - // Parenthesis. - '[' | ']' | '{' | '}' => true, + // Parenthesis and hashtag. + '[' | ']' | '{' | '}' | '#' => true, // Markup. - '*' | '_' | '#' | '~' | '`' | '$' => true, + '*' | '_' | '=' | '~' | '`' | '$' => true, // Escaping. '\\' => true, _ => false, @@ -198,21 +221,6 @@ impl<'s> Tokens<'s> { Token::Text(self.s.eaten_from(start)) } - fn hash(&mut self, start: usize) -> Token<'s> { - if self.s.check(is_id_start) { - self.s.eat(); - self.s.eat_while(is_id_continue); - let read = self.s.eaten_from(start); - if let Some(keyword) = keyword(read) { - keyword - } else { - Token::Invalid(read) - } - } else { - Token::Hash - } - } - fn raw(&mut self) -> Token<'s> { let mut backticks = 1; while self.s.eat_if('`') { @@ -276,10 +284,10 @@ impl<'s> Tokens<'s> { match c { // Backslash and comments. '\\' | '/' | - // Parenthesis. - '[' | ']' | '{' | '}' | + // Parenthesis and hashtag. + '[' | ']' | '{' | '}' | '#' | // Markup. - '*' | '_' | '#' | '~' | '`' | '$' => { + '*' | '_' | '=' | '~' | '`' | '$' => { let start = self.s.index(); self.s.eat_assert(c); Token::Text(&self.s.eaten_from(start)) @@ -367,18 +375,6 @@ impl<'s> Tokens<'s> { } } - fn hex(&mut self, start: usize) -> Token<'s> { - self.s.eat_while(is_id_continue); - let read = self.s.eaten_from(start); - if let Some(keyword) = keyword(read) { - keyword - } else if let Ok(color) = RgbaColor::from_str(read) { - Token::Color(color) - } else { - Token::Invalid(read) - } - } - fn string(&mut self) -> Token<'s> { let mut escaped = false; Token::Str(TokenStr { @@ -596,8 +592,8 @@ mod tests { // Test markup tokens. t!(Markup[" a1"]: "*" => Star); t!(Markup: "_" => Underscore); - t!(Markup[""]: "###" => Hash, Hash, Hash); - t!(Markup["a1/"]: "# " => Hash, Space(0)); + t!(Markup[""]: "===" => Eq, Eq, Eq); + t!(Markup["a1/"]: "= " => Eq, Space(0)); t!(Markup: "~" => Tilde); t!(Markup[" "]: r"\" => Backslash); } @@ -655,10 +651,9 @@ mod tests { ]; for &(s, t) in &both { - t!(Code[" "]: format!("#{}", s) => t); - t!(Markup[" "]: format!("#{}", s) => t); - t!(Markup[" "]: format!("#{0}#{0}", s) => t, t); - t!(Markup[" /"]: format!("# {}", s) => Hash, Space(0), Text(s)); + t!(Both[" "]: format!("#{}", s) => t); + t!(Both[" "]: format!("#{0}#{0}", s) => t, t); + t!(Markup[" /"]: format!("# {}", s) => Token::Invalid("#"), Space(0), Text(s)); } let code = [ @@ -713,7 +708,7 @@ mod tests { // Test code symbols in text. t!(Markup[" /"]: "a():\"b" => Text("a():\"b")); - t!(Markup[" /"]: ";:,=|/+-" => Text(";:,=|/+-")); + t!(Markup[" /"]: ";:,|/+-" => Text(";:,|/+-")); // Test text ends. t!(Markup[""]: "hello " => Text("hello"), Space(0)); @@ -765,17 +760,17 @@ mod tests { t!(Markup: r"\}" => Text("}")); t!(Markup: r"\*" => Text("*")); t!(Markup: r"\_" => Text("_")); - t!(Markup: r"\#" => Text("#")); + t!(Markup: r"\=" => Text("=")); t!(Markup: r"\~" => Text("~")); t!(Markup: r"\`" => Text("`")); t!(Markup: r"\$" => Text("$")); + t!(Markup: r"\#" => Text("#")); // Test unescapable symbols. t!(Markup[" /"]: r"\a" => Text(r"\"), Text("a")); t!(Markup[" /"]: r"\u" => Text(r"\"), Text("u")); t!(Markup[" /"]: r"\1" => Text(r"\"), Text("1")); t!(Markup[" /"]: r"\:" => Text(r"\"), Text(":")); - t!(Markup[" /"]: r"\=" => Text(r"\"), Text("=")); t!(Markup[" /"]: r#"\""# => Text(r"\"), Text("\"")); // Test basic unicode escapes. @@ -947,7 +942,7 @@ mod tests { t!(Code: "1%%" => Percent(1.0), Invalid("%")); // Test invalid keyword. - t!(Markup[" /"]: "#-" => Hash, Text("-")); + t!(Markup[" /"]: "#-" => Invalid("#-")); t!(Markup[" /"]: "#do" => Invalid("#do")); t!(Code[" /"]: r"#letter" => Invalid(r"#letter")); } -- cgit v1.2.3