From 6bbedeaa2c6e0068e2fb6602cbf0002fb6a6ce03 Mon Sep 17 00:00:00 2001 From: Laurenz Date: Wed, 16 Dec 2020 15:42:02 +0100 Subject: =?UTF-8?q?Better=20tokenization=20testing=20=F0=9F=8C=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Better tokenization test coverage. - Suffix testing: Each test case is tested with many different suffixes to ensure correct token ends. - Improves expression parsing (fixes #3). --- src/parse/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/parse/mod.rs') diff --git a/src/parse/mod.rs b/src/parse/mod.rs index 96ef18d2..33de5c24 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -66,7 +66,6 @@ fn node(p: &mut Parser, at_start: bool) -> Option> { // Markup. Token::Star => SynNode::Strong, Token::Underscore => SynNode::Emph, - Token::Backslash => SynNode::Linebreak, Token::Hashtag => { if at_start { SynNode::Heading(heading(p, start)) @@ -74,9 +73,10 @@ fn node(p: &mut Parser, at_start: bool) -> Option> { SynNode::Text(p.eaten_from(start).into()) } } - Token::NonBreakingSpace => SynNode::Text("\u{00A0}".into()), - Token::Raw(token) => SynNode::Raw(raw(p, token)), + Token::Tilde => SynNode::Text("\u{00A0}".into()), + Token::Backslash => SynNode::Linebreak, Token::UnicodeEscape(token) => SynNode::Text(unicode_escape(p, token, start)), + Token::Raw(token) => SynNode::Raw(raw(p, token)), // Functions. Token::LeftBracket => { -- cgit v1.2.3