Better tokenization testing 🌋

- Better tokenization test coverage.
- Suffix testing: Each test case is tested with many different suffixes to ensure correct token ends.
- Improves expression parsing (fixes #3).

1 files changed, 7 insertions, 7 deletions

diff --git a/src/syntax/token.rs b/src/syntax/token.rs
index e630c50c..21a56004 100644
--- a/src/syntax/token.rs
+++ b/src/syntax/token.rs
@@ -24,16 +24,16 @@ pub enum Token<'s> {
     Star,
     /// An underscore: `_`.
     Underscore,
-    /// A backslash followed by whitespace: `\`.
-    Backslash,
     /// A hashtag indicating a section heading: `#`.
     Hashtag,
-    /// A non-breaking space: `~`.
-    NonBreakingSpace,
-    /// A raw block: `` `...` ``.
-    Raw(TokenRaw<'s>),
+    /// A tilde: `~`.
+    Tilde,
+    /// A backslash followed by whitespace: `\`.
+    Backslash,
     /// A unicode escape sequence: `\u{1F5FA}`.
     UnicodeEscape(TokenUnicodeEscape<'s>),
+    /// A raw block: `` `...` ``.
+    Raw(TokenRaw<'s>),
 
     /// A left bracket: `[`.
     LeftBracket,
@@ -134,7 +134,7 @@ impl<'s> Token<'s> {
             Self::Underscore => "underscore",
             Self::Backslash => "backslash",
             Self::Hashtag => "hashtag",
-            Self::NonBreakingSpace => "non-breaking space",
+            Self::Tilde => "tidle",
             Self::Raw { .. } => "raw block",
             Self::UnicodeEscape { .. } => "unicode escape sequence",