Better tokenization testing 🌋

- Better tokenization test coverage.
- Suffix testing: Each test case is tested with many different suffixes to ensure correct token ends.
- Improves expression parsing (fixes #3).

1 files changed, 11 insertions, 7 deletions

diff --git a/src/syntax/ident.rs b/src/syntax/ident.rs
index f8c38cfb..55f97f95 100644
--- a/src/syntax/ident.rs
+++ b/src/syntax/ident.rs
@@ -46,13 +46,17 @@ impl Deref for Ident {
 /// Whether the string is a valid identifier.
 pub fn is_ident(string: &str) -> bool {
     let mut chars = string.chars();
-    if matches!(chars.next(), Some(c) if c.is_xid_start() || is_also_ok(c)) {
-        chars.all(|c| c.is_xid_continue() || is_also_ok(c))
-    } else {
-        false
-    }
+    chars
+        .next()
+        .map_or(false, |c| is_id_start(c) && chars.all(is_id_continue))
+}
+
+/// Whether the character can start an identifier.
+pub fn is_id_start(c: char) -> bool {
+    c.is_xid_start() || c == '_'
 }
 
-fn is_also_ok(c: char) -> bool {
-    c == '-' || c == '_'
+/// Whether the character can continue an identifier.
+pub fn is_id_continue(c: char) -> bool {
+    c.is_xid_continue() || c == '_' || c == '-'
 }