summaryrefslogtreecommitdiff
path: root/src/syntax
diff options
context:
space:
mode:
Diffstat (limited to 'src/syntax')
-rw-r--r--src/syntax/lexer.rs29
1 files changed, 20 insertions, 9 deletions
diff --git a/src/syntax/lexer.rs b/src/syntax/lexer.rs
index e3c29150..433d0def 100644
--- a/src/syntax/lexer.rs
+++ b/src/syntax/lexer.rs
@@ -1,3 +1,4 @@
+use unicode_segmentation::UnicodeSegmentation;
use unicode_xid::UnicodeXID;
use unscanny::Scanner;
@@ -103,7 +104,7 @@ impl Lexer<'_> {
Some(c) => match self.mode {
LexMode::Markup => self.markup(start, c),
- LexMode::Math => self.math(c),
+ LexMode::Math => self.math(start, c),
LexMode::Code => self.code(start, c),
},
@@ -404,7 +405,7 @@ impl Lexer<'_> {
/// Math.
impl Lexer<'_> {
- fn math(&mut self, c: char) -> SyntaxKind {
+ fn math(&mut self, start: usize, c: char) -> SyntaxKind {
match c {
'\\' => self.backslash(),
':' if self.s.at(is_id_start) => self.maybe_symbol(),
@@ -441,13 +442,7 @@ impl Lexer<'_> {
}
// Other math atoms.
- _ => {
- // Keep numbers together.
- if c.is_numeric() {
- self.s.eat_while(char::is_numeric);
- }
- SyntaxKind::Atom
- }
+ _ => self.atom(start, c),
}
}
@@ -469,6 +464,22 @@ impl Lexer<'_> {
SyntaxKind::Ident
}
+
+ fn atom(&mut self, start: usize, c: char) -> SyntaxKind {
+ // Keep numbers and grapheme clusters together.
+ if c.is_numeric() {
+ self.s.eat_while(char::is_numeric);
+ } else {
+ let len = self
+ .s
+ .get(start..self.s.string().len())
+ .graphemes(true)
+ .next()
+ .map_or(0, str::len);
+ self.s.jump(start + len);
+ }
+ SyntaxKind::Atom
+ }
}
/// Code.