Implement low-level char parser 🥜

3 files changed, 26 insertions, 16 deletions

diff --git a/src/syntax/lines.rs b/src/syntax/lines.rs
index 86fc461b..7f7ee049 100644
--- a/src/syntax/lines.rs
+++ b/src/syntax/lines.rs
@@ -3,7 +3,7 @@
 use std::fmt::{self, Debug, Display, Formatter};
 
 use super::Pos;
-use crate::parse::is_newline_char;
+use crate::parse::{is_newline_char, CharParser};
 
 /// Enables conversion of byte position to locations.
 pub struct LineMap<'s> {
@@ -15,17 +15,11 @@ impl<'s> LineMap<'s> {
     /// Create a new line map for a source string.
     pub fn new(src: &'s str) -> Self {
         let mut line_starts = vec![Pos::ZERO];
-        let mut iter = src.char_indices().peekable();
+        let mut p = CharParser::new(src);
 
-        while let Some((mut i, c)) = iter.next() {
+        while let Some(c) = p.eat_merging_crlf() {
             if is_newline_char(c) {
-                i += c.len_utf8();
-                if c == '\r' && matches!(iter.peek(), Some((_, '\n'))) {
-                    i += '\n'.len_utf8();
-                    iter.next();
-                }
-
-                line_starts.push(Pos(i as u32));
+                line_starts.push(p.index().into());
             }
         }
 
diff --git a/src/syntax/token.rs b/src/syntax/token.rs
index b7d4c4e2..4cb8501f 100644
--- a/src/syntax/token.rs
+++ b/src/syntax/token.rs
@@ -19,14 +19,15 @@ pub enum Token<'s> {
     LeftBracket,
     /// A right bracket ending a function invocation or body: `]`.
     RightBracket,
+    /// A left brace indicating the start of content: `{`.
+    LeftBrace,
+    /// A right brace indicating the end of content: `}`.
+    RightBrace,
     /// A left parenthesis in a function header: `(`.
     LeftParen,
     /// A right parenthesis in a function header: `)`.
     RightParen,
-    /// A left brace in a function header: `{`.
-    LeftBrace,
-    /// A right brace in a function header: `}`.
-    RightBrace,
+
     /// A double forward chevron in a function header: `>>`.
     Chain,
 
diff --git a/src/syntax/tree.rs b/src/syntax/tree.rs
index 51a7937a..bfbb3706 100644
--- a/src/syntax/tree.rs
+++ b/src/syntax/tree.rs
@@ -2,6 +2,8 @@
 
 use std::fmt::{self, Debug, Formatter};
 
+use unicode_xid::UnicodeXID;
+
 use super::span::{SpanVec, SpanWith, Spanned};
 use super::Decoration;
 use crate::color::RgbaColor;
@@ -9,7 +11,6 @@ use crate::compute::table::{SpannedEntry, Table};
 use crate::compute::value::{TableValue, Value};
 use crate::layout::LayoutContext;
 use crate::length::Length;
-use crate::parse::is_identifier;
 use crate::{DynFuture, Feedback};
 
 /// A collection of nodes which form a tree together with the nodes' children.
@@ -233,7 +234,7 @@ pub struct Ident(pub String);
 impl Ident {
     /// Create a new identifier from a string checking that it is a valid.
     pub fn new(ident: impl AsRef<str> + Into<String>) -> Option<Self> {
-        if is_identifier(ident.as_ref()) {
+        if Self::is_ident(ident.as_ref()) {
             Some(Self(ident.into()))
         } else {
             None
@@ -244,6 +245,20 @@ impl Ident {
     pub fn as_str(&self) -> &str {
         self.0.as_str()
     }
+
+    /// Whether the string is a valid identifier.
+    pub fn is_ident(string: &str) -> bool {
+        fn is_ok(c: char) -> bool {
+            c == '-' || c == '_'
+        }
+
+        let mut chars = string.chars();
+        if matches!(chars.next(), Some(c) if c.is_xid_start() || is_ok(c)) {
+            chars.all(|c| c.is_xid_continue() || is_ok(c))
+        } else {
+            false
+        }
+    }
 }
 
 impl Debug for Ident {