diff options
| -rw-r--r-- | src/syntax/expr.rs | 21 | ||||
| -rw-r--r-- | src/syntax/tokens.rs | 127 | ||||
| -rw-r--r-- | tests/parse.rs | 12 |
3 files changed, 66 insertions, 94 deletions
diff --git a/src/syntax/expr.rs b/src/syntax/expr.rs index a1680861..ed5e50df 100644 --- a/src/syntax/expr.rs +++ b/src/syntax/expr.rs @@ -166,27 +166,6 @@ impl Display for Ident { debug_display!(Ident); -/// Whether this word is a valid identifier. -pub fn is_identifier(string: &str) -> bool { - let mut chars = string.chars(); - - match chars.next() { - Some('-') => {} - Some(c) if UnicodeXID::is_xid_start(c) => {} - _ => return false, - } - - while let Some(c) = chars.next() { - match c { - '.' | '-' => {} - c if UnicodeXID::is_xid_continue(c) => {} - _ => return false, - } - } - - true -} - /// Kinds of expressions. pub trait ExpressionKind: Sized { const NAME: &'static str; diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs index d355b3cc..ae5cfe48 100644 --- a/src/syntax/tokens.rs +++ b/src/syntax/tokens.rs @@ -72,9 +72,11 @@ pub fn tokenize(src: &str) -> Tokens { /// An iterator over the tokens of a string of source code. pub struct Tokens<'s> { src: &'s str, - chars: Characters<'s>, state: State, stack: Vec<State>, + iter: Peekable<Chars<'s>>, + position: Position, + index: usize, } #[derive(Debug, Copy, Clone, Eq, PartialEq)] @@ -88,9 +90,11 @@ impl<'s> Tokens<'s> { pub fn new(src: &'s str) -> Tokens<'s> { Tokens { src, - chars: Characters::new(src), state: State::Body, stack: vec![], + iter: src.chars().peekable(), + position: Position::ZERO, + index: 0, } } } @@ -100,26 +104,29 @@ impl<'s> Iterator for Tokens<'s> { /// Parse the next token in the source code. fn next(&mut self) -> Option<Spanned<Token<'s>>> { - let start = self.chars.position(); - let first = self.chars.next()?; - let second = self.chars.peek(); + let start = self.pos(); + let first = self.eat()?; let token = match first { // Comments. - '/' if second == Some('/') => self.parse_line_comment(), - '/' if second == Some('*') => self.parse_block_comment(), - '*' if second == Some('/') => { self.eat(); StarSlash } + '/' if self.peek() == Some('/') => self.parse_line_comment(), + '/' if self.peek() == Some('*') => self.parse_block_comment(), + '*' if self.peek() == Some('/') => { self.eat(); StarSlash } // Whitespace. c if c.is_whitespace() => self.parse_whitespace(start), // Functions. - '[' => { self.set_state(Header); LeftBracket } + '[' => { + self.stack.push(self.state); + self.state = Header; + LeftBracket + } ']' => { - if self.state == Header && second == Some('[') { + if self.state == Header && self.peek() == Some('[') { self.state = StartBody; } else { - self.pop_state(); + self.state = self.stack.pop().unwrap_or(Body); } RightBracket @@ -164,7 +171,7 @@ impl<'s> Iterator for Tokens<'s> { } }; - let end = self.chars.position(); + let end = self.pos(); let span = Span { start, end }; Some(Spanned { v: token, span }) @@ -206,7 +213,7 @@ impl<'s> Tokens<'s> { fn parse_whitespace(&mut self, start: Position) -> Token<'s> { self.read_string_until(|n| !n.is_whitespace(), false, 0, 0); - let end = self.chars.position(); + let end = self.pos(); Whitespace(end.line - start.line) } @@ -234,9 +241,9 @@ impl<'s> Tokens<'s> { } } - let c = self.chars.peek().unwrap_or('n'); + let c = self.peek().unwrap_or('n'); if self.state == Body && is_escapable(c) { - let index = self.chars.index(); + let index = self.index(); self.eat(); Text(&self.src[index .. index + c.len_utf8()]) } else { @@ -267,22 +274,22 @@ impl<'s> Tokens<'s> { offset_start: isize, offset_end: isize, ) -> &'s str where F: FnMut(char) -> bool { - let start = ((self.chars.index() as isize) + offset_start) as usize; + let start = ((self.index() as isize) + offset_start) as usize; let mut matched = false; - while let Some(c) = self.chars.peek() { + while let Some(c) = self.peek() { if f(c) { matched = true; if eat_match { - self.chars.next(); + self.eat(); } break; } - self.chars.next(); + self.eat(); } - let mut end = self.chars.index(); + let mut end = self.index(); if matched { end = ((end as isize) + offset_end) as usize; } @@ -290,17 +297,32 @@ impl<'s> Tokens<'s> { &self.src[start .. end] } - fn set_state(&mut self, state: State) { - self.stack.push(self.state); - self.state = state; + fn eat(&mut self) -> Option<char> { + let c = self.iter.next()?; + let len = c.len_utf8(); + + self.index += len; + + if is_newline_char(c) && !(c == '\r' && self.peek() == Some('\n')) { + self.position.line += 1; + self.position.column = 0; + } else { + self.position.column += len; + } + + Some(c) + } + + fn peek(&mut self) -> Option<char> { + self.iter.peek().copied() } - fn pop_state(&mut self) { - self.state = self.stack.pop().unwrap_or(Body); + fn index(&self) -> usize { + self.index } - fn eat(&mut self) { - self.chars.next(); + fn pos(&self) -> Position { + self.position } } @@ -313,7 +335,7 @@ fn parse_percentage(text: &str) -> Option<f64> { } /// Whether this character denotes a newline. -fn is_newline_char(character: char) -> bool { +pub fn is_newline_char(character: char) -> bool { match character { // Line Feed, Vertical Tab, Form Feed, Carriage Return. '\x0A' ..= '\x0D' => true, @@ -323,46 +345,23 @@ fn is_newline_char(character: char) -> bool { } } -struct Characters<'s> { - iter: Peekable<Chars<'s>>, - position: Position, - index: usize, -} +/// Whether this word is a valid identifier. +pub fn is_identifier(string: &str) -> bool { + let mut chars = string.chars(); -impl<'s> Characters<'s> { - fn new(src: &'s str) -> Characters<'s> { - Characters { - iter: src.chars().peekable(), - position: Position::ZERO, - index: 0, - } + match chars.next() { + Some('-') => {} + Some(c) if UnicodeXID::is_xid_start(c) => {} + _ => return false, } - fn next(&mut self) -> Option<char> { - let c = self.iter.next()?; - let len = c.len_utf8(); - - self.index += len; - - if is_newline_char(c) && !(c == '\r' && self.peek() == Some('\n')) { - self.position.line += 1; - self.position.column = 0; - } else { - self.position.column += len; + while let Some(c) = chars.next() { + match c { + '.' | '-' => {} + c if UnicodeXID::is_xid_continue(c) => {} + _ => return false, } - - Some(c) - } - - fn peek(&mut self) -> Option<char> { - self.iter.peek().copied() } - fn index(&self) -> usize { - self.index - } - - fn position(&self) -> Position { - self.position - } + true } diff --git a/tests/parse.rs b/tests/parse.rs index 14a5b22d..02c4f9b7 100644 --- a/tests/parse.rs +++ b/tests/parse.rs @@ -9,18 +9,12 @@ use Token::{ LeftBracket as LB, RightBracket as RB, LeftParen as LP, RightParen as RP, LeftBrace as LBR, RightBrace as RBR, - Colon as CL, Comma as CM, Equals as EQ, Expr as E, + Colon as CL, Comma as CM, Equals as EQ, + ExprIdent as ID, ExprString as STR, ExprSize as SIZE, + ExprNumber as NUM, ExprBool as BOOL, Star as ST, Underscore as U, Backtick as B, Text as T, }; -use Expression as Expr; -fn ID(ident: &str) -> Token { E(Expr::Ident(Ident::new(ident.to_string()).unwrap())) } -fn STR(ident: &str) -> Token { E(Expr::Str(ident.to_string())) } -fn SIZE(size: Size) -> Token<'static> { E(Expr::Size(size)) } -fn NUM(num: f64) -> Token<'static> { E(Expr::Num(num)) } -fn BOOL(b: bool) -> Token<'static> { E(Expr::Bool(b)) } - - /// Parses the test syntax. macro_rules! tokens { ($($task:ident $src:expr =>($line:expr)=> [$($target:tt)*])*) => ({ |