diff options
Diffstat (limited to 'src/syntax/tokens.rs')
| -rw-r--r-- | src/syntax/tokens.rs | 39 |
1 files changed, 36 insertions, 3 deletions
diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs index d0adbf60..0a8e2f17 100644 --- a/src/syntax/tokens.rs +++ b/src/syntax/tokens.rs @@ -2,9 +2,11 @@ use std::iter::Peekable; use std::str::Chars; use unicode_xid::UnicodeXID; -use super::*; -use Token::*; -use TokenizationMode::*; +use crate::size::Size; +use super::span::{Position, Span, Spanned}; + +use self::Token::*; +use self::TokenizationMode::*; /// A minimal semantic entity of source code. @@ -68,6 +70,37 @@ pub enum Token<'s> { Invalid(&'s str), } +impl<'s> Token<'s> { + /// The natural-language name for this token for use in error messages. + pub fn name(self) -> &'static str { + match self { + Space(_) => "space", + LineComment(_) => "line comment", + BlockComment(_) => "block comment", + Function { .. } => "function", + LeftParen => "opening paren", + RightParen => "closing paren", + LeftBrace => "opening brace", + RightBrace => "closing brace", + Colon => "colon", + Comma => "comma", + Equals => "equals sign", + ExprIdent(_) => "identifier", + ExprStr { .. } => "string", + ExprNumber(_) => "number", + ExprSize(_) => "size", + ExprBool(_) => "boolean", + Star => "star", + Underscore => "underscore", + Backtick => "backtick", + Text(_) => "invalid identifier", + Invalid("]") => "closing bracket", + Invalid("*/") => "end of block comment", + Invalid(_) => "invalid token", + } + } +} + /// An iterator over the tokens of a string of source code. pub struct Tokens<'s> { src: &'s str, |