diff options
Diffstat (limited to 'src/syntax/token.rs')
| -rw-r--r-- | src/syntax/token.rs | 147 |
1 files changed, 80 insertions, 67 deletions
diff --git a/src/syntax/token.rs b/src/syntax/token.rs index 4cb8501f..5c159bbd 100644 --- a/src/syntax/token.rs +++ b/src/syntax/token.rs @@ -1,4 +1,4 @@ -//! Tokenization. +//! Token definition. use crate::length::Length; @@ -8,6 +8,8 @@ pub enum Token<'s> { /// One or more whitespace characters. The contained `usize` denotes the /// number of newlines that were contained in the whitespace. Space(usize), + /// A consecutive non-markup string. + Text(&'s str), /// A line comment with inner string contents `//<str>\n`. LineComment(&'s str), @@ -15,6 +17,20 @@ pub enum Token<'s> { /// can contain nested block comments. BlockComment(&'s str), + /// A star. It can appear in a function header where it signifies the + /// multiplication of expressions or the body where it modifies the styling. + Star, + /// An underscore in body-text. + Underscore, + /// A backslash followed by whitespace in text. + Backslash, + /// A hashtag indicating a section heading. + Hashtag, + /// A raw block. + Raw(TokenRaw<'s>), + /// A unicode escape sequence. + UnicodeEscape(TokenUnicodeEscape<'s>), + /// A left bracket starting a function invocation or body: `[`. LeftBracket, /// A right bracket ending a function invocation or body: `]`. @@ -28,29 +44,24 @@ pub enum Token<'s> { /// A right parenthesis in a function header: `)`. RightParen, - /// A double forward chevron in a function header: `>>`. - Chain, - /// A colon in a function header: `:`. Colon, /// A comma in a function header: `,`. Comma, /// An equals sign in a function header: `=`. Equals, + /// A double forward chevron in a function header: `>>`. + Chain, + /// A plus in a function header, signifying the addition of expressions. + Plus, + /// A hyphen in a function header, signifying the subtraction of + /// expressions. + Hyphen, + /// A slash in a function header, signifying the division of expressions. + Slash, /// An identifier in a function header: `center`. Ident(&'s str), - /// A quoted string in a function header: `"..."`. - Str { - /// The string inside the quotes. - /// - /// _Note_: If the string contains escape sequences these are not yet - /// applied to be able to just store a string slice here instead of - /// a String. The escaping is done later in the parser. - string: &'s str, - /// Whether the closing quote was present. - terminated: bool, - }, /// A boolean in a function header: `true | false`. Bool(bool), /// A number in a function header: `3.14`. @@ -59,48 +70,44 @@ pub enum Token<'s> { Length(Length), /// A hex value in a function header: `#20d82a`. Hex(&'s str), - /// A plus in a function header, signifying the addition of expressions. - Plus, - /// A hyphen in a function header, signifying the subtraction of - /// expressions. - Hyphen, - /// A slash in a function header, signifying the division of expressions. - Slash, + /// A quoted string in a function header: `"..."`. + Str(TokenStr<'s>), - /// A star. It can appear in a function header where it signifies the - /// multiplication of expressions or the body where it modifies the styling. - Star, - /// An underscore in body-text. - Underscore, - /// A backslash followed by whitespace in text. - Backslash, + /// Things that are not valid in the context they appeared in. + Invalid(&'s str), +} - /// A hashtag token in the body can indicate compute mode or headings. - Hashtag, +/// A quoted string in a function header: `"..."`. +#[derive(Debug, Copy, Clone, PartialEq)] +pub struct TokenStr<'s> { + /// The string inside the quotes. + /// + /// _Note_: If the string contains escape sequences these are not yet + /// applied to be able to just store a string slice here instead of + /// a `String`. The resolving is done later in the parser. + pub string: &'s str, + /// Whether the closing quote was present. + pub terminated: bool, +} - /// A unicode escape sequence. - UnicodeEscape { - /// The escape sequence between two braces. - sequence: &'s str, - /// Whether the closing brace was present. - terminated: bool, - }, - - /// Raw block. - Raw { - /// The raw text between the backticks. - raw: &'s str, - /// The number of opening backticks. - backticks: usize, - /// Whether all closing backticks were present. - terminated: bool, - }, - - /// Any other consecutive string. - Text(&'s str), +/// A unicode escape sequence. +#[derive(Debug, Copy, Clone, PartialEq)] +pub struct TokenUnicodeEscape<'s> { + /// The escape sequence between two braces. + pub sequence: &'s str, + /// Whether the closing brace was present. + pub terminated: bool, +} - /// Things that are not valid in the context they appeared in. - Invalid(&'s str), +/// A raw block. +#[derive(Debug, Copy, Clone, PartialEq)] +pub struct TokenRaw<'s> { + /// The raw text between the backticks. + pub text: &'s str, + /// The number of opening backticks. + pub backticks: usize, + /// Whether all closing backticks were present. + pub terminated: bool, } impl<'s> Token<'s> { @@ -108,34 +115,40 @@ impl<'s> Token<'s> { pub fn name(self) -> &'static str { match self { Self::Space(_) => "space", + Self::Text(_) => "text", + Self::LineComment(_) => "line comment", Self::BlockComment(_) => "block comment", + + Self::Star => "star", + Self::Underscore => "underscore", + Self::Backslash => "backslash", + Self::Hashtag => "hashtag", + Self::Raw { .. } => "raw block", + Self::UnicodeEscape { .. } => "unicode escape sequence", + Self::LeftBracket => "opening bracket", Self::RightBracket => "closing bracket", - Self::LeftParen => "opening paren", - Self::RightParen => "closing paren", Self::LeftBrace => "opening brace", Self::RightBrace => "closing brace", - Self::Chain => "function chain operator", + Self::LeftParen => "opening paren", + Self::RightParen => "closing paren", + Self::Colon => "colon", Self::Comma => "comma", Self::Equals => "equals sign", + Self::Chain => "function chaining operator", + Self::Plus => "plus sign", + Self::Hyphen => "minus sign", + Self::Slash => "slash", + Self::Ident(_) => "identifier", - Self::Str { .. } => "string", Self::Bool(_) => "bool", Self::Number(_) => "number", Self::Length(_) => "length", Self::Hex(_) => "hex value", - Self::Plus => "plus", - Self::Hyphen => "minus", - Self::Slash => "slash", - Self::Star => "star", - Self::Underscore => "underscore", - Self::Backslash => "backslash", - Self::Hashtag => "hashtag", - Self::UnicodeEscape { .. } => "unicode escape sequence", - Self::Raw { .. } => "raw block", - Self::Text(_) => "text", + Self::Str { .. } => "string", + Self::Invalid("*/") => "end of block comment", Self::Invalid(_) => "invalid token", } |