Add lots of new tokens 🍪

1 files changed, 157 insertions, 68 deletions

diff --git a/src/syntax/token.rs b/src/syntax/token.rs
index fb50c4ec..261f2104 100644
--- a/src/syntax/token.rs
+++ b/src/syntax/token.rs
@@ -1,68 +1,114 @@
-use crate::geom::LengthUnit;
+use crate::geom::{AngularUnit, LengthUnit};
 
 /// A minimal semantic entity of source code.
 #[derive(Debug, Copy, Clone, PartialEq)]
 pub enum Token<'s> {
-    /// A consecutive non-markup string.
-    Text(&'s str),
-    /// One or more whitespace characters.
-    ///
-    /// The contained `usize` denotes the number of newlines that were contained
-    /// in the whitespace.
-    Space(usize),
-
-    /// A line comment with inner string contents `//<str>\n`.
-    LineComment(&'s str),
-    /// A block comment with inner string contents `/*<str>*/`.
-    ///
-    /// The comment can contain nested block comments.
-    BlockComment(&'s str),
-
-    /// A left bracket: `[`.
+    /// A left square bracket: `[`.
     LeftBracket,
-    /// A right bracket: `]`.
+    /// A right square bracket: `]`.
     RightBracket,
-    /// A left brace: `{`.
+    /// A left curly brace: `{`.
     LeftBrace,
-    /// A right brace: `}`.
+    /// A right curly brace: `}`.
     RightBrace,
-    /// A left parenthesis: `(`.
+    /// A left round parenthesis: `(`.
     LeftParen,
-    /// A right parenthesis: `)`.
+    /// A right round parenthesis: `)`.
     RightParen,
-
-    /// A star: `*`.
+    /// An asterisk: `*`.
     Star,
     /// An underscore: `_`.
     Underscore,
+    /// A hashtag: `#`.
+    Hash,
     /// A tilde: `~`.
     Tilde,
-    /// A backslash followed by whitespace: `\`.
+    /// A backslash followed by nothing or whitespace: `\`.
     Backslash,
-    /// A hashtag indicating a section heading: `#`.
-    Hashtag,
-    /// A raw block: `` `...` ``.
-    Raw(TokenRaw<'s>),
-    /// A unicode escape sequence: `\u{1F5FA}`.
-    UnicodeEscape(TokenUnicodeEscape<'s>),
-
-    /// A colon: `:`.
-    Colon,
     /// A comma: `,`.
     Comma,
+    /// A colon: `:`.
+    Colon,
     /// A pipe: `|`.
     Pipe,
     /// A plus: `+`.
     Plus,
     /// A hyphen: `-`.
-    Hyphen,
+    Hyph,
     /// A slash: `/`.
     Slash,
-
+    /// A single equals sign: `=`.
+    Eq,
+    /// Two equals signs: `==`.
+    EqEq,
+    /// An exclamation mark followed by an equals sign: `!=`.
+    BangEq,
+    /// A less-than sign: `<`.
+    Lt,
+    /// A less-than sign followed by an equals sign: `<=`.
+    LtEq,
+    /// A greater-than sign: `>`.
+    Gt,
+    /// A greater-than sign followed by an equals sign: `>=`.
+    GtEq,
+    /// A plus followed by an equals sign: `+=`.
+    PlusEq,
+    /// A hyphen followed by an equals sign: `-=`.
+    HyphEq,
+    /// An asterisk followed by an equals sign: `*=`.
+    StarEq,
+    /// A slash followed by an equals sign: `/=`.
+    SlashEq,
+    /// A question mark: `?`.
+    Question,
+    /// Two dots: `..`.
+    Dots,
+    /// An equals sign followed by a greater-than sign: `=>`.
+    Arrow,
+    /// The `not` operator.
+    Not,
+    /// The `and` operator.
+    And,
+    /// The `or` operator.
+    Or,
+    /// The `let` / `#let` keyword.
+    Let,
+    /// The `if` / `#if` keyword.
+    If,
+    /// The `else` / `#else` keyword.
+    Else,
+    /// The `for` / `#for` keyword.
+    For,
+    /// The `in` / `#in` keyword.
+    In,
+    /// The `while` / `#while` keyword.
+    While,
+    /// The `break` / `#break` keyword.
+    Break,
+    /// The `continue` / `#continue` keyword.
+    Continue,
+    /// The `return` / `#return` keyword.
+    Return,
+    /// The none literal: `none`.
+    None,
+    /// One or more whitespace characters.
+    ///
+    /// The contained `usize` denotes the number of newlines that were contained
+    /// in the whitespace.
+    Space(usize),
+    /// A consecutive non-markup string.
+    Text(&'s str),
+    /// An arbitrary number of backticks followed by inner contents, terminated
+    /// with the same number of backticks: `` `...` ``.
+    Raw(TokenRaw<'s>),
+    /// One or two dollar signs followed by inner contents, terminated with the
+    /// same number of dollar signs.
+    Math(TokenMath<'s>),
+    /// A slash and the letter "u" followed by a hexadecimal unicode entity
+    /// enclosed in curly braces: `\u{1F5FA}`.
+    UnicodeEscape(TokenUnicodeEscape<'s>),
     /// An identifier: `center`.
     Ident(&'s str),
-    /// A none: `none`.
-    None,
     /// A boolean: `true`, `false`.
     Bool(bool),
     /// An integer: `120`.
@@ -76,11 +122,20 @@ pub enum Token<'s> {
     /// _Note_: `50%` is stored as `50.0` here, as in the corresponding
     /// [literal](super::Expr::Percent).
     Percent(f64),
+    /// An angle: `90deg`.
+    Angle(f64, AngularUnit),
     /// A hex value: `#20d82a`.
     Hex(&'s str),
     /// A quoted string: `"..."`.
     Str(TokenStr<'s>),
-
+    /// Two slashes followed by inner contents, terminated with a newline:
+    /// `//<str>\n`.
+    LineComment(&'s str),
+    /// A slash and a star followed by inner contents,  terminated with a star
+    /// and a slash: `/*<str>*/`.
+    ///
+    /// The comment can contain nested block comments.
+    BlockComment(&'s str),
     /// Things that are not valid tokens.
     Invalid(&'s str),
 }
@@ -98,15 +153,6 @@ pub struct TokenStr<'s> {
     pub terminated: bool,
 }
 
-/// A unicode escape sequence: `\u{1F5FA}`.
-#[derive(Debug, Copy, Clone, PartialEq)]
-pub struct TokenUnicodeEscape<'s> {
-    /// The escape sequence between two braces.
-    pub sequence: &'s str,
-    /// Whether the closing brace was present.
-    pub terminated: bool,
-}
-
 /// A raw block: `` `...` ``.
 #[derive(Debug, Copy, Clone, PartialEq)]
 pub struct TokenRaw<'s> {
@@ -118,48 +164,91 @@ pub struct TokenRaw<'s> {
     pub terminated: bool,
 }
 
+/// A math formula: `$2pi + x$`, `$$f'(x) = x^2$$`.
+#[derive(Debug, Copy, Clone, PartialEq)]
+pub struct TokenMath<'s> {
+    /// The formula between the dollars.
+    pub formula: &'s str,
+    /// Whether the formula was surrounded by one dollar (true) or two dollars
+    /// (false).
+    pub inline: bool,
+    /// Whether the closing dollars were present.
+    pub terminated: bool,
+}
+
+/// A unicode escape sequence: `\u{1F5FA}`.
+#[derive(Debug, Copy, Clone, PartialEq)]
+pub struct TokenUnicodeEscape<'s> {
+    /// The escape sequence between the braces.
+    pub sequence: &'s str,
+    /// Whether the closing brace was present.
+    pub terminated: bool,
+}
+
 impl<'s> Token<'s> {
-    /// The natural-language name of this token for use in error messages.
+    /// The English name of this token for use in error messages.
     pub fn name(self) -> &'static str {
         match self {
-            Self::Text(_) => "text",
-            Self::Space(_) => "space",
-
-            Self::LineComment(_) => "line comment",
-            Self::BlockComment(_) => "block comment",
-
             Self::LeftBracket => "opening bracket",
             Self::RightBracket => "closing bracket",
             Self::LeftBrace => "opening brace",
             Self::RightBrace => "closing brace",
             Self::LeftParen => "opening paren",
             Self::RightParen => "closing paren",
-
             Self::Star => "star",
             Self::Underscore => "underscore",
+            Self::Hash => "hashtag",
+            Self::Tilde => "tilde",
             Self::Backslash => "backslash",
-            Self::Hashtag => "hashtag",
-            Self::Tilde => "tidle",
-            Self::Raw { .. } => "raw block",
-            Self::UnicodeEscape { .. } => "unicode escape sequence",
-
-            Self::Colon => "colon",
             Self::Comma => "comma",
+            Self::Colon => "colon",
             Self::Pipe => "pipe",
-            Self::Plus => "plus sign",
-            Self::Hyphen => "minus sign",
+            Self::Plus => "plus",
+            Self::Hyph => "minus",
             Self::Slash => "slash",
-
+            Self::Eq => "assignment operator",
+            Self::EqEq => "equality operator",
+            Self::BangEq => "inequality operator",
+            Self::Lt => "less than operator",
+            Self::LtEq => "less than or equal operator",
+            Self::Gt => "greater than operator",
+            Self::GtEq => "greater than or equal operator",
+            Self::PlusEq => "add-assign operator",
+            Self::HyphEq => "subtract-assign operator",
+            Self::StarEq => "multiply-assign operator",
+            Self::SlashEq => "divide-assign operator",
+            Self::Question => "question mark",
+            Self::Dots => "dots",
+            Self::Arrow => "arrow",
+            Self::Not => "not operator",
+            Self::And => "and operator",
+            Self::Or => "or operator",
+            Self::Let => "let keyword",
+            Self::If => "if keyword",
+            Self::Else => "else keyword",
+            Self::For => "for keyword",
+            Self::In => "in keyword",
+            Self::While => "while keyword",
+            Self::Break => "break keyword",
+            Self::Continue => "continue keyword",
+            Self::Return => "return keyword",
             Self::None => "none",
+            Self::Space(_) => "space",
+            Self::Text(_) => "text",
+            Self::Raw(_) => "raw block",
+            Self::Math(_) => "math formula",
+            Self::UnicodeEscape(_) => "unicode escape sequence",
             Self::Ident(_) => "identifier",
-            Self::Bool(_) => "bool",
+            Self::Bool(_) => "boolean",
             Self::Int(_) => "integer",
             Self::Float(_) => "float",
             Self::Length(..) => "length",
+            Self::Angle(..) => "angle",
             Self::Percent(_) => "percentage",
             Self::Hex(_) => "hex value",
-            Self::Str { .. } => "string",
-
+            Self::Str(_) => "string",
+            Self::LineComment(_) => "line comment",
+            Self::BlockComment(_) => "block comment",
             Self::Invalid("*/") => "end of block comment",
             Self::Invalid(_) => "invalid token",
         }