Port tests 🚁

5 files changed, 450 insertions, 5 deletions

diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs
index 7f4052ab..cfa4c2e5 100644
--- a/src/syntax/mod.rs
+++ b/src/syntax/mod.rs
@@ -11,11 +11,13 @@ use self::span::{Spanned, SpanVec};
 pub mod expr;
 pub mod func;
 pub mod span;
-
 pub_use_mod!(scope);
 pub_use_mod!(parsing);
 pub_use_mod!(tokens);
 
+#[cfg(test)]
+mod test;
+
 
 /// Represents a parsed piece of source that can be layouted and in the future
 /// also be queried for information used for refactorings, autocomplete, etc.
diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs
index a7f39640..03866c2c 100644
--- a/src/syntax/parsing.rs
+++ b/src/syntax/parsing.rs
@@ -397,3 +397,193 @@ impl<'s> FuncParser<'s> {
             .unwrap_or_else(|| self.tokens.pos())
     }
 }
+
+
+#[cfg(test)]
+#[allow(non_snake_case)]
+mod tests {
+    use crate::size::Size;
+    use super::super::test::{DebugFn, SpanlessEq};
+    use super::*;
+
+    use Node::{
+        Space as S, Newline as N,
+        ToggleItalic as Italic, ToggleBolder as Bold, ToggleMonospace as Mono,
+    };
+
+    pub use Expr::{Number as Num, Bool};
+    pub fn Id(text: &str) -> Expr { Expr::Ident(Ident(text.to_string())) }
+    pub fn Str(text: &str) -> Expr { Expr::Str(text.to_string()) }
+
+    fn T(text: &str) -> Node { Node::Text(text.to_string()) }
+
+    /// Test whether the given string parses into the given node list.
+    macro_rules! p {
+        ($s:expr => [$($b:tt)*]) => {
+            let ctx = ParseContext { scope: &scope() };
+            let model = parse(Position::ZERO, $s, ctx).output;
+            let (expected, cmp) = model!([$($b)*]);
+
+            if !cmp(&model, &expected) {
+                fail($s, model, expected);
+            }
+        };
+    }
+
+    /// Test whether the given string yields the given parse errors.
+    macro_rules! e {
+        ($s:expr => [$(($sl:tt:$sc:tt, $el:tt:$ec:tt, $e:expr)),* $(,)?]) => {
+            let ctx = ParseContext { scope: &scope() };
+            let errors = parse(Position::ZERO, $s, ctx).errors
+                .into_iter()
+                .map(|s| s.map(|e| e.message))
+                .collect::<Vec<_>>();
+
+            let expected = vec![
+                $(Spanned {
+                    v: $e.to_string(),
+                    span: Span {
+                        start: Position { line: $sl, column: $sc },
+                        end:   Position { line: $el, column: $ec },
+                    },
+                }),*
+            ];
+
+            if errors != expected {
+                fail($s, errors, expected);
+            }
+        };
+    }
+
+    fn scope() -> Scope {
+        let mut scope = Scope::new::<DebugFn>();
+        scope.add::<DebugFn>("f");
+        scope.add::<DebugFn>("box");
+        scope
+    }
+
+    fn fail(src: &str, found: impl Debug, expected: impl Debug) {
+        eprintln!("source:   {:?}", src);
+        eprintln!("found:    {:#?}", found);
+        eprintln!("expected: {:#?}", expected);
+        panic!("test failed");
+    }
+
+    /// Parse a list of optionally spanned nodes into a syntax model.
+    macro_rules! model {
+        ([$(($sl:tt:$sc:tt, $el:tt:$ec:tt, $n:expr)),* $(,)?]) => ((SyntaxModel {
+            nodes: vec![
+                $(Spanned { v: $n, span: Span {
+                    start: Position { line: $sl, column: $sc },
+                    end:   Position { line: $el, column: $ec },
+                }}),*
+            ]
+        }, <SyntaxModel as PartialEq>::eq));
+
+        ([$($e:tt)*]) => ((SyntaxModel {
+            nodes: vec![$($e)*].into_iter().map(zspan).collect::<Vec<_>>()
+        }, <SyntaxModel as SpanlessEq>::spanless_eq));
+    }
+
+    /// Build a `DebugFn` function model.
+    macro_rules! func {
+        ($name:expr
+         $(,pos: [$($item:expr),* $(,)?])?
+         $(,key: [$($key:expr => $value:expr),* $(,)?])?;
+         $($b:tt)*) => ({
+            #![allow(unused_mut, unused_assignments)]
+
+            let mut pos = Tuple::new();
+            let mut key = Object::new();
+            $(pos = Tuple { items: vec![$(zspan($item)),*] };)?
+            $(key = Object {
+                pairs: vec![$(Pair {
+                    key: zspan(Ident($key.to_string())),
+                    value: zspan($value),
+                }),*]
+            };)?
+
+            Node::Model(Box::new(DebugFn {
+                header: FuncHeader {
+                    name: zspan(Ident($name.to_string())),
+                    args: FuncArgs {
+                        pos,
+                        key,
+                    },
+                },
+                body: func!(@body $($b)*),
+            }))
+        });
+
+        (@body Some([$($b:tt)*])) => (Some(model!([$($b)*]).0));
+        (@body None) => (None);
+    }
+
+    /// Span an element with a zero span.
+    fn zspan<T>(v: T) -> Spanned<T> {
+        Spanned { v, span: Span::ZERO }
+    }
+
+    #[test]
+    fn parse_flat_nodes() {
+        p!(""           => []);
+        p!("hi"         => [T("hi")]);
+        p!("*hi"        => [Bold, T("hi")]);
+        p!("hi_"        => [T("hi"), Italic]);
+        p!("`py`"       => [Mono, T("py"), Mono]);
+        p!("hi you"     => [T("hi"), S, T("you")]);
+        p!("💜\n\n 🌍"  => [T("💜"), N, T("🌍")]);
+    }
+
+    #[test]
+    fn parse_functions() {
+        p!("[func]" => [func!("func"; None)]);
+        p!("[tree][hi *you*]" => [func!("tree"; Some([T("hi"), S, Bold, T("you"), Bold]))]);
+        p!("[f: , hi, * \"du\"]" => [func!("f", pos: [Id("hi"), Str("du")]; None)]);
+        p!("from [align: left] to" => [
+            T("from"), S, func!("align", pos: [Id("left")]; None), S, T("to")
+        ]);
+
+        p!("[f: left, 12pt, false]" => [
+            func!("f", pos: [Id("left"), Expr::Size(Size::pt(12.0)), Bool(false)]; None)
+        ]);
+
+        p!("[box: x=1.2pt, false][a b c] bye" => [
+            func!(
+                "box",
+                pos: [Bool(false)],
+                key: ["x" => Expr::Size(Size::pt(1.2))];
+                Some([T("a"), S, T("b"), S, T("c")])
+            ),
+            S, T("bye"),
+        ]);
+    }
+
+    #[test]
+    fn parse_spanned() {
+        p!("hi you" => [(0:0, 0:2, T("hi")), (0:2, 0:3, S), (0:3, 0:6, T("you"))]);
+    }
+
+    #[test]
+    fn parse_errors() {
+        e!("[f: , hi, * \"du\"]" => [
+            (0:4,  0:5,  "expected value, found comma"),
+            (0:10, 0:11, "expected value, found invalid token"),
+        ]);
+        e!("[f:, , ,]" => [
+            (0:3, 0:4, "expected value, found comma"),
+            (0:5, 0:6, "expected value, found comma"),
+            (0:7, 0:8, "expected value, found comma"),
+        ]);
+        e!("[f:" => [(0:3, 0:3, "expected closing bracket")]);
+        e!("[f: hi" => [(0:6, 0:6, "expected closing bracket")]);
+        e!("[f: hey   12pt]" => [(0:7, 0:7, "expected comma")]);
+        e!("[box: x=, false z=y=4" => [
+            (0:8,  0:9,  "expected value, found comma"),
+            (0:15, 0:15, "expected comma"),
+            (0:19, 0:19, "expected comma"),
+            (0:19, 0:20, "expected value, found equals sign"),
+            (0:21, 0:21, "expected closing bracket"),
+        ]);
+    }
+}
diff --git a/src/syntax/span.rs b/src/syntax/span.rs
index 7a051d99..8973ef89 100644
--- a/src/syntax/span.rs
+++ b/src/syntax/span.rs
@@ -116,7 +116,7 @@ impl Debug for Span {
 }
 
 /// A value with the span it corresponds to in the source code.
-#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash, Serialize)]
+#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)]
 pub struct Spanned<T> {
     /// The value.
     pub v: T,
@@ -147,6 +147,14 @@ impl<T> Spanned<T> {
     }
 }
 
+impl<T: Debug> Debug for Spanned<T> {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        self.span.fmt(f)?;
+        f.write_str(": ")?;
+        self.v.fmt(f)
+    }
+}
+
 /// A vector of spanned things.
 pub type SpanVec<T> = Vec<Spanned<T>>;
 
diff --git a/src/syntax/test.rs b/src/syntax/test.rs
new file mode 100644
index 00000000..e37e8cf5
--- /dev/null
+++ b/src/syntax/test.rs
@@ -0,0 +1,88 @@
+use super::func::FuncHeader;
+use super::expr::{Expr, Tuple, Object};
+use super::*;
+
+
+function! {
+    /// Most functions in the tests are parsed into the debug function for easy
+    /// inspection of arguments and body.
+    #[derive(Debug, Clone, PartialEq)]
+    pub struct DebugFn {
+        pub header: FuncHeader,
+        pub body: Option<SyntaxModel>,
+    }
+
+    parse(header, body, ctx, errors, decos) {
+        let cloned = header.clone();
+        header.args.pos.items.clear();
+        header.args.key.pairs.clear();
+        DebugFn {
+            header: cloned,
+            body: body!(opt: body, ctx, errors, decos),
+        }
+    }
+
+    layout(self, ctx, errors) { vec![] }
+}
+
+/// Compares elements by only looking at values and ignoring spans.
+pub trait SpanlessEq<Rhs=Self> {
+    fn spanless_eq(&self, other: &Rhs) -> bool;
+}
+
+impl SpanlessEq for Vec<Spanned<Token<'_>>> {
+    fn spanless_eq(&self, other: &Vec<Spanned<Token>>) -> bool {
+        self.len() == other.len()
+        && self.iter().zip(other).all(|(x, y)| x.v == y.v)
+    }
+}
+
+impl SpanlessEq for SyntaxModel {
+    fn spanless_eq(&self, other: &SyntaxModel) -> bool {
+        fn downcast<'a>(func: &'a (dyn Model + 'static)) -> &'a DebugFn {
+            func.downcast::<DebugFn>().expect("not a debug fn")
+        }
+
+        self.nodes.len() == other.nodes.len()
+        && self.nodes.iter().zip(&other.nodes).all(|(x, y)| match (&x.v, &y.v) {
+            (Node::Model(a), Node::Model(b)) => {
+                downcast(a.as_ref()).spanless_eq(downcast(b.as_ref()))
+            }
+            (a, b) => a == b,
+        })
+    }
+}
+
+impl SpanlessEq for DebugFn {
+    fn spanless_eq(&self, other: &DebugFn) -> bool {
+        self.header.name.v == other.header.name.v
+        && self.header.args.pos.spanless_eq(&other.header.args.pos)
+        && self.header.args.key.spanless_eq(&other.header.args.key)
+    }
+}
+
+impl SpanlessEq for Expr {
+    fn spanless_eq(&self, other: &Expr) -> bool {
+        match (self, other) {
+            (Expr::Tuple(a), Expr::Tuple(b)) => a.spanless_eq(b),
+            (Expr::Object(a), Expr::Object(b)) => a.spanless_eq(b),
+            (a, b) => a == b,
+        }
+    }
+}
+
+impl SpanlessEq for Tuple {
+    fn spanless_eq(&self, other: &Tuple) -> bool {
+        self.items.len() == other.items.len()
+        && self.items.iter().zip(&other.items)
+            .all(|(x, y)| x.v.spanless_eq(&y.v))
+    }
+}
+
+impl SpanlessEq for Object {
+    fn spanless_eq(&self, other: &Object) -> bool {
+        self.pairs.len() == other.pairs.len()
+        && self.pairs.iter().zip(&other.pairs)
+            .all(|(x, y)| x.key.v == y.key.v && x.value.v.spanless_eq(&y.value.v))
+    }
+}
diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs
index 747b6b93..7b52f655 100644
--- a/src/syntax/tokens.rs
+++ b/src/syntax/tokens.rs
@@ -341,12 +341,17 @@ impl<'s> Tokens<'s> {
         }
 
         let c = self.peek().unwrap_or('n');
-        if self.mode == Body && is_escapable(c) {
+        let string = if is_escapable(c) {
             let index = self.index();
             self.eat();
-            Text(&self.src[index .. index + c.len_utf8()])
+            &self.src[index .. index + c.len_utf8()]
         } else {
-            Text("\\")
+            "\\"
+        };
+
+        match self.mode {
+            Header => Invalid(string),
+            Body => Text(string),
         }
     }
 
@@ -456,3 +461,155 @@ pub fn is_identifier(string: &str) -> bool {
 
     true
 }
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use Token::{
+        Space as S,
+        LineComment as LC, BlockComment as BC,
+        LeftParen as LP, RightParen as RP,
+        LeftBrace as LB, RightBrace as RB,
+        ExprIdent as Id, ExprNumber as Num, ExprBool as Bool,
+        Text as T,
+    };
+
+    #[allow(non_snake_case)]
+    fn Str(string: &'static str, terminated: bool) -> Token<'static> {
+        Token::ExprStr { string, terminated }
+    }
+
+    /// Test whether the given string tokenizes into the given list of tokens.
+    macro_rules! t {
+        ($m:expr, $s:expr => [$(($sl:tt:$sc:tt, $el:tt:$ec:tt, $t:expr)),* $(,)?]) => {
+            let tokens = Tokens::new(Position::ZERO, $s, $m).collect::<Vec<_>>();
+            assert_eq!(tokens, vec![$(Spanned {
+                span: Span::new(Position::new($sl, $sc), Position::new($el, $ec)),
+                v: $t
+            }),*]);
+
+        };
+
+        ($m:expr, $s:expr => [$($t:expr),* $(,)?]) => {
+            let tokens = Tokens::new(Position::ZERO, $s, $m)
+                .map(Spanned::value)
+                .collect::<Vec<_>>();
+            assert_eq!(tokens, vec![$($t),*]);
+        };
+    }
+
+    /// Parse a function token.
+    macro_rules! func {
+        ($header:expr, Some(($sl:tt:$sc:tt, $el:tt:$ec:tt, $body:expr)), $terminated:expr) => {
+            Function {
+                header: $header,
+                body: Some(Spanned {
+                    span: Span::new(Position::new($sl, $sc), Position::new($el, $ec)),
+                    v: $body,
+                }),
+                terminated: $terminated,
+            }
+        };
+        ($header:expr, None, $terminated:expr) => {
+            Function { header: $header, body: None, terminated: $terminated }
+        }
+    }
+
+    #[test]
+    fn tokenize_whitespace() {
+        t!(Body, ""             => []);
+        t!(Body, " "            => [S(0)]);
+        t!(Body, "    "         => [S(0)]);
+        t!(Body, "\t"           => [S(0)]);
+        t!(Body, "  \t"         => [S(0)]);
+        t!(Body, "\n"           => [S(1)]);
+        t!(Body, "\n "          => [S(1)]);
+        t!(Body, "  \n"         => [S(1)]);
+        t!(Body, "  \n   "      => [S(1)]);
+        t!(Body, "\r\n"         => [S(1)]);
+        t!(Body, "  \n\t \n  "  => [S(2)]);
+        t!(Body, "\n\r"         => [S(2)]);
+        t!(Body, " \r\r\n \x0D" => [S(3)]);
+    }
+
+    #[test]
+    fn tokenize_comments() {
+        t!(Body, "a // bc\n "        => [T("a"), S(0), LC(" bc"),  S(1)]);
+        t!(Body, "a //a//b\n "       => [T("a"), S(0), LC("a//b"), S(1)]);
+        t!(Body, "a //a//b\r\n"      => [T("a"), S(0), LC("a//b"), S(1)]);
+        t!(Body, "a //a//b\n\nhello" => [T("a"), S(0), LC("a//b"), S(2), T("hello")]);
+        t!(Body, "/**/"              => [BC("")]);
+        t!(Body, "_/*_/*a*/*/"       => [Underscore, BC("_/*a*/")]);
+        t!(Body, "/*/*/"             => [BC("/*/")]);
+        t!(Body, "abc*/"             => [T("abc"), Invalid("*/")]);
+    }
+
+    #[test]
+    fn tokenize_header_only_tokens() {
+        t!(Body, "\"hi\""              => [T("\"hi"), T("\"")]);
+        t!(Body, "a: b"                => [T("a"), T(":"), S(0), T("b")]);
+        t!(Body, "c=d, "               => [T("c"), T("=d"), T(","), S(0)]);
+        t!(Header, "["                 => [func!("", None, false)]);
+        t!(Header, "]"                 => [Invalid("]")]);
+        t!(Header, "(){}:=,"           => [LP, RP, LB, RB, Colon, Equals, Comma]);
+        t!(Header, "a:b"               => [Id("a"), Colon, Id("b")]);
+        t!(Header, "="                 => [Equals]);
+        t!(Header, ","                 => [Comma]);
+        t!(Header, r#""hello\"world""# => [Str(r#"hello\"world"#, true)]);
+        t!(Header, r#""hi", 12pt"#     => [Str("hi", true), Comma, S(0), ExprSize(Size::pt(12.0))]);
+        t!(Header, "a: true, x=1"      => [Id("a"), Colon, S(0), Bool(true), Comma, S(0), Id("x"), Equals, Num(1.0)]);
+        t!(Header, "120%"              => [Num(1.2)]);
+        t!(Header, "🌓, 🌍,"           => [Invalid("🌓"), Comma, S(0), Invalid("🌍"), Comma]);
+    }
+
+    #[test]
+    fn tokenize_body_only_tokens() {
+        t!(Body, "_*`"           => [Underscore, Star, Backtick]);
+        t!(Body, "[func]*bold*"  => [func!("func", None, true), Star, T("bold"), Star]);
+        t!(Body, "hi_you_ there" => [T("hi"), Underscore, T("you"), Underscore, S(0), T("there")]);
+        t!(Header, "_*`"         => [Invalid("_"), Invalid("*"), Invalid("`")]);
+    }
+
+    #[test]
+    fn tokenize_nested_functions() {
+        t!(Body, "[f: [=][*]]"    => [func!("f: [=][*]", None, true)]);
+        t!(Body, "[_][[,],],"     => [func!("_", Some((0:3, 0:9, "[,],")), true), T(",")]);
+        t!(Body, "[=][=][=]"      => [func!("=", Some((0:3, 0:6, "=")), true), func!("=", None, true)]);
+        t!(Body, "[=][[=][=][=]]" => [func!("=", Some((0:3, 0:14, "[=][=][=]")), true)]);
+    }
+
+    #[test]
+    fn tokenize_escaped_symbols() {
+        t!(Body, r"\\" => [T(r"\")]);
+        t!(Body, r"\[" => [T("[")]);
+        t!(Body, r"\]" => [T("]")]);
+        t!(Body, r"\*" => [T("*")]);
+        t!(Body, r"\_" => [T("_")]);
+        t!(Body, r"\`" => [T("`")]);
+        t!(Body, r"\/" => [T("/")]);
+    }
+
+    #[test]
+    fn tokenize_unescapable_symbols() {
+        t!(Body, r"\a"     => [T("\\"), T("a")]);
+        t!(Body, r"\:"     => [T(r"\"), T(":")]);
+        t!(Body, r"\="     => [T(r"\"), T("=")]);
+        t!(Header, r"\\\\" => [Invalid("\\"), Invalid("\\")]);
+        t!(Header, r"\a"   => [Invalid("\\"), Id("a")]);
+        t!(Header, r"\:"   => [Invalid(r"\"), Colon]);
+        t!(Header, r"\="   => [Invalid(r"\"), Equals]);
+        t!(Header, r"\,"   => [Invalid(r"\"), Comma]);
+    }
+
+    #[test]
+    fn tokenize_with_spans() {
+        t!(Body, "hello"          => [(0:0, 0:5, T("hello"))]);
+        t!(Body, "ab\r\nc"        => [(0:0, 0:2, T("ab")), (0:2, 1:0, S(1)), (1:0, 1:1, T("c"))]);
+        t!(Body, "[x = \"(1)\"]*" => [(0:0, 0:11, func!("x = \"(1)\"", None, true)), (0:11, 0:12, Star)]);
+        t!(Body, "// ab\r\n\nf"   => [(0:0, 0:5, LC(" ab")), (0:5, 2:0, S(2)), (2:0, 2:1, T("f"))]);
+        t!(Body, "/*b*/_"         => [(0:0, 0:5, BC("b")), (0:5, 0:6, Underscore)]);
+        t!(Header, "a=10"         => [(0:0, 0:1, Id("a")), (0:1, 0:2, Equals), (0:2, 0:4, Num(10.0))]);
+    }
+}