diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/syntax/expr.rs | 6 | ||||
| -rw-r--r-- | src/syntax/parsing.rs | 417 | ||||
| -rw-r--r-- | src/syntax/span.rs | 173 | ||||
| -rw-r--r-- | src/syntax/test.rs | 64 | ||||
| -rw-r--r-- | src/syntax/tokens.rs | 55 |
5 files changed, 332 insertions, 383 deletions
diff --git a/src/syntax/expr.rs b/src/syntax/expr.rs index a27bdb62..a1b3fd62 100644 --- a/src/syntax/expr.rs +++ b/src/syntax/expr.rs @@ -93,9 +93,6 @@ impl Debug for Expr { /// A unicode identifier. /// -/// The identifier must be valid! This is checked in [`Ident::new`] or -/// [`is_identifier`]. -/// /// # Example /// ```typst /// [func: "hi", ident] @@ -105,7 +102,8 @@ impl Debug for Expr { pub struct Ident(pub String); impl Ident { - /// Create a new identifier from a string checking that it is valid. + /// Create a new identifier from a string checking that it is a valid + /// unicode identifier. pub fn new<S>(ident: S) -> Option<Ident> where S: AsRef<str> + Into<String> { if is_identifier(ident.as_ref()) { Some(Ident(ident.into())) diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs index 09405d7f..89d4f0fd 100644 --- a/src/syntax/parsing.rs +++ b/src/syntax/parsing.rs @@ -639,84 +639,17 @@ fn unescape_raw(raw: &str) -> Vec<String> { #[allow(non_snake_case)] mod tests { use crate::size::Size; - use crate::syntax::test::{DebugFn, check, zspan}; + use crate::syntax::test::{DebugFn, check}; use crate::syntax::func::Value; use super::*; use Decoration::*; + use Expr::{Number as Num, Size as Sz, Bool}; use Node::{ Space as S, ToggleItalic as Italic, ToggleBolder as Bold, Parbreak, Linebreak, }; - use Expr::{Number as Num, Size as Sz, Bool}; - fn Id(text: &str) -> Expr { Expr::Ident(Ident(text.to_string())) } - fn Str(text: &str) -> Expr { Expr::Str(text.to_string()) } - fn Pt(points: f32) -> Expr { Expr::Size(Size::pt(points)) } - fn Neg(e1: Expr) -> Expr { Expr::Neg(Box::new(zspan(e1))) } - fn Add(e1: Expr, e2: Expr) -> Expr { - Expr::Add(Box::new(zspan(e1)), Box::new(zspan(e2))) - } - fn Sub(e1: Expr, e2: Expr) -> Expr { - Expr::Sub(Box::new(zspan(e1)), Box::new(zspan(e2))) - } - fn Mul(e1: Expr, e2: Expr) -> Expr { - Expr::Mul(Box::new(zspan(e1)), Box::new(zspan(e2))) - } - fn Div(e1: Expr, e2: Expr) -> Expr { - Expr::Div(Box::new(zspan(e1)), Box::new(zspan(e2))) - } - - fn Clr(r: u8, g: u8, b: u8, a: u8) -> Expr { - Expr::Color(RgbaColor::new(r, g, b, a)) - } - fn ClrStr(color: &str) -> Expr { - Expr::Color(RgbaColor::from_str(color).expect("invalid test color")) - } - fn ClrStrHealed() -> Expr { - let mut c = RgbaColor::from_str("000f").expect("invalid test color"); - c.healed = true; - Expr::Color(c) - } - - fn T(text: &str) -> Node { Node::Text(text.to_string()) } - - /// Create a raw text node. - macro_rules! raw { - ($($line:expr),* $(,)?) => { - Node::Raw(vec![$($line.to_string()),*]) - }; - } - - /// Create a tuple expression. - macro_rules! tuple { - ($($items:expr),* $(,)?) => { - Expr::Tuple(Tuple { items: spanned![vec $($items),*].0 }) - }; - } - - /// Create a named tuple expression. - macro_rules! named_tuple { - ($name:expr $(, $items:expr)* $(,)?) => { - Expr::NamedTuple(NamedTuple::new( - zspan(Ident($name.to_string())), - zspan(Tuple { items: spanned![vec $($items),*].0 }) - )) - }; - } - - /// Create an object expression. - macro_rules! object { - ($($key:expr => $value:expr),* $(,)?) => { - Expr::Object(Object { - pairs: vec![$(zspan(Pair { - key: zspan(Ident($key.to_string())), - value: zspan($value), - })),*] - }) - }; - } - /// Test whether the given string parses into /// - the given node list (required). /// - the given error list (optional, if omitted checks against empty list). @@ -736,12 +669,12 @@ mod tests { let ctx = ParseContext { scope: &scope }; let pass = parse(Position::ZERO, $source, ctx); - // Test model - let (exp, cmp) = spanned![vec $($model)*]; + // Test model. + let (exp, cmp) = span_vec![$($model)*]; check($source, exp, pass.output.nodes, cmp); - // Test problems - let (exp, cmp) = spanned![vec $($problems)*]; + // Test problems. + let (exp, cmp) = span_vec![$($problems)*]; let exp = exp.into_iter() .map(|s: Spanned<&str>| s.map(|e| e.to_string())) .collect::<Vec<_>>(); @@ -750,42 +683,90 @@ mod tests { .collect::<Vec<_>>(); check($source, exp, found, cmp); - // Test decos - $(let (exp, cmp) = spanned![vec $($decos)*]; + // Test decos. + $(let (exp, cmp) = span_vec![$($decos)*]; check($source, exp, pass.feedback.decos, cmp);)? }; } - /// Write down a `DebugFn` function model compactly. + /// Shorthand for `p!("[val: ...]" => func!("val", ...))`. + macro_rules! pval { + ($header:expr => $($tts:tt)*) => { + p!(concat!("[val: ", $header, "]") => [func!("val": $($tts)*)]); + } + } + + fn Id(text: &str) -> Expr { Expr::Ident(Ident(text.to_string())) } + fn Str(text: &str) -> Expr { Expr::Str(text.to_string()) } + fn Pt(points: f32) -> Expr { Expr::Size(Size::pt(points)) } + fn Color(r: u8, g: u8, b: u8, a: u8) -> Expr { Expr::Color(RgbaColor::new(r, g, b, a)) } + fn ColorStr(color: &str) -> Expr { Expr::Color(RgbaColor::from_str(color).expect("invalid test color")) } + fn ColorHealed() -> Expr { Expr::Color(RgbaColor::new_healed(0, 0, 0, 255)) } + fn Neg(e1: Expr) -> Expr { Expr::Neg(Box::new(Z(e1))) } + fn Add(e1: Expr, e2: Expr) -> Expr { Expr::Add(Box::new(Z(e1)), Box::new(Z(e2))) } + fn Sub(e1: Expr, e2: Expr) -> Expr { Expr::Sub(Box::new(Z(e1)), Box::new(Z(e2))) } + fn Mul(e1: Expr, e2: Expr) -> Expr { Expr::Mul(Box::new(Z(e1)), Box::new(Z(e2))) } + fn Div(e1: Expr, e2: Expr) -> Expr { Expr::Div(Box::new(Z(e1)), Box::new(Z(e2))) } + fn T(text: &str) -> Node { Node::Text(text.to_string()) } + fn Z<T>(v: T) -> Spanned<T> { Spanned::zero(v) } + + macro_rules! tuple { + ($($items:expr),* $(,)?) => { + Expr::Tuple(Tuple { items: span_vec![$($items),*].0 }) + }; + } + + macro_rules! named_tuple { + ($name:expr $(, $items:expr)* $(,)?) => { + Expr::NamedTuple(NamedTuple::new( + Z(Ident($name.to_string())), + Z(Tuple { items: span_vec![$($items),*].0 }) + )) + }; + } + + macro_rules! object { + ($($key:expr => $value:expr),* $(,)?) => { + Expr::Object(Object { + pairs: vec![$(Z(Pair { + key: Z(Ident($key.to_string())), + value: Z($value), + })),*] + }) + }; + } + + macro_rules! raw { + ($($line:expr),* $(,)?) => { + Node::Raw(vec![$($line.to_string()),*]) + }; + } + macro_rules! func { - ($name:tt $(: ($($pos:tt)*), { $($key:tt)* } )? $(; $($body:tt)*)?) => ({ + ($name:tt $(: ($($pos:tt)*) $(, { $($key:tt)* })? )? $(; $($body:tt)*)?) => {{ #[allow(unused_mut)] let mut args = FuncArgs::new(); - $(args.pos = Tuple::parse(zspan(tuple!($($pos)*))).unwrap();)? - $(args.key = Object::parse(zspan(object! { $($key)* })).unwrap();)? - + $(args.pos = Tuple::parse(Z(tuple!($($pos)*))).unwrap(); + $(args.key = Object::parse(Z(object! { $($key)* })).unwrap();)?)? Node::Model(Box::new(DebugFn { header: FuncHeader { - name: spanned!(item $name).map(|s| Ident(s.to_string())), + name: span_item!($name).map(|s| Ident(s.to_string())), args, }, body: func!(@body $($($body)*)?), })) - }); - - (@body [$($body:tt)*]) => ({ - Some(SyntaxModel { nodes: spanned![vec $($body)*].0 }) - }); - (@body) => (None); + }}; + (@body [$($body:tt)*]) => { Some(SyntaxModel { nodes: span_vec![$($body)*].0 }) }; + (@body) => { None }; } #[test] fn parse_color_strings() { - assert_eq!(Clr(0xf6, 0x12, 0x43, 0xff), ClrStr("f61243ff")); - assert_eq!(Clr(0xb3, 0xd8, 0xb3, 0xff), ClrStr("b3d8b3")); - assert_eq!(Clr(0xfc, 0xd2, 0xa9, 0xad), ClrStr("fCd2a9AD")); - assert_eq!(Clr(0x22, 0x33, 0x33, 0xff), ClrStr("233")); - assert_eq!(Clr(0x11, 0x11, 0x11, 0xbb), ClrStr("111b")); + assert_eq!(Color(0xf6, 0x12, 0x43, 0xff), ColorStr("f61243ff")); + assert_eq!(Color(0xb3, 0xd8, 0xb3, 0xff), ColorStr("b3d8b3")); + assert_eq!(Color(0xfc, 0xd2, 0xa9, 0xad), ColorStr("fCd2a9AD")); + assert_eq!(Color(0x22, 0x33, 0x33, 0xff), ColorStr("233")); + assert_eq!(Color(0x11, 0x11, 0x11, 0xbb), ColorStr("111b")); } #[test] @@ -824,7 +805,7 @@ mod tests { #[test] fn parse_basic_nodes() { - // Basic nodes + // Basic nodes. p!("" => []); p!("hi" => [T("hi")]); p!("*hi" => [Bold, T("hi")]); @@ -838,13 +819,13 @@ mod tests { p!(r"a\ b" => [T("a"), Linebreak, S, T("b")]); p!("π\n\n π" => [T("π"), Parbreak, T("π")]); - // Raw markup + // Raw markup. p!("`py`" => [raw!["py"]]); p!("[val][`hi]`]" => [func!("val"; [raw!["hi]"]])]); p!("`hi\nyou" => [raw!["hi", "you"]], [(1:3, 1:3, "expected backtick")]); p!("`hi\\`du`" => [raw!["hi`du"]]); - // Spanned nodes + // Spanned nodes. p!("Hi" => [(0:0, 0:2, T("Hi"))]); p!("*Hi*" => [(0:0, 0:1, Bold), (0:1, 0:3, T("Hi")), (0:3, 0:4, Bold)]); p!("π\n*/[n]" => @@ -856,31 +837,31 @@ mod tests { #[test] fn parse_function_names() { - // No closing bracket + // No closing bracket. p!("[" => [func!("")], [ (0:1, 0:1, "expected identifier"), (0:1, 0:1, "expected closing bracket") ]); - // No name + // No name. p!("[]" => [func!("")], [(0:1, 0:1, "expected identifier")]); p!("[\"]" => [func!("")], [ (0:1, 0:3, "expected identifier, found string"), (0:3, 0:3, "expected closing bracket"), ]); - // An unknown name + // An unknown name. p!("[hi]" => [func!("hi")], [(0:1, 0:3, "unknown function")], [(0:1, 0:3, InvalidFuncName)], ); - // A valid name + // A valid name. p!("[f]" => [func!("f")], [], [(0:1, 0:2, ValidFuncName)]); p!("[ f]" => [func!("f")], [], [(0:3, 0:4, ValidFuncName)]); - // An invalid token for a name + // An invalid token for a name. p!("[12]" => [func!("")], [(0:1, 0:3, "expected identifier, found number")], []); p!("[π]" => [func!("")], [(0:1, 0:2, "expected identifier, found invalid token")], []); p!("[ π]" => [func!("")], [(0:3, 0:4, "expected identifier, found invalid token")], []); @@ -888,13 +869,19 @@ mod tests { #[test] fn parse_colon_starting_function_arguments() { - // No colon before arg + // Valid. + p!("[val: true]" => + [func!["val": (Bool(true))]], [], + [(0:1, 0:4, ValidFuncName)], + ); + + // No colon before arg. p!("[val\"s\"]" => [func!("val")], [(0:4, 0:4, "expected colon")]); - // No colon before valid, but wrong token + // No colon before valid, but wrong token. p!("[val=]" => [func!("val")], [(0:4, 0:4, "expected colon")]); - // No colon before invalid tokens, which are ignored + // No colon before invalid tokens, which are ignored. p!("[val/π:$]" => [func!("val")], [(0:4, 0:4, "expected colon")], @@ -909,36 +896,38 @@ mod tests { (0:7, 0:7, "expected closing bracket"), ]); - // Just colon without args + // Just colon without args. p!("[val:]" => [func!("val")]); p!("[val:/*12pt*/]" => [func!("val")]); - // Whitespace / comments around colon - p!("[val\n:\ntrue]" => [func!("val": (Bool(true)), {})]); - p!("[val/*:*/://\ntrue]" => [func!("val": (Bool(true)), {})]); + // Whitespace / comments around colon. + p!("[val\n:\ntrue]" => [func!("val": (Bool(true)))]); + p!("[val/*:*/://\ntrue]" => [func!("val": (Bool(true)))]); } #[test] fn parse_one_positional_argument() { - // Different expressions - p!("[val: true]" => - [func!("val": (Bool(true)), {})], [], - [(0:1, 0:4, ValidFuncName)], - ); - p!("[val: _]" => [func!("val": (Id("_")), {})]); - p!("[val: name]" => [func!("val": (Id("name")), {})]); - p!("[val: \"hi\"]" => [func!("val": (Str("hi")), {})]); - p!("[val: \"a\n[]\\\"string\"]" => [func!("val": (Str("a\n[]\"string")), {})]); - p!("[val: 3.14]" => [func!("val": (Num(3.14)), {})]); - p!("[val: 4.5cm]" => [func!("val": (Sz(Size::cm(4.5))), {})]); - p!("[val: 12e1pt]" => [func!("val": (Pt(12e1)), {})]); - p!("[val: #f7a20500]" => [func!("val": (ClrStr("f7a20500")), {})]); - - // Math - p!("[val: 3.2in + 6pt]" => [func!("val": (Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0)))), {})]); - p!("[val: 5 - 0.01]" => [func!("val": (Sub(Num(5.0), Num(0.01))), {})]); - p!("[val: (3mm * 2)]" => [func!("val": (Mul(Sz(Size::mm(3.0)), Num(2.0))), {})]); - p!("[val: 12e-3cm/1pt]" => [func!("val": (Div(Sz(Size::cm(12e-3)), Sz(Size::pt(1.0)))), {})]); + // Different expressions. + pval!("_" => (Id("_"))); + pval!("name" => (Id("name"))); + pval!("\"hi\"" => (Str("hi"))); + pval!("3.14" => (Num(3.14))); + pval!("4.5cm" => (Sz(Size::cm(4.5)))); + pval!("12e1pt" => (Pt(12e1))); + pval!("#f7a20500" => (ColorStr("f7a20500"))); + pval!("\"a\n[]\\\"string\"" => (Str("a\n[]\"string"))); + + // Trailing comma. + pval!("a," => (Id("a"))); + + // Simple coerced tuple. + pval!("(hi)" => (Id("hi"))); + + // Math. + pval!("3.2in + 6pt" => (Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0))))); + pval!("5 - 0.01" => (Sub(Num(5.0), Num(0.01)))); + pval!("(3mm * 2)" => (Mul(Sz(Size::mm(3.0)), Num(2.0)))); + pval!("12e-3cm/1pt" => (Div(Sz(Size::cm(12e-3)), Sz(Size::pt(1.0))))); // Unclosed string. p!("[val: \"hello]" => [func!("val": (Str("hello]")), {})], [ @@ -946,112 +935,80 @@ mod tests { (0:13, 0:13, "expected closing bracket"), ]); - //Invalid colors - p!("[val: #12345]" => [func!("val": (ClrStrHealed()), {})], [ - (0:6, 0:12, "invalid color"), - ]); - p!("[val: #a5]" => [func!("val": (ClrStrHealed()), {})], [ - (0:6, 0:9, "invalid color"), - ]); - p!("[val: #14b2ah]" => [func!("val": (ClrStrHealed()), {})], [ - (0:6, 0:13, "invalid color"), - ]); - p!("[val: #f075ff011]" => [func!("val": (ClrStrHealed()), {})], [ - (0:6, 0:16, "invalid color"), - ]); + // Invalid, healed colors. + p!("[val: #12345]" => [func!("val": (ColorHealed()))], [(0:6, 0:12, "invalid color")]); + p!("[val: #a5]" => [func!("val": (ColorHealed()))], [(0:6, 0:9, "invalid color")]); + p!("[val: #14b2ah]" => [func!("val": (ColorHealed()))], [(0:6, 0:13, "invalid color")]); + p!("[val: #f075ff011]" => [func!("val": (ColorHealed()))], [(0:6, 0:16, "invalid color")]); } #[test] fn parse_complex_mathematical_expressions() { - p!("[val: (3.2in + 6pt)*(5/2-1)]" => [func!("val": ( - Mul( - Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0))), - Sub(Div(Num(5.0), Num(2.0)), Num(1.0)) - ) - ), {})]); - p!("[val: (6.3E+2+4* - 3.2pt)/2]" => [func!("val": ( - Div(Add(Num(6.3e2),Mul(Num(4.0), Neg(Pt(3.2)))), Num(2.0)) - ), {})]); - p!("[val: 4pt--]" => - [func!("val": (Pt(4.0)), {})], - [ - (0:10, 0:11, "dangling minus"), - (0:6, 0:10, "missing right summand") - ], - ); + // Valid expressions. + pval!("(3.2in + 6pt)*(5/2-1)" => (Mul( + Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0))), + Sub(Div(Num(5.0), Num(2.0)), Num(1.0)) + ))); + pval!("(6.3E+2+4* - 3.2pt)/2" => (Div( + Add(Num(6.3e2),Mul(Num(4.0), Neg(Pt(3.2)))), + Num(2.0) + ))); + + // Invalid expressions. + p!("[val: 4pt--]" => [func!("val": (Pt(4.0)))], [ + (0:10, 0:11, "dangling minus"), + (0:6, 0:10, "missing right summand") + ]); p!("[val: 3mm+4pt*]" => - [func!("val": (Add(Sz(Size::mm(3.0)), Pt(4.0))), {})], + [func!("val": (Add(Sz(Size::mm(3.0)), Pt(4.0))))], [(0:10, 0:14, "missing right factor")], ); } #[test] fn parse_tuples() { - // Empty tuple - p!("[val: ()]" => [func!("val": (tuple!()), {})]); - p!("[val: empty()]" => [func!("val": (named_tuple!("empty")), {})]); + // Empty tuple. + pval!("()" => (tuple!())); + pval!("empty()" => (named_tuple!("empty"))); - // Invalid value - p!("[val: (π)]" => - [func!("val": (tuple!()), {})], - [(0:7, 0:8, "expected value, found invalid token")], - ); + // Invalid value. p!("[val: sound(\x07)]" => [func!("val": (named_tuple!("sound")), {})], [(0:12, 0:13, "expected value, found invalid token")], ); - // Invalid tuple name + // Invalid tuple name. p!("[val: π (\"abc\", 13e-5)]" => [func!("val": (tuple!(Str("abc"), Num(13.0e-5))), {})], [(0:6, 0:7, "expected argument, found invalid token")], ); - // Unclosed tuple - p!("[val: (hello,]" => - [func!("val": (tuple!(Id("hello"),)), {})], - [(0:13, 0:13, "expected closing paren")], - ); + // Unclosed tuple. p!("[val: lang(δΈζ]" => [func!("val": (named_tuple!("lang", Id("δΈζ"))), {})], [(0:13, 0:13, "expected closing paren")], ); - // Valid values - p!("[val: (1, 2)]" => [func!("val": (tuple!(Num(1.0), Num(2.0))), {})]); - p!("[val: (\"s\",)]" => [func!("val": (tuple!(Str("s"))), {})]); - p!("[val: cmyk(1, 46, 0, 0)]" => - [func!("val": (named_tuple!( - "cmyk", Num(1.0), Num(46.0), Num(0.0), Num(0.0) - )), {})] - ); - p!("[val: items(\"fire\", #f93a6d)]" => - [func!("val": (named_tuple!( - "items", Str("fire"), ClrStr("f93a6d") - )), {})] - ); - - // Nested tuples - p!("[val: (1, (2, 3))]" => - [func!("val": (tuple!(Num(1.0), tuple!(Num(2.0), Num(3.0)))), {})] - ); - p!("[val: css(1pt, rgb(90, 102, 254), \"solid\")]" => - [func!("val": (named_tuple!( - "css", Pt(1.0), named_tuple!( - "rgb", Num(90.0), Num(102.0), Num(254.0) - ), Str("solid") - )), {})] - ); - - // Invalid commas + // Valid values. + pval!("(1, 2)" => (tuple!(Num(1.0), Num(2.0)))); + pval!("(\"s\",)" => (tuple!(Str("s")))); + pval!("items(\"fire\", #f93a6d)" => ( + named_tuple!("items", Str("fire"), ColorStr("f93a6d") + ))); + + // Nested tuples. + pval!("css(1pt, rgb(90, 102, 254), \"solid\")" => (named_tuple!( + "css", + Pt(1.0), + named_tuple!("rgb", Num(90.0), Num(102.0), Num(254.0)), + Str("solid"), + ))); + + // Invalid commas. p!("[val: (,)]" => [func!("val": (tuple!()), {})], [(0:7, 0:8, "expected value, found comma")], ); - p!("[val: nose(,)]" => - [func!("val": (named_tuple!("nose")), {})], - [(0:11, 0:12, "expected value, found comma")], - ); p!("[val: (true false)]" => [func!("val": (tuple!(Bool(true), Bool(false))), {})], [(0:11, 0:11, "expected comma")], @@ -1060,14 +1017,13 @@ mod tests { #[test] fn parse_objects() { - let f = || func!("val": (object! {}), {}); + let val = || func!("val": (object! {}), {}); - // Okay objects - p!("[val: {}]" => [f()]); - p!("[val: { key: value }]" => - [func!("val": (object! { "key" => Id("value") }), {})]); + // Okay objects. + pval!("{}" => (object! {})); + pval!("{ key: value }" => (object! { "key" => Id("value") })); - // Unclosed object + // Unclosed object. p!("[val: {hello: world]" => [func!("val": (object! { "hello" => Id("world") }), {})], [(0:19, 0:19, "expected closing brace")], @@ -1077,14 +1033,14 @@ mod tests { [(0:9, 0:9, "expected colon"), (0:9, 0:9, "expected closing brace")], ); - // Missing key - p!("[val: {,}]" => [f()], [(0:7, 0:8, "expected key, found comma")]); - p!("[val: { 12pt }]" => [f()], [(0:8, 0:12, "expected key, found size")]); - p!("[val: { : }]" => [f()], [(0:8, 0:9, "expected key, found colon")]); + // Missing key. + p!("[val: {,}]" => [val()], [(0:7, 0:8, "expected key, found comma")]); + p!("[val: { 12pt }]" => [val()], [(0:8, 0:12, "expected key, found size")]); + p!("[val: { : }]" => [val()], [(0:8, 0:9, "expected key, found colon")]); - // Missing colon - p!("[val: { key }]" => [f()], [(0:11, 0:11, "expected colon")]); - p!("[val: { key false }]" => [f()], [ + // Missing colon. + p!("[val: { key }]" => [val()], [(0:11, 0:11, "expected colon")]); + p!("[val: { key false }]" => [val()], [ (0:11, 0:11, "expected colon"), (0:12, 0:17, "expected key, found bool"), ]); @@ -1093,14 +1049,14 @@ mod tests { [(0:9, 0:9, "expected colon")], ); - // Missing value - p!("[val: { key: : }]" => [f()], [(0:13, 0:14, "expected value, found colon")]); + // Missing value. + p!("[val: { key: : }]" => [val()], [(0:13, 0:14, "expected value, found colon")]); p!("[val: { key: , k: \"s\" }]" => [func!("val": (object! { "k" => Str("s") }), {})], [(0:13, 0:14, "expected value, found comma")], ); - // Missing comma, invalid token + // Missing comma, invalid token. p!("[val: left={ a: 2, b: false π }]" => [func!("val": (), { "left" => object! { @@ -1115,7 +1071,7 @@ mod tests { #[test] fn parse_nested_tuples_and_objects() { - p!("[val: (1, { ab: (), d: (3, 14pt) }), false]" => [func!("val": ( + pval!("(1, { ab: (), d: (3, 14pt) }), false" => ( tuple!( Num(1.0), object!( @@ -1124,7 +1080,7 @@ mod tests { ), ), Bool(false), - ), {})]); + )); } #[test] @@ -1151,12 +1107,11 @@ mod tests { #[test] fn parse_multiple_mixed_arguments() { - p!("[val: a,]" => [func!("val": (Id("a")), {})]); p!("[val: 12pt, key=value]" => [func!("val": (Pt(12.0)), { "key" => Id("value") })], [], [(0:12, 0:15, ArgumentKey), (0:1, 0:4, ValidFuncName)], ); - p!("[val: a , \"b\" , c]" => [func!("val": (Id("a"), Str("b"), Id("c")), {})]); + pval!("a , x=\"b\" , c" => (Id("a"), Id("c")), { "x" => Str("b"), }); } #[test] @@ -1177,7 +1132,7 @@ mod tests { #[test] fn parse_invalid_key_value_pairs() { - // Invalid keys + // Invalid keys. p!("[val: true=you]" => [func!("val": (Bool(true), Id("you")), {})], [(0:10, 0:10, "expected comma"), @@ -1185,13 +1140,14 @@ mod tests { [(0:1, 0:4, ValidFuncName)], ); + // Unexpected equals. p!("[box: z=y=4]" => [func!("box": (Num(4.0)), { "z" => Id("y") })], [(0:9, 0:9, "expected comma"), (0:9, 0:10, "expected argument, found equals sign")], ); - // Invalid colon after keyable positional argument + // Invalid colon after keyable positional argument. p!("[val: key:12]" => [func!("val": (Id("key"), Num(12.0)), {})], [(0:9, 0:9, "expected comma"), @@ -1199,7 +1155,7 @@ mod tests { [(0:1, 0:4, ValidFuncName)], ); - // Invalid colon after non-keyable positional argument + // Invalid colon after unkeyable positional argument. p!("[val: true:12]" => [func!("val": (Bool(true), Num(12.0)), {})], [(0:10, 0:10, "expected comma"), (0:10, 0:11, "expected argument, found colon")], @@ -1209,7 +1165,7 @@ mod tests { #[test] fn parse_invalid_commas() { - // Missing commas + // Missing commas. p!("[val: 1pt 1]" => [func!("val": (Pt(1.0), Num(1.0)), {})], [(0:9, 0:9, "expected comma")], @@ -1219,7 +1175,7 @@ mod tests { [(0:7, 0:7, "expected comma")], ); - // Unexpected commas + // Unexpected commas. p!("[val:,]" => [func!("val")], [(0:5, 0:6, "expected argument, found comma")]); p!("[val: key=,]" => [func!("val")], [(0:10, 0:11, "expected value, found comma")]); p!("[val:, true]" => @@ -1231,13 +1187,10 @@ mod tests { #[test] fn parse_bodies() { p!("[val][Hi]" => [func!("val"; [T("Hi")])]); - - // Body nodes in bodies. p!("[val:*][*Hi*]" => [func!("val"; [Bold, T("Hi"), Bold])], [(0:5, 0:6, "expected argument, found star")], ); - // Errors in bodies. p!(" [val][ */ ]" => [S, func!("val"; [S, S])], @@ -1267,7 +1220,8 @@ mod tests { (0:6, 0:18, func!((0:1, 0:4, "val"); [(0:6, 0:11, T("world"))])), (0:18, 0:19, S), (0:19, 0:20, T("π")) - ], [], + ], + [], [(0:7, 0:10, ValidFuncName)], ); @@ -1281,7 +1235,8 @@ mod tests { (1:4, 1:10, func!((0:2, 0:5, "box"))), (1:10, 2:1, S), ])) - ], [], + ], + [], [(0:2, 0:5, ValidFuncName), (1:6, 1:9, ValidFuncName)], ); } diff --git a/src/syntax/span.rs b/src/syntax/span.rs index 9eb80d92..8083bc06 100644 --- a/src/syntax/span.rs +++ b/src/syntax/span.rs @@ -4,65 +4,63 @@ use std::fmt::{self, Debug, Formatter}; use std::ops::{Add, Sub}; use serde::Serialize; +/// A vector of spanned things. +pub type SpanVec<T> = Vec<Spanned<T>>; -/// Zero-indexed line-column position in source code. -#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize)] -pub struct Position { - /// The zero-indexed line. - pub line: usize, - /// The zero-indexed column. - pub column: usize, +/// [Offset](Span::offset) all spans in a vector of spanned things by a start +/// position. +pub fn offset_spans<T>( + start: Position, + vec: SpanVec<T>, +) -> impl Iterator<Item=Spanned<T>> { + vec.into_iter().map(move |s| s.map_span(|span| span.offset(start))) } -impl Position { - /// The line 0, column 0 position. - pub const ZERO: Position = Position { line: 0, column: 0 }; +/// A value with the span it corresponds to in the source code. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)] +pub struct Spanned<T> { + /// The value. + pub v: T, + /// The corresponding span. + pub span: Span, +} - /// Crete a new instance from line and column. - pub fn new(line: usize, column: usize) -> Position { - Position { line, column } +impl<T> Spanned<T> { + /// Create a new instance from a value and its span. + pub fn new(v: T, span: Span) -> Spanned<T> { + Spanned { v, span } } -} -impl Add for Position { - type Output = Position; + /// Create a new instance from a value with the zero span. + pub fn zero(v: T) -> Spanned<T> { + Spanned { v, span: Span::ZERO } + } - fn add(self, rhs: Position) -> Position { - if rhs.line == 0 { - Position { - line: self.line, - column: self.column + rhs.column - } - } else { - Position { - line: self.line + rhs.line, - column: rhs.column, - } - } + /// Access the value. + pub fn value(self) -> T { + self.v } -} -impl Sub for Position { - type Output = Position; + /// Map the value using a function while keeping the span. + pub fn map<V, F>(self, f: F) -> Spanned<V> where F: FnOnce(T) -> V { + Spanned { v: f(self.v), span: self.span } + } - fn sub(self, rhs: Position) -> Position { - if self.line == rhs.line { - Position { - line: 0, - column: self.column - rhs.column - } - } else { - Position { - line: self.line - rhs.line, - column: self.column, - } - } + /// Maps the span while keeping the value. + pub fn map_span<F>(mut self, f: F) -> Spanned<T> where F: FnOnce(Span) -> Span { + self.span = f(self.span); + self } } -impl Debug for Position { +impl<T: Debug> Debug for Spanned<T> { fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!(f, "{}:{}", self.line, self.column) + self.v.fmt(f)?; + if f.alternate() { + f.write_str(" ")?; + self.span.fmt(f)?; + } + Ok(()) } } @@ -111,56 +109,67 @@ impl Span { impl Debug for Span { fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!(f, "({:?} -> {:?})", self.start, self.end) + write!(f, "<{:?}-{:?}>", self.start, self.end) } } -/// A value with the span it corresponds to in the source code. -#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)] -pub struct Spanned<T> { - /// The value. - pub v: T, - /// The corresponding span. - pub span: Span, +/// Zero-indexed line-column position in source code. +#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize)] +pub struct Position { + /// The zero-indexed line. + pub line: usize, + /// The zero-indexed column. + pub column: usize, } -impl<T> Spanned<T> { - /// Create a new instance from a value and its span. - pub fn new(v: T, span: Span) -> Spanned<T> { - Spanned { v, span } - } +impl Position { + /// The line 0, column 0 position. + pub const ZERO: Position = Position { line: 0, column: 0 }; - /// Access the value. - pub fn value(self) -> T { - self.v + /// Crete a new instance from line and column. + pub fn new(line: usize, column: usize) -> Position { + Position { line, column } } +} - /// Map the value using a function while keeping the span. - pub fn map<V, F>(self, f: F) -> Spanned<V> where F: FnOnce(T) -> V { - Spanned { v: f(self.v), span: self.span } - } +impl Add for Position { + type Output = Position; - /// Maps the span while keeping the value. - pub fn map_span<F>(mut self, f: F) -> Spanned<T> where F: FnOnce(Span) -> Span { - self.span = f(self.span); - self + fn add(self, rhs: Position) -> Position { + if rhs.line == 0 { + Position { + line: self.line, + column: self.column + rhs.column + } + } else { + Position { + line: self.line + rhs.line, + column: rhs.column, + } + } } } -impl<T: Debug> Debug for Spanned<T> { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - self.v.fmt(f)?; - f.write_str(" ")?; - self.span.fmt(f) +impl Sub for Position { + type Output = Position; + + fn sub(self, rhs: Position) -> Position { + if self.line == rhs.line { + Position { + line: 0, + column: self.column - rhs.column + } + } else { + Position { + line: self.line - rhs.line, + column: self.column, + } + } } } -/// A vector of spanned things. -pub type SpanVec<T> = Vec<Spanned<T>>; - -/// [Offset](Span::offset) all spans in a vector of spanned things by a start -/// position. -pub fn offset_spans<T>(start: Position, vec: SpanVec<T>) -> impl Iterator<Item=Spanned<T>> { - vec.into_iter() - .map(move |s| s.map_span(|span| span.offset(start))) +impl Debug for Position { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "{}:{}", self.line, self.column) + } } diff --git a/src/syntax/test.rs b/src/syntax/test.rs index 465f475a..aaad5c12 100644 --- a/src/syntax/test.rs +++ b/src/syntax/test.rs @@ -1,17 +1,14 @@ use std::fmt::Debug; use super::func::FuncHeader; +use super::span::Spanned; use super::expr::{Expr, Tuple, NamedTuple, Object}; -use super::span::{Span, Spanned}; -use super::tokens::Token; use super::*; - -/// Check whether the expected and found results for the given source code -/// match by the comparison function, and print them out otherwise. -pub fn check<T>(src: &str, exp: T, found: T, spans: bool) +/// Check whether the expected and found results are the same. +pub fn check<T>(src: &str, exp: T, found: T, cmp_spans: bool) where T: Debug + PartialEq + SpanlessEq { - let cmp = if spans { PartialEq::eq } else { SpanlessEq::spanless_eq }; + let cmp = if cmp_spans { PartialEq::eq } else { SpanlessEq::spanless_eq }; if !cmp(&exp, &found) { println!("source: {:?}", src); println!("expected: {:#?}", exp); @@ -23,16 +20,25 @@ where T: Debug + PartialEq + SpanlessEq { /// Create a vector of optionally spanned expressions from a list description. /// /// # Examples -/// When you want to add span information to the items, the format is as -/// follows. /// ``` +/// // With spans /// spanned![(0:0, 0:5, "hello"), (0:5, 0:3, "world")] +/// +/// // Without spans: Implicit zero spans. +/// spanned!["hello", "world"] /// ``` -/// The span information can simply be omitted to create a vector with items -/// that are spanned with zero spans. -macro_rules! spanned { - (item ($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)) => ({ - #[allow(unused_imports)] +macro_rules! span_vec { + ($(($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)),* $(,)?) => { + (vec![$(span_item!(($sl:$sc, $el:$ec, $v))),*], true) + }; + + ($($v:expr),* $(,)?) => { + (vec![$(span_item!($v)),*], false) + }; +} + +macro_rules! span_item { + (($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)) => ({ use $crate::syntax::span::{Position, Span, Spanned}; Spanned { span: Span::new( @@ -43,24 +49,11 @@ macro_rules! spanned { } }); - (item $v:expr) => { - $crate::syntax::test::zspan($v) - }; - - (vec $(($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)),* $(,)?) => { - (vec![$(spanned![item ($sl:$sc, $el:$ec, $v)]),*], true) - }; - - (vec $($v:expr),* $(,)?) => { - (vec![$($crate::syntax::test::zspan($v)),*], false) + ($v:expr) => { + $crate::syntax::span::Spanned::zero($v) }; } -/// Span an element with a zero span. -pub fn zspan<T>(v: T) -> Spanned<T> { - Spanned { v, span: Span::ZERO } -} - function! { /// Most functions in the tests are parsed into the debug function for easy /// inspection of arguments and body. @@ -120,8 +113,8 @@ impl SpanlessEq for DebugFn { impl SpanlessEq for Expr { fn spanless_eq(&self, other: &Expr) -> bool { match (self, other) { - (Expr::NamedTuple(a), Expr::NamedTuple(b)) => a.spanless_eq(b), (Expr::Tuple(a), Expr::Tuple(b)) => a.spanless_eq(b), + (Expr::NamedTuple(a), Expr::NamedTuple(b)) => a.spanless_eq(b), (Expr::Object(a), Expr::Object(b)) => a.spanless_eq(b), (Expr::Neg(a), Expr::Neg(b)) => a.spanless_eq(&b), (Expr::Add(a1, a2), Expr::Add(b1, b2)) => a1.spanless_eq(&b1) && a2.spanless_eq(&b2), @@ -175,8 +168,7 @@ impl<T: SpanlessEq> SpanlessEq for Box<T> { } } -/// Implement `SpanlessEq` by just forwarding to `PartialEq`. -macro_rules! forward { +macro_rules! impl_through_partial_eq { ($type:ty) => { impl SpanlessEq for $type { fn spanless_eq(&self, other: &$type) -> bool { @@ -186,6 +178,8 @@ macro_rules! forward { }; } -forward!(String); -forward!(Token<'_>); -forward!(Decoration); +impl_through_partial_eq!(Token<'_>); + +// Implement for string and decoration to be able to compare feedback. +impl_through_partial_eq!(String); +impl_through_partial_eq!(Decoration); diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs index 0fc52f26..b1bf76e3 100644 --- a/src/syntax/tokens.rs +++ b/src/syntax/tokens.rs @@ -532,6 +532,7 @@ pub fn is_identifier(string: &str) -> bool { #[cfg(test)] +#[allow(non_snake_case)] mod tests { use super::super::test::check; use super::*; @@ -552,31 +553,23 @@ mod tests { Slash, }; - #[allow(non_snake_case)] - fn Str(string: &'static str, terminated: bool) -> Token<'static> { - Token::ExprStr { string, terminated } - } - - #[allow(non_snake_case)] - fn Raw(raw: &'static str, terminated: bool) -> Token<'static> { - Token::Raw { raw, terminated } - } - /// Test whether the given string tokenizes into the given list of tokens. macro_rules! t { ($mode:expr, $source:expr => [$($tokens:tt)*]) => { - let (exp, spans) = spanned![vec $($tokens)*]; + let (exp, spans) = span_vec![$($tokens)*]; let found = Tokens::new(Position::ZERO, $source, $mode).collect::<Vec<_>>(); check($source, exp, found, spans); } } - /// Write down a function token compactly. + fn Str(string: &str, terminated: bool) -> Token { Token::ExprStr { string, terminated } } + fn Raw(raw: &str, terminated: bool) -> Token { Token::Raw { raw, terminated } } + macro_rules! func { ($header:expr, Some($($tokens:tt)*), $terminated:expr) => { Function { header: $header, - body: Some(spanned![item $($tokens)*]), + body: Some(span_item!(($($tokens)*))), terminated: $terminated, } }; @@ -677,12 +670,12 @@ mod tests { fn tokenize_functions() { t!(Body, "a[f]" => [T("a"), func!("f", None, true)]); t!(Body, "[f]a" => [func!("f", None, true), T("a")]); - t!(Body, "\n\n[f][ ]" => [S(2), func!("f", Some((0:4, 0:5, " ")), true)]); - t!(Body, "abc [f][ ]a" => [T("abc"), S(0), func!("f", Some((0:4, 0:5, " ")), true), T("a")]); + t!(Body, "\n\n[f][ ]" => [S(2), func!("f", Some(0:4, 0:5, " "), true)]); + t!(Body, "abc [f][ ]a" => [T("abc"), S(0), func!("f", Some(0:4, 0:5, " "), true), T("a")]); t!(Body, "[f: [=][*]]" => [func!("f: [=][*]", None, true)]); - t!(Body, "[_][[,],]," => [func!("_", Some((0:4, 0:8, "[,],")), true), T(",")]); - t!(Body, "[=][=][=]" => [func!("=", Some((0:4, 0:5, "=")), true), func!("=", None, true)]); - t!(Body, "[=][[=][=][=]]" => [func!("=", Some((0:4, 0:13, "[=][=][=]")), true)]); + t!(Body, "[_][[,],]," => [func!("_", Some(0:4, 0:8, "[,],"), true), T(",")]); + t!(Body, "[=][=][=]" => [func!("=", Some(0:4, 0:5, "="), true), func!("=", None, true)]); + t!(Body, "[=][[=][=][=]]" => [func!("=", Some(0:4, 0:13, "[=][=][=]"), true)]); t!(Header, "[" => [func!("", None, false)]); t!(Header, "]" => [Invalid("]")]); } @@ -696,25 +689,25 @@ mod tests { t!(Body, "[f: `]" => [func!("f: `", None, true)]); // End of function with strings and carets in bodies - t!(Body, "[f][\"]" => [func!("f", Some((0:4, 0:5, "\"")), true)]); - t!(Body, r#"[f][\"]"# => [func!("f", Some((0:4, 0:6, r#"\""#)), true)]); - t!(Body, "[f][`]" => [func!("f", Some((0:4, 0:6, "`]")), false)]); - t!(Body, "[f][\\`]" => [func!("f", Some((0:4, 0:6, "\\`")), true)]); - t!(Body, "[f][`raw`]" => [func!("f", Some((0:4, 0:9, "`raw`")), true)]); - t!(Body, "[f][`raw]" => [func!("f", Some((0:4, 0:9, "`raw]")), false)]); - t!(Body, "[f][`raw]`]" => [func!("f", Some((0:4, 0:10, "`raw]`")), true)]); - t!(Body, "[f][`\\`]" => [func!("f", Some((0:4, 0:8, "`\\`]")), false)]); - t!(Body, "[f][`\\\\`]" => [func!("f", Some((0:4, 0:8, "`\\\\`")), true)]); + t!(Body, "[f][\"]" => [func!("f", Some(0:4, 0:5, "\""), true)]); + t!(Body, r#"[f][\"]"# => [func!("f", Some(0:4, 0:6, r#"\""#), true)]); + t!(Body, "[f][`]" => [func!("f", Some(0:4, 0:6, "`]"), false)]); + t!(Body, "[f][\\`]" => [func!("f", Some(0:4, 0:6, "\\`"), true)]); + t!(Body, "[f][`raw`]" => [func!("f", Some(0:4, 0:9, "`raw`"), true)]); + t!(Body, "[f][`raw]" => [func!("f", Some(0:4, 0:9, "`raw]"), false)]); + t!(Body, "[f][`raw]`]" => [func!("f", Some(0:4, 0:10, "`raw]`"), true)]); + t!(Body, "[f][`\\`]" => [func!("f", Some(0:4, 0:8, "`\\`]"), false)]); + t!(Body, "[f][`\\\\`]" => [func!("f", Some(0:4, 0:8, "`\\\\`"), true)]); // End of function with comments - t!(Body, "[f][/*]" => [func!("f", Some((0:4, 0:7, "/*]")), false)]); - t!(Body, "[f][/*`*/]" => [func!("f", Some((0:4, 0:9, "/*`*/")), true)]); + t!(Body, "[f][/*]" => [func!("f", Some(0:4, 0:7, "/*]"), false)]); + t!(Body, "[f][/*`*/]" => [func!("f", Some(0:4, 0:9, "/*`*/"), true)]); t!(Body, "[f: //]\n]" => [func!("f: //]\n", None, true)]); t!(Body, "[f: \"//]\n]" => [func!("f: \"//]\n]", None, false)]); // End of function with escaped brackets - t!(Body, "[f][\\]]" => [func!("f", Some((0:4, 0:6, "\\]")), true)]); - t!(Body, "[f][\\[]" => [func!("f", Some((0:4, 0:6, "\\[")), true)]); + t!(Body, "[f][\\]]" => [func!("f", Some(0:4, 0:6, "\\]"), true)]); + t!(Body, "[f][\\[]" => [func!("f", Some(0:4, 0:6, "\\["), true)]); } #[test] |