summaryrefslogtreecommitdiff
path: root/src/syntax
diff options
context:
space:
mode:
Diffstat (limited to 'src/syntax')
-rw-r--r--src/syntax/mod.rs4
-rw-r--r--src/syntax/parsing.rs190
-rw-r--r--src/syntax/span.rs10
-rw-r--r--src/syntax/test.rs88
-rw-r--r--src/syntax/tokens.rs163
5 files changed, 450 insertions, 5 deletions
diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs
index 7f4052ab..cfa4c2e5 100644
--- a/src/syntax/mod.rs
+++ b/src/syntax/mod.rs
@@ -11,11 +11,13 @@ use self::span::{Spanned, SpanVec};
pub mod expr;
pub mod func;
pub mod span;
-
pub_use_mod!(scope);
pub_use_mod!(parsing);
pub_use_mod!(tokens);
+#[cfg(test)]
+mod test;
+
/// Represents a parsed piece of source that can be layouted and in the future
/// also be queried for information used for refactorings, autocomplete, etc.
diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs
index a7f39640..03866c2c 100644
--- a/src/syntax/parsing.rs
+++ b/src/syntax/parsing.rs
@@ -397,3 +397,193 @@ impl<'s> FuncParser<'s> {
.unwrap_or_else(|| self.tokens.pos())
}
}
+
+
+#[cfg(test)]
+#[allow(non_snake_case)]
+mod tests {
+ use crate::size::Size;
+ use super::super::test::{DebugFn, SpanlessEq};
+ use super::*;
+
+ use Node::{
+ Space as S, Newline as N,
+ ToggleItalic as Italic, ToggleBolder as Bold, ToggleMonospace as Mono,
+ };
+
+ pub use Expr::{Number as Num, Bool};
+ pub fn Id(text: &str) -> Expr { Expr::Ident(Ident(text.to_string())) }
+ pub fn Str(text: &str) -> Expr { Expr::Str(text.to_string()) }
+
+ fn T(text: &str) -> Node { Node::Text(text.to_string()) }
+
+ /// Test whether the given string parses into the given node list.
+ macro_rules! p {
+ ($s:expr => [$($b:tt)*]) => {
+ let ctx = ParseContext { scope: &scope() };
+ let model = parse(Position::ZERO, $s, ctx).output;
+ let (expected, cmp) = model!([$($b)*]);
+
+ if !cmp(&model, &expected) {
+ fail($s, model, expected);
+ }
+ };
+ }
+
+ /// Test whether the given string yields the given parse errors.
+ macro_rules! e {
+ ($s:expr => [$(($sl:tt:$sc:tt, $el:tt:$ec:tt, $e:expr)),* $(,)?]) => {
+ let ctx = ParseContext { scope: &scope() };
+ let errors = parse(Position::ZERO, $s, ctx).errors
+ .into_iter()
+ .map(|s| s.map(|e| e.message))
+ .collect::<Vec<_>>();
+
+ let expected = vec![
+ $(Spanned {
+ v: $e.to_string(),
+ span: Span {
+ start: Position { line: $sl, column: $sc },
+ end: Position { line: $el, column: $ec },
+ },
+ }),*
+ ];
+
+ if errors != expected {
+ fail($s, errors, expected);
+ }
+ };
+ }
+
+ fn scope() -> Scope {
+ let mut scope = Scope::new::<DebugFn>();
+ scope.add::<DebugFn>("f");
+ scope.add::<DebugFn>("box");
+ scope
+ }
+
+ fn fail(src: &str, found: impl Debug, expected: impl Debug) {
+ eprintln!("source: {:?}", src);
+ eprintln!("found: {:#?}", found);
+ eprintln!("expected: {:#?}", expected);
+ panic!("test failed");
+ }
+
+ /// Parse a list of optionally spanned nodes into a syntax model.
+ macro_rules! model {
+ ([$(($sl:tt:$sc:tt, $el:tt:$ec:tt, $n:expr)),* $(,)?]) => ((SyntaxModel {
+ nodes: vec![
+ $(Spanned { v: $n, span: Span {
+ start: Position { line: $sl, column: $sc },
+ end: Position { line: $el, column: $ec },
+ }}),*
+ ]
+ }, <SyntaxModel as PartialEq>::eq));
+
+ ([$($e:tt)*]) => ((SyntaxModel {
+ nodes: vec![$($e)*].into_iter().map(zspan).collect::<Vec<_>>()
+ }, <SyntaxModel as SpanlessEq>::spanless_eq));
+ }
+
+ /// Build a `DebugFn` function model.
+ macro_rules! func {
+ ($name:expr
+ $(,pos: [$($item:expr),* $(,)?])?
+ $(,key: [$($key:expr => $value:expr),* $(,)?])?;
+ $($b:tt)*) => ({
+ #![allow(unused_mut, unused_assignments)]
+
+ let mut pos = Tuple::new();
+ let mut key = Object::new();
+ $(pos = Tuple { items: vec![$(zspan($item)),*] };)?
+ $(key = Object {
+ pairs: vec![$(Pair {
+ key: zspan(Ident($key.to_string())),
+ value: zspan($value),
+ }),*]
+ };)?
+
+ Node::Model(Box::new(DebugFn {
+ header: FuncHeader {
+ name: zspan(Ident($name.to_string())),
+ args: FuncArgs {
+ pos,
+ key,
+ },
+ },
+ body: func!(@body $($b)*),
+ }))
+ });
+
+ (@body Some([$($b:tt)*])) => (Some(model!([$($b)*]).0));
+ (@body None) => (None);
+ }
+
+ /// Span an element with a zero span.
+ fn zspan<T>(v: T) -> Spanned<T> {
+ Spanned { v, span: Span::ZERO }
+ }
+
+ #[test]
+ fn parse_flat_nodes() {
+ p!("" => []);
+ p!("hi" => [T("hi")]);
+ p!("*hi" => [Bold, T("hi")]);
+ p!("hi_" => [T("hi"), Italic]);
+ p!("`py`" => [Mono, T("py"), Mono]);
+ p!("hi you" => [T("hi"), S, T("you")]);
+ p!("💜\n\n 🌍" => [T("💜"), N, T("🌍")]);
+ }
+
+ #[test]
+ fn parse_functions() {
+ p!("[func]" => [func!("func"; None)]);
+ p!("[tree][hi *you*]" => [func!("tree"; Some([T("hi"), S, Bold, T("you"), Bold]))]);
+ p!("[f: , hi, * \"du\"]" => [func!("f", pos: [Id("hi"), Str("du")]; None)]);
+ p!("from [align: left] to" => [
+ T("from"), S, func!("align", pos: [Id("left")]; None), S, T("to")
+ ]);
+
+ p!("[f: left, 12pt, false]" => [
+ func!("f", pos: [Id("left"), Expr::Size(Size::pt(12.0)), Bool(false)]; None)
+ ]);
+
+ p!("[box: x=1.2pt, false][a b c] bye" => [
+ func!(
+ "box",
+ pos: [Bool(false)],
+ key: ["x" => Expr::Size(Size::pt(1.2))];
+ Some([T("a"), S, T("b"), S, T("c")])
+ ),
+ S, T("bye"),
+ ]);
+ }
+
+ #[test]
+ fn parse_spanned() {
+ p!("hi you" => [(0:0, 0:2, T("hi")), (0:2, 0:3, S), (0:3, 0:6, T("you"))]);
+ }
+
+ #[test]
+ fn parse_errors() {
+ e!("[f: , hi, * \"du\"]" => [
+ (0:4, 0:5, "expected value, found comma"),
+ (0:10, 0:11, "expected value, found invalid token"),
+ ]);
+ e!("[f:, , ,]" => [
+ (0:3, 0:4, "expected value, found comma"),
+ (0:5, 0:6, "expected value, found comma"),
+ (0:7, 0:8, "expected value, found comma"),
+ ]);
+ e!("[f:" => [(0:3, 0:3, "expected closing bracket")]);
+ e!("[f: hi" => [(0:6, 0:6, "expected closing bracket")]);
+ e!("[f: hey 12pt]" => [(0:7, 0:7, "expected comma")]);
+ e!("[box: x=, false z=y=4" => [
+ (0:8, 0:9, "expected value, found comma"),
+ (0:15, 0:15, "expected comma"),
+ (0:19, 0:19, "expected comma"),
+ (0:19, 0:20, "expected value, found equals sign"),
+ (0:21, 0:21, "expected closing bracket"),
+ ]);
+ }
+}
diff --git a/src/syntax/span.rs b/src/syntax/span.rs
index 7a051d99..8973ef89 100644
--- a/src/syntax/span.rs
+++ b/src/syntax/span.rs
@@ -116,7 +116,7 @@ impl Debug for Span {
}
/// A value with the span it corresponds to in the source code.
-#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash, Serialize)]
+#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)]
pub struct Spanned<T> {
/// The value.
pub v: T,
@@ -147,6 +147,14 @@ impl<T> Spanned<T> {
}
}
+impl<T: Debug> Debug for Spanned<T> {
+ fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+ self.span.fmt(f)?;
+ f.write_str(": ")?;
+ self.v.fmt(f)
+ }
+}
+
/// A vector of spanned things.
pub type SpanVec<T> = Vec<Spanned<T>>;
diff --git a/src/syntax/test.rs b/src/syntax/test.rs
new file mode 100644
index 00000000..e37e8cf5
--- /dev/null
+++ b/src/syntax/test.rs
@@ -0,0 +1,88 @@
+use super::func::FuncHeader;
+use super::expr::{Expr, Tuple, Object};
+use super::*;
+
+
+function! {
+ /// Most functions in the tests are parsed into the debug function for easy
+ /// inspection of arguments and body.
+ #[derive(Debug, Clone, PartialEq)]
+ pub struct DebugFn {
+ pub header: FuncHeader,
+ pub body: Option<SyntaxModel>,
+ }
+
+ parse(header, body, ctx, errors, decos) {
+ let cloned = header.clone();
+ header.args.pos.items.clear();
+ header.args.key.pairs.clear();
+ DebugFn {
+ header: cloned,
+ body: body!(opt: body, ctx, errors, decos),
+ }
+ }
+
+ layout(self, ctx, errors) { vec![] }
+}
+
+/// Compares elements by only looking at values and ignoring spans.
+pub trait SpanlessEq<Rhs=Self> {
+ fn spanless_eq(&self, other: &Rhs) -> bool;
+}
+
+impl SpanlessEq for Vec<Spanned<Token<'_>>> {
+ fn spanless_eq(&self, other: &Vec<Spanned<Token>>) -> bool {
+ self.len() == other.len()
+ && self.iter().zip(other).all(|(x, y)| x.v == y.v)
+ }
+}
+
+impl SpanlessEq for SyntaxModel {
+ fn spanless_eq(&self, other: &SyntaxModel) -> bool {
+ fn downcast<'a>(func: &'a (dyn Model + 'static)) -> &'a DebugFn {
+ func.downcast::<DebugFn>().expect("not a debug fn")
+ }
+
+ self.nodes.len() == other.nodes.len()
+ && self.nodes.iter().zip(&other.nodes).all(|(x, y)| match (&x.v, &y.v) {
+ (Node::Model(a), Node::Model(b)) => {
+ downcast(a.as_ref()).spanless_eq(downcast(b.as_ref()))
+ }
+ (a, b) => a == b,
+ })
+ }
+}
+
+impl SpanlessEq for DebugFn {
+ fn spanless_eq(&self, other: &DebugFn) -> bool {
+ self.header.name.v == other.header.name.v
+ && self.header.args.pos.spanless_eq(&other.header.args.pos)
+ && self.header.args.key.spanless_eq(&other.header.args.key)
+ }
+}
+
+impl SpanlessEq for Expr {
+ fn spanless_eq(&self, other: &Expr) -> bool {
+ match (self, other) {
+ (Expr::Tuple(a), Expr::Tuple(b)) => a.spanless_eq(b),
+ (Expr::Object(a), Expr::Object(b)) => a.spanless_eq(b),
+ (a, b) => a == b,
+ }
+ }
+}
+
+impl SpanlessEq for Tuple {
+ fn spanless_eq(&self, other: &Tuple) -> bool {
+ self.items.len() == other.items.len()
+ && self.items.iter().zip(&other.items)
+ .all(|(x, y)| x.v.spanless_eq(&y.v))
+ }
+}
+
+impl SpanlessEq for Object {
+ fn spanless_eq(&self, other: &Object) -> bool {
+ self.pairs.len() == other.pairs.len()
+ && self.pairs.iter().zip(&other.pairs)
+ .all(|(x, y)| x.key.v == y.key.v && x.value.v.spanless_eq(&y.value.v))
+ }
+}
diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs
index 747b6b93..7b52f655 100644
--- a/src/syntax/tokens.rs
+++ b/src/syntax/tokens.rs
@@ -341,12 +341,17 @@ impl<'s> Tokens<'s> {
}
let c = self.peek().unwrap_or('n');
- if self.mode == Body && is_escapable(c) {
+ let string = if is_escapable(c) {
let index = self.index();
self.eat();
- Text(&self.src[index .. index + c.len_utf8()])
+ &self.src[index .. index + c.len_utf8()]
} else {
- Text("\\")
+ "\\"
+ };
+
+ match self.mode {
+ Header => Invalid(string),
+ Body => Text(string),
}
}
@@ -456,3 +461,155 @@ pub fn is_identifier(string: &str) -> bool {
true
}
+
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ use Token::{
+ Space as S,
+ LineComment as LC, BlockComment as BC,
+ LeftParen as LP, RightParen as RP,
+ LeftBrace as LB, RightBrace as RB,
+ ExprIdent as Id, ExprNumber as Num, ExprBool as Bool,
+ Text as T,
+ };
+
+ #[allow(non_snake_case)]
+ fn Str(string: &'static str, terminated: bool) -> Token<'static> {
+ Token::ExprStr { string, terminated }
+ }
+
+ /// Test whether the given string tokenizes into the given list of tokens.
+ macro_rules! t {
+ ($m:expr, $s:expr => [$(($sl:tt:$sc:tt, $el:tt:$ec:tt, $t:expr)),* $(,)?]) => {
+ let tokens = Tokens::new(Position::ZERO, $s, $m).collect::<Vec<_>>();
+ assert_eq!(tokens, vec![$(Spanned {
+ span: Span::new(Position::new($sl, $sc), Position::new($el, $ec)),
+ v: $t
+ }),*]);
+
+ };
+
+ ($m:expr, $s:expr => [$($t:expr),* $(,)?]) => {
+ let tokens = Tokens::new(Position::ZERO, $s, $m)
+ .map(Spanned::value)
+ .collect::<Vec<_>>();
+ assert_eq!(tokens, vec![$($t),*]);
+ };
+ }
+
+ /// Parse a function token.
+ macro_rules! func {
+ ($header:expr, Some(($sl:tt:$sc:tt, $el:tt:$ec:tt, $body:expr)), $terminated:expr) => {
+ Function {
+ header: $header,
+ body: Some(Spanned {
+ span: Span::new(Position::new($sl, $sc), Position::new($el, $ec)),
+ v: $body,
+ }),
+ terminated: $terminated,
+ }
+ };
+ ($header:expr, None, $terminated:expr) => {
+ Function { header: $header, body: None, terminated: $terminated }
+ }
+ }
+
+ #[test]
+ fn tokenize_whitespace() {
+ t!(Body, "" => []);
+ t!(Body, " " => [S(0)]);
+ t!(Body, " " => [S(0)]);
+ t!(Body, "\t" => [S(0)]);
+ t!(Body, " \t" => [S(0)]);
+ t!(Body, "\n" => [S(1)]);
+ t!(Body, "\n " => [S(1)]);
+ t!(Body, " \n" => [S(1)]);
+ t!(Body, " \n " => [S(1)]);
+ t!(Body, "\r\n" => [S(1)]);
+ t!(Body, " \n\t \n " => [S(2)]);
+ t!(Body, "\n\r" => [S(2)]);
+ t!(Body, " \r\r\n \x0D" => [S(3)]);
+ }
+
+ #[test]
+ fn tokenize_comments() {
+ t!(Body, "a // bc\n " => [T("a"), S(0), LC(" bc"), S(1)]);
+ t!(Body, "a //a//b\n " => [T("a"), S(0), LC("a//b"), S(1)]);
+ t!(Body, "a //a//b\r\n" => [T("a"), S(0), LC("a//b"), S(1)]);
+ t!(Body, "a //a//b\n\nhello" => [T("a"), S(0), LC("a//b"), S(2), T("hello")]);
+ t!(Body, "/**/" => [BC("")]);
+ t!(Body, "_/*_/*a*/*/" => [Underscore, BC("_/*a*/")]);
+ t!(Body, "/*/*/" => [BC("/*/")]);
+ t!(Body, "abc*/" => [T("abc"), Invalid("*/")]);
+ }
+
+ #[test]
+ fn tokenize_header_only_tokens() {
+ t!(Body, "\"hi\"" => [T("\"hi"), T("\"")]);
+ t!(Body, "a: b" => [T("a"), T(":"), S(0), T("b")]);
+ t!(Body, "c=d, " => [T("c"), T("=d"), T(","), S(0)]);
+ t!(Header, "[" => [func!("", None, false)]);
+ t!(Header, "]" => [Invalid("]")]);
+ t!(Header, "(){}:=," => [LP, RP, LB, RB, Colon, Equals, Comma]);
+ t!(Header, "a:b" => [Id("a"), Colon, Id("b")]);
+ t!(Header, "=" => [Equals]);
+ t!(Header, "," => [Comma]);
+ t!(Header, r#""hello\"world""# => [Str(r#"hello\"world"#, true)]);
+ t!(Header, r#""hi", 12pt"# => [Str("hi", true), Comma, S(0), ExprSize(Size::pt(12.0))]);
+ t!(Header, "a: true, x=1" => [Id("a"), Colon, S(0), Bool(true), Comma, S(0), Id("x"), Equals, Num(1.0)]);
+ t!(Header, "120%" => [Num(1.2)]);
+ t!(Header, "🌓, 🌍," => [Invalid("🌓"), Comma, S(0), Invalid("🌍"), Comma]);
+ }
+
+ #[test]
+ fn tokenize_body_only_tokens() {
+ t!(Body, "_*`" => [Underscore, Star, Backtick]);
+ t!(Body, "[func]*bold*" => [func!("func", None, true), Star, T("bold"), Star]);
+ t!(Body, "hi_you_ there" => [T("hi"), Underscore, T("you"), Underscore, S(0), T("there")]);
+ t!(Header, "_*`" => [Invalid("_"), Invalid("*"), Invalid("`")]);
+ }
+
+ #[test]
+ fn tokenize_nested_functions() {
+ t!(Body, "[f: [=][*]]" => [func!("f: [=][*]", None, true)]);
+ t!(Body, "[_][[,],]," => [func!("_", Some((0:3, 0:9, "[,],")), true), T(",")]);
+ t!(Body, "[=][=][=]" => [func!("=", Some((0:3, 0:6, "=")), true), func!("=", None, true)]);
+ t!(Body, "[=][[=][=][=]]" => [func!("=", Some((0:3, 0:14, "[=][=][=]")), true)]);
+ }
+
+ #[test]
+ fn tokenize_escaped_symbols() {
+ t!(Body, r"\\" => [T(r"\")]);
+ t!(Body, r"\[" => [T("[")]);
+ t!(Body, r"\]" => [T("]")]);
+ t!(Body, r"\*" => [T("*")]);
+ t!(Body, r"\_" => [T("_")]);
+ t!(Body, r"\`" => [T("`")]);
+ t!(Body, r"\/" => [T("/")]);
+ }
+
+ #[test]
+ fn tokenize_unescapable_symbols() {
+ t!(Body, r"\a" => [T("\\"), T("a")]);
+ t!(Body, r"\:" => [T(r"\"), T(":")]);
+ t!(Body, r"\=" => [T(r"\"), T("=")]);
+ t!(Header, r"\\\\" => [Invalid("\\"), Invalid("\\")]);
+ t!(Header, r"\a" => [Invalid("\\"), Id("a")]);
+ t!(Header, r"\:" => [Invalid(r"\"), Colon]);
+ t!(Header, r"\=" => [Invalid(r"\"), Equals]);
+ t!(Header, r"\," => [Invalid(r"\"), Comma]);
+ }
+
+ #[test]
+ fn tokenize_with_spans() {
+ t!(Body, "hello" => [(0:0, 0:5, T("hello"))]);
+ t!(Body, "ab\r\nc" => [(0:0, 0:2, T("ab")), (0:2, 1:0, S(1)), (1:0, 1:1, T("c"))]);
+ t!(Body, "[x = \"(1)\"]*" => [(0:0, 0:11, func!("x = \"(1)\"", None, true)), (0:11, 0:12, Star)]);
+ t!(Body, "// ab\r\n\nf" => [(0:0, 0:5, LC(" ab")), (0:5, 2:0, S(2)), (2:0, 2:1, T("f"))]);
+ t!(Body, "/*b*/_" => [(0:0, 0:5, BC("b")), (0:5, 0:6, Underscore)]);
+ t!(Header, "a=10" => [(0:0, 0:1, Id("a")), (0:1, 0:2, Equals), (0:2, 0:4, Num(10.0))]);
+ }
+}
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-15 19:16:37 +0000