From 1e4cab393e55df8875c6303ebb7bde8f09f911c9 Mon Sep 17 00:00:00 2001 From: Martin Haug Date: Tue, 2 Nov 2021 12:06:22 +0100 Subject: Introduce incremental parsing --- src/source.rs | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) (limited to 'src/source.rs') diff --git a/src/source.rs b/src/source.rs index 432688a0..069edd29 100644 --- a/src/source.rs +++ b/src/source.rs @@ -268,7 +268,7 @@ impl SourceFile { /// This panics if the `replace` range is out of bounds. pub fn edit(&mut self, replace: Range, with: &str) { let start = replace.start; - self.src.replace_range(replace, with); + self.src.replace_range(replace.clone(), with); // Remove invalidated line starts. let line = self.byte_to_line(start).unwrap(); @@ -283,8 +283,39 @@ impl SourceFile { self.line_starts .extend(newlines(&self.src[start ..]).map(|idx| start + idx)); - // Reparse. - self.root = parse(&self.src); + // Update the root node. + #[cfg(not(feature = "parse-cache"))] + { + self.root = parse(&self.src); + } + + #[cfg(feature = "parse-cache")] + { + let insertion_span = replace.into_span(self.id); + let incremental_target = + Rc::make_mut(&mut self.root).incremental_parent(insertion_span); + + match incremental_target { + Some((child_idx, parent, offset)) => { + let child = &parent.children()[child_idx]; + let src = &self.src[offset .. offset + child.len()]; + let parse_res = match child.kind() { + NodeKind::Markup => Some(parse(src)), + _ => parse_block(src), + } + .and_then(|x| x.data().erroneous().not().then(|| x)); + + if let Some(parse_res) = parse_res { + parent.replace_child(child_idx, parse_res); + } else { + self.root = parse(&self.src); + } + } + None => { + self.root = parse(&self.src); + } + } + } } /// Provide highlighting categories for the given range of the source file. @@ -473,4 +504,21 @@ mod tests { // Test removing everything. test(TEST, 0 .. 21, "", ""); } + + #[test] + fn test_source_file_edit_2() { + #[track_caller] + fn test(prev: &str, range: Range, with: &str, after: &str) { + let mut source = SourceFile::detached(prev); + let result = SourceFile::detached(after); + dbg!(Green::from(source.root.clone())); + source.edit(range, with); + assert_eq!(source.src, result.src); + assert_eq!(source.line_starts, result.line_starts); + dbg!(Green::from(source.root)); + } + + // Test inserting at the begining. + test("abc #f()[def] ghi", 10 .. 11, "xyz", "abc #f()[dxyzf] ghi"); + } } -- cgit v1.2.3 From 7016ab0d123ba06d0bbc6ed5001fa02fbd261bfa Mon Sep 17 00:00:00 2001 From: Martin Haug Date: Wed, 3 Nov 2021 11:03:00 +0100 Subject: Make stuff more elegant --- src/source.rs | 52 ++++++++++++++++++++++------------------------------ 1 file changed, 22 insertions(+), 30 deletions(-) (limited to 'src/source.rs') diff --git a/src/source.rs b/src/source.rs index 069edd29..2d18ece3 100644 --- a/src/source.rs +++ b/src/source.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use std::io; -use std::ops::Range; +use std::ops::{Not, Range}; use std::path::{Path, PathBuf}; use std::rc::Rc; @@ -10,9 +10,9 @@ use serde::{Deserialize, Serialize}; use crate::diag::TypResult; use crate::loading::{FileHash, Loader}; -use crate::parse::{is_newline, parse, Scanner}; +use crate::parse::{is_newline, parse, parse_block, Scanner}; use crate::syntax::ast::Markup; -use crate::syntax::{self, Category, GreenNode, RedNode}; +use crate::syntax::{self, Category, GreenNode, NodeKind, RedNode, Span}; use crate::util::PathExt; #[cfg(feature = "codespan-reporting")] @@ -284,37 +284,28 @@ impl SourceFile { .extend(newlines(&self.src[start ..]).map(|idx| start + idx)); // Update the root node. - #[cfg(not(feature = "parse-cache"))] - { - self.root = parse(&self.src); - } - - #[cfg(feature = "parse-cache")] - { - let insertion_span = replace.into_span(self.id); - let incremental_target = - Rc::make_mut(&mut self.root).incremental_parent(insertion_span); - - match incremental_target { - Some((child_idx, parent, offset)) => { - let child = &parent.children()[child_idx]; - let src = &self.src[offset .. offset + child.len()]; - let parse_res = match child.kind() { - NodeKind::Markup => Some(parse(src)), - _ => parse_block(src), - } - .and_then(|x| x.data().erroneous().not().then(|| x)); - - if let Some(parse_res) = parse_res { - parent.replace_child(child_idx, parse_res); - } else { - self.root = parse(&self.src); - } + let insertion_span = Span::new(self.id, replace.start, replace.end); + let incremental_target = + Rc::make_mut(&mut self.root).incremental_parent(insertion_span); + + match incremental_target { + Some((child, offset)) => { + let src = &self.src[offset .. offset + child.len()]; + let parse_res = match child.kind() { + NodeKind::Markup => Some(parse(src)), + _ => parse_block(src), } - None => { + .and_then(|x| x.erroneous.not().then(|| x)); + + if let Some(parse_res) = parse_res { + *child = Rc::try_unwrap(parse_res).unwrap(); + } else { self.root = parse(&self.src); } } + None => { + self.root = parse(&self.src); + } } } @@ -405,6 +396,7 @@ impl<'a> Files<'a> for SourceStore { #[cfg(test)] mod tests { use super::*; + use crate::syntax::Green; const TEST: &str = "ä\tcde\nf💛g\r\nhi\rjkl"; -- cgit v1.2.3 From eba7fc34effbec3bcc6d5c40d831b1e15af77c4d Mon Sep 17 00:00:00 2001 From: Martin Haug Date: Sat, 6 Nov 2021 16:07:21 +0100 Subject: Incremental-safety based approach --- src/source.rs | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) (limited to 'src/source.rs') diff --git a/src/source.rs b/src/source.rs index 2d18ece3..f7e6cb5e 100644 --- a/src/source.rs +++ b/src/source.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use std::io; -use std::ops::{Not, Range}; +use std::ops::Range; use std::path::{Path, PathBuf}; use std::rc::Rc; @@ -10,9 +10,9 @@ use serde::{Deserialize, Serialize}; use crate::diag::TypResult; use crate::loading::{FileHash, Loader}; -use crate::parse::{is_newline, parse, parse_block, Scanner}; +use crate::parse::{is_newline, parse, Scanner}; use crate::syntax::ast::Markup; -use crate::syntax::{self, Category, GreenNode, NodeKind, RedNode, Span}; +use crate::syntax::{self, Category, GreenNode, RedNode, Span}; use crate::util::PathExt; #[cfg(feature = "codespan-reporting")] @@ -285,27 +285,10 @@ impl SourceFile { // Update the root node. let insertion_span = Span::new(self.id, replace.start, replace.end); - let incremental_target = - Rc::make_mut(&mut self.root).incremental_parent(insertion_span); - - match incremental_target { - Some((child, offset)) => { - let src = &self.src[offset .. offset + child.len()]; - let parse_res = match child.kind() { - NodeKind::Markup => Some(parse(src)), - _ => parse_block(src), - } - .and_then(|x| x.erroneous.not().then(|| x)); - - if let Some(parse_res) = parse_res { - *child = Rc::try_unwrap(parse_res).unwrap(); - } else { - self.root = parse(&self.src); - } - } - None => { - self.root = parse(&self.src); - } + let source = self.src().to_string(); + if !Rc::make_mut(&mut self.root).incremental(&source, insertion_span, with.len()) + { + self.root = parse(self.src()); } } @@ -511,6 +494,6 @@ mod tests { } // Test inserting at the begining. - test("abc #f()[def] ghi", 10 .. 11, "xyz", "abc #f()[dxyzf] ghi"); + test("abc #f()[def] ghi", 5 .. 6, "g", "abc #g()[def] ghi"); } } -- cgit v1.2.3 From 0663758fbb42651a08bfcd46c27b5cdeab90fb75 Mon Sep 17 00:00:00 2001 From: Martin Haug Date: Sun, 7 Nov 2021 19:43:01 +0100 Subject: Tests - length updates - dealing with keywords and comments --- src/source.rs | 102 +++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 90 insertions(+), 12 deletions(-) (limited to 'src/source.rs') diff --git a/src/source.rs b/src/source.rs index f7e6cb5e..36db50dd 100644 --- a/src/source.rs +++ b/src/source.rs @@ -128,6 +128,7 @@ pub struct SourceFile { src: String, line_starts: Vec, root: Rc, + was_incremental: bool, } impl SourceFile { @@ -141,6 +142,7 @@ impl SourceFile { root: parse(&src), src, line_starts, + was_incremental: false, } } @@ -286,12 +288,20 @@ impl SourceFile { // Update the root node. let insertion_span = Span::new(self.id, replace.start, replace.end); let source = self.src().to_string(); - if !Rc::make_mut(&mut self.root).incremental(&source, insertion_span, with.len()) - { + if Rc::make_mut(&mut self.root).incremental(&source, insertion_span, with.len()) { + self.was_incremental = true; + } else { self.root = parse(self.src()); + self.was_incremental = false; } } + /// Forces a non-incremental reparsing of the source file. + fn force_reparse(&mut self) { + self.root = parse(self.src()); + self.was_incremental = false; + } + /// Provide highlighting categories for the given range of the source file. pub fn highlight(&self, range: Range, mut f: F) where @@ -379,7 +389,6 @@ impl<'a> Files<'a> for SourceStore { #[cfg(test)] mod tests { use super::*; - use crate::syntax::Green; const TEST: &str = "ä\tcde\nf💛g\r\nhi\rjkl"; @@ -481,19 +490,88 @@ mod tests { } #[test] - fn test_source_file_edit_2() { + fn test_incremental_parse() { #[track_caller] - fn test(prev: &str, range: Range, with: &str, after: &str) { + fn test(prev: &str, range: Range, with: &str, incr: bool) { let mut source = SourceFile::detached(prev); - let result = SourceFile::detached(after); - dbg!(Green::from(source.root.clone())); source.edit(range, with); - assert_eq!(source.src, result.src); - assert_eq!(source.line_starts, result.line_starts); - dbg!(Green::from(source.root)); + + if incr { + assert!(source.was_incremental); + let incr_tree = source.root.clone(); + source.force_reparse(); + assert_eq!(source.root, incr_tree); + } else { + assert!(!source.was_incremental); + } } - // Test inserting at the begining. - test("abc #f()[def] ghi", 5 .. 6, "g", "abc #g()[def] ghi"); + // Test simple replacements. + test("hello world", 6 .. 11, "wankers", true); + test("{(0, 1, 2)}", 5 .. 6, "11pt", true); + test("= A heading", 3 .. 3, "n evocative", true); + test( + "#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", + 16 .. 20, + "none", + true, + ); + test( + "#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", + 33 .. 42, + "[_gronk_]", + true, + ); + test( + "#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", + 34 .. 41, + "_bar_", + true, + ); + test("{let i=1; for x in range(5) {i}}", 6 .. 6, " ", true); + test("{let i=1; for x in range(5) {i}}", 13 .. 14, " ", true); + test("hello {x}", 6 .. 9, "#f()", false); + test( + "this is -- in my opinion -- spectacular", + 8 .. 10, + "---", + true, + ); + test("understanding `code` is complicated", 15 .. 15, "C ", true); + test("{ let x = g() }", 10 .. 12, "f(54", true); + test( + "#let rect with (fill: eastern)", + 14 .. 29, + " (stroke: conifer", + true, + ); + test("a b c", 1 .. 1, " /* letters */", false); + + // Test the whitespace invariants. + test("hello \\ world", 7 .. 8, "a ", false); + test("hello \\ world", 7 .. 8, "\n\n", true); + test("x = y", 2 .. 2, "+ y ", true); + test("x = y", 2 .. 2, "+ y \n ", false); + test("abc\n= a heading", 3 .. 4, "\nsome more test\n\n", true); + test("abc\n= a heading", 3 .. 4, "\nnot ", false); + + // Test type invariants. + test("#for x in array {x}", 16 .. 19, "[#x]", true); + test("#let x = 1 {5}", 1 .. 4, "if", false); + test("#let x = 1 {5}", 4 .. 4, " if", false); + test("a // b c #f()", 3 .. 4, "", false); + + // this appearantly works but the assertion fails. + // test("a b c", 1 .. 1, "{[}", true); + + // Test unclosed things. + test(r#"{"hi"}"#, 4 .. 5, "c", false); + test(r"this \u{abcd}", 8 .. 9, "", true); + test(r"this \u{abcd} that", 12 .. 13, "", false); + test(r"{{let x = z}; a = 1} b", 6 .. 6, "//", false); + + // these appearantly works but the assertion fails. + // test(r#"a ```typst hello``` b"#, 16 .. 17, "", false); + // test(r#"a ```typst hello```"#, 16 .. 17, "", true); } } -- cgit v1.2.3 From 9141cba6a9db6ae3106e39d92508cb91c390049b Mon Sep 17 00:00:00 2001 From: Martin Haug Date: Mon, 8 Nov 2021 12:01:35 +0100 Subject: Deal with the effects of keywords --- src/source.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'src/source.rs') diff --git a/src/source.rs b/src/source.rs index 36db50dd..2bba86a9 100644 --- a/src/source.rs +++ b/src/source.rs @@ -558,11 +558,15 @@ mod tests { // Test type invariants. test("#for x in array {x}", 16 .. 19, "[#x]", true); test("#let x = 1 {5}", 1 .. 4, "if", false); + test("{let x = 1 {5}}", 1 .. 4, "if", true); test("#let x = 1 {5}", 4 .. 4, " if", false); + test("{let x = 1 {5}}", 4 .. 4, " if", true); test("a // b c #f()", 3 .. 4, "", false); + test("{\nf()\n//g(a)\n}", 6 .. 8, "", true); + test("{(1, 2)}", 1 .. 1, "while ", true); // this appearantly works but the assertion fails. - // test("a b c", 1 .. 1, "{[}", true); + test("a b c", 1 .. 1, "{[}", true); // Test unclosed things. test(r#"{"hi"}"#, 4 .. 5, "c", false); @@ -571,7 +575,7 @@ mod tests { test(r"{{let x = z}; a = 1} b", 6 .. 6, "//", false); // these appearantly works but the assertion fails. - // test(r#"a ```typst hello``` b"#, 16 .. 17, "", false); - // test(r#"a ```typst hello```"#, 16 .. 17, "", true); + test(r#"a ```typst hello``` b"#, 16 .. 17, "", false); + test(r#"a ```typst hello```"#, 16 .. 17, "", true); } } -- cgit v1.2.3 From 7a631d8b09bbffa8c7d90a1038d986876370ea7a Mon Sep 17 00:00:00 2001 From: Martin Haug Date: Tue, 9 Nov 2021 13:07:55 +0100 Subject: Simplify node mode management --- src/source.rs | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) (limited to 'src/source.rs') diff --git a/src/source.rs b/src/source.rs index 2bba86a9..3117f5b6 100644 --- a/src/source.rs +++ b/src/source.rs @@ -286,22 +286,15 @@ impl SourceFile { .extend(newlines(&self.src[start ..]).map(|idx| start + idx)); // Update the root node. - let insertion_span = Span::new(self.id, replace.start, replace.end); - let source = self.src().to_string(); - if Rc::make_mut(&mut self.root).incremental(&source, insertion_span, with.len()) { + let span = Span::new(self.id, replace.start, replace.end); + if Rc::make_mut(&mut self.root).incremental(&self.src, span, with.len()) { self.was_incremental = true; } else { - self.root = parse(self.src()); + self.root = parse(&self.src); self.was_incremental = false; } } - /// Forces a non-incremental reparsing of the source file. - fn force_reparse(&mut self) { - self.root = parse(self.src()); - self.was_incremental = false; - } - /// Provide highlighting categories for the given range of the source file. pub fn highlight(&self, range: Range, mut f: F) where @@ -499,8 +492,7 @@ mod tests { if incr { assert!(source.was_incremental); let incr_tree = source.root.clone(); - source.force_reparse(); - assert_eq!(source.root, incr_tree); + assert_eq!(parse(source.src()), incr_tree); } else { assert!(!source.was_incremental); } -- cgit v1.2.3 From 91f2f97572c64d7eb25c88ad0ebb18192cf8eddf Mon Sep 17 00:00:00 2001 From: Martin Haug Date: Tue, 9 Nov 2021 13:34:23 +0100 Subject: Multiple replacements, escapes --- src/source.rs | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/source.rs') diff --git a/src/source.rs b/src/source.rs index 3117f5b6..797e815b 100644 --- a/src/source.rs +++ b/src/source.rs @@ -500,6 +500,8 @@ mod tests { // Test simple replacements. test("hello world", 6 .. 11, "wankers", true); + test("a d e", 1 .. 3, " b c d", true); + test("a #f() e", 1 .. 6, " b c d", false); test("{(0, 1, 2)}", 5 .. 6, "11pt", true); test("= A heading", 3 .. 3, "n evocative", true); test( @@ -546,6 +548,8 @@ mod tests { test("x = y", 2 .. 2, "+ y \n ", false); test("abc\n= a heading", 3 .. 4, "\nsome more test\n\n", true); test("abc\n= a heading", 3 .. 4, "\nnot ", false); + test("hey #myfriend", 4 .. 4, "\\", false); + test("hey #myfriend", 4 .. 4, "\\", true); // Test type invariants. test("#for x in array {x}", 16 .. 19, "[#x]", true); -- cgit v1.2.3 From 3162c6a83a910f34d6ed7e966c11b7e7b5bd4088 Mon Sep 17 00:00:00 2001 From: Martin Haug Date: Wed, 10 Nov 2021 20:41:10 +0100 Subject: Comments and neighbors --- src/source.rs | 135 ++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 75 insertions(+), 60 deletions(-) (limited to 'src/source.rs') diff --git a/src/source.rs b/src/source.rs index 797e815b..7eb1d3a7 100644 --- a/src/source.rs +++ b/src/source.rs @@ -128,7 +128,6 @@ pub struct SourceFile { src: String, line_starts: Vec, root: Rc, - was_incremental: bool, } impl SourceFile { @@ -142,7 +141,6 @@ impl SourceFile { root: parse(&src), src, line_starts, - was_incremental: false, } } @@ -268,7 +266,7 @@ impl SourceFile { /// Edit the source file by replacing the given range. /// /// This panics if the `replace` range is out of bounds. - pub fn edit(&mut self, replace: Range, with: &str) { + pub fn edit(&mut self, replace: Range, with: &str) -> Range { let start = replace.start; self.src.replace_range(replace.clone(), with); @@ -287,11 +285,13 @@ impl SourceFile { // Update the root node. let span = Span::new(self.id, replace.start, replace.end); - if Rc::make_mut(&mut self.root).incremental(&self.src, span, with.len()) { - self.was_incremental = true; + if let Ok(range) = + Rc::make_mut(&mut self.root).incremental(&self.src, span, with.len()) + { + range } else { self.root = parse(&self.src); - self.was_incremental = false; + 0 .. self.src.len() } } @@ -485,93 +485,108 @@ mod tests { #[test] fn test_incremental_parse() { #[track_caller] - fn test(prev: &str, range: Range, with: &str, incr: bool) { + fn test(prev: &str, range: Range, with: &str, incr: Range) { let mut source = SourceFile::detached(prev); - source.edit(range, with); + let range = source.edit(range, with); + assert_eq!(range, incr); - if incr { - assert!(source.was_incremental); - let incr_tree = source.root.clone(); - assert_eq!(parse(source.src()), incr_tree); - } else { - assert!(!source.was_incremental); - } + let incr_tree = source.root.clone(); + assert_eq!(parse(source.src()), incr_tree); } // Test simple replacements. - test("hello world", 6 .. 11, "wankers", true); - test("a d e", 1 .. 3, " b c d", true); - test("a #f() e", 1 .. 6, " b c d", false); - test("{(0, 1, 2)}", 5 .. 6, "11pt", true); - test("= A heading", 3 .. 3, "n evocative", true); + test("hello world", 6 .. 11, "wankers", 5 .. 13); + test("a d e", 1 .. 3, " b c d", 0 .. 8); + test("a #f() e", 1 .. 6, " b c d", 0 .. 8); + test("{(0, 1, 2)}", 5 .. 6, "11pt", 5 .. 9); + test("= A heading", 3 .. 3, "n evocative", 2 .. 15); + test("your thing", 5 .. 5, "a", 4 .. 11); + test("a your thing a", 6 .. 7, "a", 2 .. 12); test( "#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", 16 .. 20, "none", - true, + 16 .. 20, ); test( "#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", 33 .. 42, "[_gronk_]", - true, + 33 .. 42, ); test( "#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", 34 .. 41, "_bar_", - true, + 34 .. 39, ); - test("{let i=1; for x in range(5) {i}}", 6 .. 6, " ", true); - test("{let i=1; for x in range(5) {i}}", 13 .. 14, " ", true); - test("hello {x}", 6 .. 9, "#f()", false); + test("{let i=1; for x in range(5) {i}}", 6 .. 6, " ", 1 .. 9); + test("{let i=1; for x in range(5) {i}}", 13 .. 14, " ", 13 .. 15); + test("hello {x}", 6 .. 9, "#f()", 5 .. 10); test( "this is -- in my opinion -- spectacular", 8 .. 10, "---", - true, + 7 .. 12, + ); + test( + "understanding `code` is complicated", + 15 .. 15, + "C ", + 14 .. 22, ); - test("understanding `code` is complicated", 15 .. 15, "C ", true); - test("{ let x = g() }", 10 .. 12, "f(54", true); + test("{ let x = g() }", 10 .. 12, "f(54", 2 .. 15); test( - "#let rect with (fill: eastern)", - 14 .. 29, + "a #let rect with (fill: eastern)\nb", + 16 .. 31, " (stroke: conifer", - true, + 2 .. 34, ); - test("a b c", 1 .. 1, " /* letters */", false); // Test the whitespace invariants. - test("hello \\ world", 7 .. 8, "a ", false); - test("hello \\ world", 7 .. 8, "\n\n", true); - test("x = y", 2 .. 2, "+ y ", true); - test("x = y", 2 .. 2, "+ y \n ", false); - test("abc\n= a heading", 3 .. 4, "\nsome more test\n\n", true); - test("abc\n= a heading", 3 .. 4, "\nnot ", false); - test("hey #myfriend", 4 .. 4, "\\", false); - test("hey #myfriend", 4 .. 4, "\\", true); + test("hello \\ world", 7 .. 8, "a ", 6 .. 14); + test("hello \\ world", 7 .. 8, " a", 6 .. 14); + test("x = y", 1 .. 1, " + y", 0 .. 6); + test("x = y", 1 .. 1, " + y\n", 0 .. 10); + test("abc\n= a heading\njoke", 3 .. 4, "\nmore\n\n", 0 .. 21); + test("abc\n= a heading\njoke", 3 .. 4, "\nnot ", 0 .. 19); + test("hey #myfriend", 4 .. 4, "\\", 0 .. 14); + test("hey #myfriend", 4 .. 4, "\\", 3 .. 6); // Test type invariants. - test("#for x in array {x}", 16 .. 19, "[#x]", true); - test("#let x = 1 {5}", 1 .. 4, "if", false); - test("{let x = 1 {5}}", 1 .. 4, "if", true); - test("#let x = 1 {5}", 4 .. 4, " if", false); - test("{let x = 1 {5}}", 4 .. 4, " if", true); - test("a // b c #f()", 3 .. 4, "", false); - test("{\nf()\n//g(a)\n}", 6 .. 8, "", true); - test("{(1, 2)}", 1 .. 1, "while ", true); - - // this appearantly works but the assertion fails. - test("a b c", 1 .. 1, "{[}", true); + test("a #for x in array {x}", 18 .. 21, "[#x]", 2 .. 22); + test("a #let x = 1 {5}", 3 .. 6, "if", 0 .. 15); + test("a {let x = 1 {5}} b", 3 .. 6, "if", 2 .. 16); + test("#let x = 1 {5}", 4 .. 4, " if", 0 .. 17); + test("{let x = 1 {5}}", 4 .. 4, " if", 0 .. 18); + test("a // b c #f()", 3 .. 4, "", 0 .. 12); + test("{\nf()\n//g(a)\n}", 6 .. 8, "", 0 .. 12); + test("a{\nf()\n//g(a)\n}b", 7 .. 9, "", 1 .. 13); + test("a #while x {\n g(x) \n} b", 11 .. 11, "//", 0 .. 26); + test("{(1, 2)}", 1 .. 1, "while ", 0 .. 14); + test("a b c", 1 .. 1, "{[}", 0 .. 5); // Test unclosed things. - test(r#"{"hi"}"#, 4 .. 5, "c", false); - test(r"this \u{abcd}", 8 .. 9, "", true); - test(r"this \u{abcd} that", 12 .. 13, "", false); - test(r"{{let x = z}; a = 1} b", 6 .. 6, "//", false); - - // these appearantly works but the assertion fails. - test(r#"a ```typst hello``` b"#, 16 .. 17, "", false); - test(r#"a ```typst hello```"#, 16 .. 17, "", true); + test(r#"{"hi"}"#, 4 .. 5, "c", 0 .. 6); + test(r"this \u{abcd}", 8 .. 9, "", 5 .. 12); + test(r"this \u{abcd} that", 12 .. 13, "", 0 .. 17); + test(r"{{let x = z}; a = 1} b", 6 .. 6, "//", 0 .. 24); + test("a b c", 1 .. 1, " /* letters */", 0 .. 16); + test("a b c", 1 .. 1, " /* letters", 0 .. 16); + test( + "{if i==1 {a} else [b]; b()}", + 12 .. 12, + " /* letters */", + 1 .. 35, + ); + test( + "{if i==1 {a} else [b]; b()}", + 12 .. 12, + " /* letters", + 0 .. 38, + ); + + test(r#"a ```typst hello``` b"#, 16 .. 17, "", 0 .. 20); + test(r#"a ```typst hello```"#, 16 .. 17, "", 2 .. 18); } } -- cgit v1.2.3 From fdb9d0743d73c278136b9254286fdc4be71c42a5 Mon Sep 17 00:00:00 2001 From: Martin Haug Date: Thu, 18 Nov 2021 16:21:45 +0100 Subject: Refactoring and bugfixes --- src/source.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'src/source.rs') diff --git a/src/source.rs b/src/source.rs index 7eb1d3a7..aaf009e0 100644 --- a/src/source.rs +++ b/src/source.rs @@ -12,7 +12,7 @@ use crate::diag::TypResult; use crate::loading::{FileHash, Loader}; use crate::parse::{is_newline, parse, Scanner}; use crate::syntax::ast::Markup; -use crate::syntax::{self, Category, GreenNode, RedNode, Span}; +use crate::syntax::{self, Category, GreenNode, RedNode, Reparser, Span}; use crate::util::PathExt; #[cfg(feature = "codespan-reporting")] @@ -285,9 +285,8 @@ impl SourceFile { // Update the root node. let span = Span::new(self.id, replace.start, replace.end); - if let Ok(range) = - Rc::make_mut(&mut self.root).incremental(&self.src, span, with.len()) - { + let reparser = Reparser::new(&self.src, span, with.len()); + if let Ok(range) = reparser.incremental(Rc::make_mut(&mut self.root)) { range } else { self.root = parse(&self.src); @@ -502,6 +501,14 @@ mod tests { test("= A heading", 3 .. 3, "n evocative", 2 .. 15); test("your thing", 5 .. 5, "a", 4 .. 11); test("a your thing a", 6 .. 7, "a", 2 .. 12); + test("{call(); abc}", 7 .. 7, "[]", 0 .. 15); + test("#call() abc", 7 .. 7, "[]", 0 .. 13); + // test( + // "hi\n- item\n- item 2\n - item 3", + // 10 .. 10, + // " ", + // 9 .. 33, + // ); test( "#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", 16 .. 20, @@ -535,7 +542,7 @@ mod tests { "C ", 14 .. 22, ); - test("{ let x = g() }", 10 .. 12, "f(54", 2 .. 15); + test("{ let x = g() }", 10 .. 12, "f(54", 0 .. 17); test( "a #let rect with (fill: eastern)\nb", 16 .. 31, -- cgit v1.2.3 From edc686d7384470068858e16f2926cf50f31b2c90 Mon Sep 17 00:00:00 2001 From: Martin Haug Date: Sat, 27 Nov 2021 16:10:22 +0100 Subject: Make incremental parsing simpler and move it somewhere else --- src/source.rs | 128 +++++----------------------------------------------------- 1 file changed, 9 insertions(+), 119 deletions(-) (limited to 'src/source.rs') diff --git a/src/source.rs b/src/source.rs index aaf009e0..421412ee 100644 --- a/src/source.rs +++ b/src/source.rs @@ -10,9 +10,9 @@ use serde::{Deserialize, Serialize}; use crate::diag::TypResult; use crate::loading::{FileHash, Loader}; -use crate::parse::{is_newline, parse, Scanner}; +use crate::parse::{is_newline, parse, Reparser, Scanner}; use crate::syntax::ast::Markup; -use crate::syntax::{self, Category, GreenNode, RedNode, Reparser, Span}; +use crate::syntax::{self, Category, GreenNode, RedNode, Span}; use crate::util::PathExt; #[cfg(feature = "codespan-reporting")] @@ -286,7 +286,7 @@ impl SourceFile { // Update the root node. let span = Span::new(self.id, replace.start, replace.end); let reparser = Reparser::new(&self.src, span, with.len()); - if let Ok(range) = reparser.incremental(Rc::make_mut(&mut self.root)) { + if let Ok(range) = reparser.reparse(Rc::make_mut(&mut self.root)) { range } else { self.root = parse(&self.src); @@ -302,6 +302,12 @@ impl SourceFile { let red = RedNode::from_root(self.root.clone(), self.id); syntax::highlight(red.as_ref(), range, &mut f) } + + /// Obtain a reference to the source's root green node. + #[cfg(test)] + pub(crate) fn root(&self) -> Rc { + self.root.clone() + } } /// The indices at which lines start (right behind newlines). @@ -480,120 +486,4 @@ mod tests { // Test removing everything. test(TEST, 0 .. 21, "", ""); } - - #[test] - fn test_incremental_parse() { - #[track_caller] - fn test(prev: &str, range: Range, with: &str, incr: Range) { - let mut source = SourceFile::detached(prev); - let range = source.edit(range, with); - assert_eq!(range, incr); - - let incr_tree = source.root.clone(); - assert_eq!(parse(source.src()), incr_tree); - } - - // Test simple replacements. - test("hello world", 6 .. 11, "wankers", 5 .. 13); - test("a d e", 1 .. 3, " b c d", 0 .. 8); - test("a #f() e", 1 .. 6, " b c d", 0 .. 8); - test("{(0, 1, 2)}", 5 .. 6, "11pt", 5 .. 9); - test("= A heading", 3 .. 3, "n evocative", 2 .. 15); - test("your thing", 5 .. 5, "a", 4 .. 11); - test("a your thing a", 6 .. 7, "a", 2 .. 12); - test("{call(); abc}", 7 .. 7, "[]", 0 .. 15); - test("#call() abc", 7 .. 7, "[]", 0 .. 13); - // test( - // "hi\n- item\n- item 2\n - item 3", - // 10 .. 10, - // " ", - // 9 .. 33, - // ); - test( - "#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", - 16 .. 20, - "none", - 16 .. 20, - ); - test( - "#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", - 33 .. 42, - "[_gronk_]", - 33 .. 42, - ); - test( - "#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", - 34 .. 41, - "_bar_", - 34 .. 39, - ); - test("{let i=1; for x in range(5) {i}}", 6 .. 6, " ", 1 .. 9); - test("{let i=1; for x in range(5) {i}}", 13 .. 14, " ", 13 .. 15); - test("hello {x}", 6 .. 9, "#f()", 5 .. 10); - test( - "this is -- in my opinion -- spectacular", - 8 .. 10, - "---", - 7 .. 12, - ); - test( - "understanding `code` is complicated", - 15 .. 15, - "C ", - 14 .. 22, - ); - test("{ let x = g() }", 10 .. 12, "f(54", 0 .. 17); - test( - "a #let rect with (fill: eastern)\nb", - 16 .. 31, - " (stroke: conifer", - 2 .. 34, - ); - - // Test the whitespace invariants. - test("hello \\ world", 7 .. 8, "a ", 6 .. 14); - test("hello \\ world", 7 .. 8, " a", 6 .. 14); - test("x = y", 1 .. 1, " + y", 0 .. 6); - test("x = y", 1 .. 1, " + y\n", 0 .. 10); - test("abc\n= a heading\njoke", 3 .. 4, "\nmore\n\n", 0 .. 21); - test("abc\n= a heading\njoke", 3 .. 4, "\nnot ", 0 .. 19); - test("hey #myfriend", 4 .. 4, "\\", 0 .. 14); - test("hey #myfriend", 4 .. 4, "\\", 3 .. 6); - - // Test type invariants. - test("a #for x in array {x}", 18 .. 21, "[#x]", 2 .. 22); - test("a #let x = 1 {5}", 3 .. 6, "if", 0 .. 15); - test("a {let x = 1 {5}} b", 3 .. 6, "if", 2 .. 16); - test("#let x = 1 {5}", 4 .. 4, " if", 0 .. 17); - test("{let x = 1 {5}}", 4 .. 4, " if", 0 .. 18); - test("a // b c #f()", 3 .. 4, "", 0 .. 12); - test("{\nf()\n//g(a)\n}", 6 .. 8, "", 0 .. 12); - test("a{\nf()\n//g(a)\n}b", 7 .. 9, "", 1 .. 13); - test("a #while x {\n g(x) \n} b", 11 .. 11, "//", 0 .. 26); - test("{(1, 2)}", 1 .. 1, "while ", 0 .. 14); - test("a b c", 1 .. 1, "{[}", 0 .. 5); - - // Test unclosed things. - test(r#"{"hi"}"#, 4 .. 5, "c", 0 .. 6); - test(r"this \u{abcd}", 8 .. 9, "", 5 .. 12); - test(r"this \u{abcd} that", 12 .. 13, "", 0 .. 17); - test(r"{{let x = z}; a = 1} b", 6 .. 6, "//", 0 .. 24); - test("a b c", 1 .. 1, " /* letters */", 0 .. 16); - test("a b c", 1 .. 1, " /* letters", 0 .. 16); - test( - "{if i==1 {a} else [b]; b()}", - 12 .. 12, - " /* letters */", - 1 .. 35, - ); - test( - "{if i==1 {a} else [b]; b()}", - 12 .. 12, - " /* letters", - 0 .. 38, - ); - - test(r#"a ```typst hello``` b"#, 16 .. 17, "", 0 .. 20); - test(r#"a ```typst hello```"#, 16 .. 17, "", 2 .. 18); - } } -- cgit v1.2.3 From e05eb5fda5d1dfeef168b6fc071b20fdbcce2dcd Mon Sep 17 00:00:00 2001 From: Martin Haug Date: Sun, 28 Nov 2021 18:18:45 +0100 Subject: Code Review: Parser, I can't let you do this --- src/source.rs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) (limited to 'src/source.rs') diff --git a/src/source.rs b/src/source.rs index 421412ee..6cca9f75 100644 --- a/src/source.rs +++ b/src/source.rs @@ -12,7 +12,7 @@ use crate::diag::TypResult; use crate::loading::{FileHash, Loader}; use crate::parse::{is_newline, parse, Reparser, Scanner}; use crate::syntax::ast::Markup; -use crate::syntax::{self, Category, GreenNode, RedNode, Span}; +use crate::syntax::{self, Category, GreenNode, RedNode}; use crate::util::PathExt; #[cfg(feature = "codespan-reporting")] @@ -265,7 +265,8 @@ impl SourceFile { /// Edit the source file by replacing the given range. /// - /// This panics if the `replace` range is out of bounds. + /// Returns the range of the section in the new source that was ultimately + /// reparsed. The method panics if the `replace` range is out of bounds. pub fn edit(&mut self, replace: Range, with: &str) -> Range { let start = replace.start; self.src.replace_range(replace.clone(), with); @@ -284,9 +285,8 @@ impl SourceFile { .extend(newlines(&self.src[start ..]).map(|idx| start + idx)); // Update the root node. - let span = Span::new(self.id, replace.start, replace.end); - let reparser = Reparser::new(&self.src, span, with.len()); - if let Ok(range) = reparser.reparse(Rc::make_mut(&mut self.root)) { + let reparser = Reparser::new(&self.src, replace, with.len()); + if let Some(range) = reparser.reparse(Rc::make_mut(&mut self.root)) { range } else { self.root = parse(&self.src); @@ -302,12 +302,6 @@ impl SourceFile { let red = RedNode::from_root(self.root.clone(), self.id); syntax::highlight(red.as_ref(), range, &mut f) } - - /// Obtain a reference to the source's root green node. - #[cfg(test)] - pub(crate) fn root(&self) -> Rc { - self.root.clone() - } } /// The indices at which lines start (right behind newlines). -- cgit v1.2.3 From c994cfa7d814e3909682b19322867ed5c676c453 Mon Sep 17 00:00:00 2001 From: Martin Haug Date: Mon, 3 Jan 2022 23:18:21 +0100 Subject: Code Review: Your parsers were so preoccupied with whether they could --- src/source.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'src/source.rs') diff --git a/src/source.rs b/src/source.rs index 6cca9f75..7afeaa8a 100644 --- a/src/source.rs +++ b/src/source.rs @@ -154,9 +154,14 @@ impl SourceFile { &self.root } + /// The root red node of the file's untyped red tree. + pub fn red(&self) -> RedNode { + RedNode::from_root(self.root.clone(), self.id) + } + /// The root node of the file's typed abstract syntax tree. pub fn ast(&self) -> TypResult { - let red = RedNode::from_root(self.root.clone(), self.id); + let red = self.red(); let errors = red.errors(); if errors.is_empty() { Ok(red.cast().unwrap()) @@ -284,14 +289,8 @@ impl SourceFile { self.line_starts .extend(newlines(&self.src[start ..]).map(|idx| start + idx)); - // Update the root node. - let reparser = Reparser::new(&self.src, replace, with.len()); - if let Some(range) = reparser.reparse(Rc::make_mut(&mut self.root)) { - range - } else { - self.root = parse(&self.src); - 0 .. self.src.len() - } + // Incrementally reparse the replaced range. + Reparser::new(&self.src, replace, with.len()).reparse(&mut self.root) } /// Provide highlighting categories for the given range of the source file. -- cgit v1.2.3