summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMartin Haug <mhaug@live.de>2021-11-18 16:21:45 +0100
committerMartin Haug <mhaug@live.de>2021-11-18 16:21:45 +0100
commitfdb9d0743d73c278136b9254286fdc4be71c42a5 (patch)
tree1ae3c19d13b7cd93477aac908e1561400dcf9092
parent3162c6a83a910f34d6ed7e966c11b7e7b5bd4088 (diff)
Refactoring and bugfixes
-rw-r--r--Cargo.lock420
-rw-r--r--Cargo.toml17
-rw-r--r--benches/timed.rs98
-rw-r--r--src/parse/mod.rs34
-rw-r--r--src/parse/parser.rs2
-rw-r--r--src/source.rs17
-rw-r--r--src/syntax/incremental.rs515
-rw-r--r--src/syntax/mod.rs563
8 files changed, 1101 insertions, 565 deletions
diff --git a/Cargo.lock b/Cargo.lock
index df3bf74a..98cb4d81 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -39,6 +39,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
[[package]]
+name = "atty"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "winapi",
+]
+
+[[package]]
name = "autocfg"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -57,6 +68,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
+name = "bstr"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
+dependencies = [
+ "lazy_static",
+ "memchr",
+ "regex-automata",
+ "serde",
+]
+
+[[package]]
+name = "bumpalo"
+version = "3.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f1e260c3a9040a7c19a12468758f4c16f31a81a1fe087482be9570ec864bb6c"
+
+[[package]]
name = "bytemuck"
version = "1.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -69,12 +98,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
+name = "cast"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c24dab4283a142afa2fdca129b80ad2c6284e073930f964c3a1293c225ee39a"
+dependencies = [
+ "rustc_version",
+]
+
+[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
+name = "clap"
+version = "2.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
+dependencies = [
+ "bitflags",
+ "textwrap",
+ "unicode-width",
+]
+
+[[package]]
name = "codespan-reporting"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -100,6 +149,108 @@ dependencies = [
]
[[package]]
+name = "criterion"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1604dafd25fba2fe2d5895a9da139f8dc9b319a5fe5354ca137cbbce4e178d10"
+dependencies = [
+ "atty",
+ "cast",
+ "clap",
+ "criterion-plot",
+ "csv",
+ "itertools",
+ "lazy_static",
+ "num-traits",
+ "oorandom",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_cbor",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d00996de9f2f7559f7f4dc286073197f83e92256a59ed395f9aac01fe717da57"
+dependencies = [
+ "cast",
+ "itertools",
+]
+
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
+dependencies = [
+ "cfg-if",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+ "lazy_static",
+ "memoffset",
+ "scopeguard",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db"
+dependencies = [
+ "cfg-if",
+ "lazy_static",
+]
+
+[[package]]
+name = "csv"
+version = "1.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1"
+dependencies = [
+ "bstr",
+ "csv-core",
+ "itoa 0.4.8",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "csv-core"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
name = "data-url"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -277,6 +428,21 @@ dependencies = [
]
[[package]]
+name = "half"
+version = "1.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
+
+[[package]]
+name = "hermit-abi"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+dependencies = [
+ "libc",
+]
+
+[[package]]
name = "iai"
version = "0.1.1"
source = "git+https://github.com/reknih/iai#f57a4452e7ac3b118e695e1feddb6ea02c4015be"
@@ -316,12 +482,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
[[package]]
+name = "itoa"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
+
+[[package]]
name = "jpeg-decoder"
version = "0.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2"
[[package]]
+name = "js-sys"
+version = "0.3.55"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7cc9ffccd38c451a86bf13657df244e9c3f37493cce8e5e21e940963777acc84"
+dependencies = [
+ "wasm-bindgen",
+]
+
+[[package]]
name = "kurbo"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -331,6 +512,12 @@ dependencies = [
]
[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
name = "libc"
version = "0.2.112"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -352,6 +539,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
[[package]]
+name = "memchr"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
+
+[[package]]
name = "memmap2"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -361,6 +554,15 @@ dependencies = [
]
[[package]]
+name = "memoffset"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
name = "miniz_oxide"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -421,19 +623,35 @@ dependencies = [
]
[[package]]
+name = "num_cpus"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
+[[package]]
name = "once_cell"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5"
[[package]]
+name = "oorandom"
+version = "11.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
+
+[[package]]
name = "pdf-writer"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36d760a6f2ac90811cba1006a298e8a7e5ce2c922bb5dc7f7000911a4a6b60f4"
dependencies = [
"bitflags",
- "itoa",
+ "itoa 0.4.8",
"ryu",
]
@@ -444,6 +662,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db8bcd96cb740d03149cbad5518db9fd87126a10ab519c011893b1754134c468"
[[package]]
+name = "plotters"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a3fd9ec30b9749ce28cd91f255d569591cdf937fe280c312143e3c4bad6f2a"
+dependencies = [
+ "num-traits",
+ "plotters-backend",
+ "plotters-svg",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "plotters-backend"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d88417318da0eaf0fdcdb51a0ee6c3bed624333bff8f946733049380be67ac1c"
+
+[[package]]
+name = "plotters-svg"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "521fa9638fa597e1dc53e9412a4f9cefb01187ee1f7413076f9e6749e2885ba9"
+dependencies = [
+ "plotters-backend",
+]
+
+[[package]]
name = "png"
version = "0.16.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -533,6 +779,31 @@ dependencies = [
]
[[package]]
+name = "rayon"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90"
+dependencies = [
+ "autocfg",
+ "crossbeam-deque",
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-utils",
+ "lazy_static",
+ "num_cpus",
+]
+
+[[package]]
name = "rctree"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -558,6 +829,27 @@ dependencies = [
]
[[package]]
+name = "regex"
+version = "1.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461"
+dependencies = [
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
+
+[[package]]
name = "resvg"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -591,6 +883,15 @@ dependencies = [
]
[[package]]
+name = "rustc_version"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
+dependencies = [
+ "semver",
+]
+
+[[package]]
name = "rustybuzz"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -631,6 +932,18 @@ dependencies = [
]
[[package]]
+name = "scopeguard"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+
+[[package]]
+name = "semver"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012"
+
+[[package]]
name = "serde"
version = "1.0.132"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -640,6 +953,16 @@ dependencies = [
]
[[package]]
+name = "serde_cbor"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5"
+dependencies = [
+ "half",
+ "serde",
+]
+
+[[package]]
name = "serde_derive"
version = "1.0.132"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -651,6 +974,17 @@ dependencies = [
]
[[package]]
+name = "serde_json"
+version = "1.0.74"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee2bb9cd061c5865d345bb02ca49fcef1391741b672b54a0bf7b679badec3142"
+dependencies = [
+ "itoa 1.0.1",
+ "ryu",
+ "serde",
+]
+
+[[package]]
name = "simplecss"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -713,6 +1047,15 @@ dependencies = [
]
[[package]]
+name = "textwrap"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
+dependencies = [
+ "unicode-width",
+]
+
+[[package]]
name = "thiserror"
version = "1.0.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -747,6 +1090,16 @@ dependencies = [
]
[[package]]
+name = "tinytemplate"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
+[[package]]
name = "ttf-parser"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -758,6 +1111,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"codespan-reporting",
+ "criterion",
"dirs",
"filedescriptor",
"fxhash",
@@ -893,6 +1247,70 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
+name = "wasm-bindgen"
+version = "0.2.78"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "632f73e236b219150ea279196e54e610f5dbafa5d61786303d4da54f84e47fce"
+dependencies = [
+ "cfg-if",
+ "wasm-bindgen-macro",
+]
+
+[[package]]
+name = "wasm-bindgen-backend"
+version = "0.2.78"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a317bf8f9fba2476b4b2c85ef4c4af8ff39c3c7f0cdfeed4f82c34a880aa837b"
+dependencies = [
+ "bumpalo",
+ "lazy_static",
+ "log",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.78"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.78"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-backend",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.78"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0237232789cf037d5480773fe568aac745bfe2afbc11a863e97901780a6b47cc"
+
+[[package]]
+name = "web-sys"
+version = "0.3.55"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38eb105f1c59d9eaa6b5cdc92b859d85b926e82cb2e0945cd0c9259faa6fe9fb"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/Cargo.toml b/Cargo.toml
index 0bf68d74..5c4dddcb 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -20,14 +20,21 @@ opt-level = 2
[dependencies]
fxhash = "0.2"
-image = { version = "0.23", default-features = false, features = ["png", "jpeg"] }
+image = { version = "0.23", default-features = false, features = [
+ "png",
+ "jpeg",
+] }
itertools = "0.10"
miniz_oxide = "0.4"
once_cell = "1"
pdf-writer = "0.4"
rustybuzz = "0.4"
serde = { version = "1", features = ["derive", "rc"] }
-svg2pdf = { version = "0.1", default-features = false, features = ["text", "png", "jpeg"] }
+svg2pdf = { version = "0.1", default-features = false, features = [
+ "text",
+ "png",
+ "jpeg",
+] }
ttf-parser = "0.12"
typst-macros = { path = "./macros" }
unicode-bidi = "0.3.5"
@@ -54,6 +61,7 @@ walkdir = "2"
# Dependencies updates:
# - Bump ttf-parser when rustybuzz is updated
# - Bump usvg and resvg in conjunction with svg2pdf
+criterion = "0.3"
[[bin]]
name = "typst"
@@ -68,3 +76,8 @@ harness = false
name = "oneshot"
path = "benches/oneshot.rs"
harness = false
+
+[[bench]]
+name = "timed"
+path = "benches/timed.rs"
+harness = false
diff --git a/benches/timed.rs b/benches/timed.rs
new file mode 100644
index 00000000..83820af2
--- /dev/null
+++ b/benches/timed.rs
@@ -0,0 +1,98 @@
+use std::path::Path;
+
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+
+use typst::eval::eval;
+use typst::layout::layout;
+use typst::loading::MemLoader;
+use typst::parse::{parse, Scanner, TokenMode, Tokens};
+use typst::source::SourceId;
+use typst::Context;
+
+const SRC: &str = include_str!("bench.typ");
+const FONT: &[u8] = include_bytes!("../fonts/IBMPlexSans-Regular.ttf");
+
+fn context() -> (Context, SourceId) {
+ let loader = MemLoader::new().with(Path::new("font.ttf"), FONT).wrap();
+ let mut ctx = Context::new(loader);
+ let id = ctx.sources.provide(Path::new("src.typ"), SRC.to_string());
+ (ctx, id)
+}
+
+fn bench_decode(c: &mut Criterion) {
+ c.bench_function("decode", |b| {
+ b.iter(|| {
+ // We don't use chars().count() because that has a special
+ // superfast implementation.
+ let mut count = 0;
+ let mut chars = black_box(SRC).chars();
+ while let Some(_) = chars.next() {
+ count += 1;
+ }
+ count
+ })
+ });
+}
+
+fn bench_scan(c: &mut Criterion) {
+ c.bench_function("scan", |b| {
+ b.iter(|| {
+ let mut count = 0;
+ let mut scanner = Scanner::new(black_box(SRC));
+ while let Some(_) = scanner.eat() {
+ count += 1;
+ }
+ count
+ })
+ });
+}
+
+fn bench_tokenize(c: &mut Criterion) {
+ c.bench_function("tokenize", |b| {
+ b.iter(|| Tokens::new(black_box(SRC), black_box(TokenMode::Markup)).count())
+ });
+}
+
+fn bench_parse(c: &mut Criterion) {
+ c.bench_function("parse", |b| b.iter(|| parse(SRC)));
+}
+
+fn bench_edit(c: &mut Criterion) {
+ let (mut ctx, id) = context();
+ c.bench_function("edit", |b| {
+ b.iter(|| black_box(ctx.sources.edit(id, 1168 .. 1171, "_Uhr_")))
+ });
+}
+
+fn bench_eval(c: &mut Criterion) {
+ let (mut ctx, id) = context();
+ let ast = ctx.sources.get(id).ast().unwrap();
+ c.bench_function("eval", |b| b.iter(|| eval(&mut ctx, id, &ast).unwrap()));
+}
+
+fn bench_to_tree(c: &mut Criterion) {
+ let (mut ctx, id) = context();
+ let module = ctx.evaluate(id).unwrap();
+ c.bench_function("to_tree", |b| {
+ b.iter(|| module.template.to_pages(ctx.style()))
+ });
+}
+
+fn bench_layout(c: &mut Criterion) {
+ let (mut ctx, id) = context();
+ let tree = ctx.execute(id).unwrap();
+ c.bench_function("layout", |b| b.iter(|| layout(&mut ctx, &tree)));
+}
+
+criterion_group!(
+ benches,
+ bench_decode,
+ bench_scan,
+ bench_tokenize,
+ bench_parse,
+ bench_edit,
+ bench_eval,
+ bench_to_tree,
+ bench_layout
+);
+criterion_main!(benches);
diff --git a/src/parse/mod.rs b/src/parse/mod.rs
index f2fae5f2..f1f1e8b6 100644
--- a/src/parse/mod.rs
+++ b/src/parse/mod.rs
@@ -32,6 +32,13 @@ pub fn parse_atomic(src: &str, _: bool) -> Option<(Vec<Green>, bool)> {
p.eject_partial()
}
+/// Parse an atomic primary. Returns `Some` if all of the input was consumed.
+pub fn parse_atomic_markup(src: &str, _: bool) -> Option<(Vec<Green>, bool)> {
+ let mut p = Parser::new(src, TokenMode::Markup);
+ markup_expr(&mut p);
+ p.eject_partial()
+}
+
/// Parse some markup. Returns `Some` if all of the input was consumed.
pub fn parse_markup(src: &str, _: bool) -> Option<(Vec<Green>, bool)> {
let mut p = Parser::new(src, TokenMode::Markup);
@@ -171,17 +178,7 @@ fn markup_node(p: &mut Parser, at_start: &mut bool) {
| NodeKind::While
| NodeKind::For
| NodeKind::Import
- | NodeKind::Include => {
- let stmt = matches!(token, NodeKind::Let | NodeKind::Set | NodeKind::Import);
- let group = if stmt { Group::Stmt } else { Group::Expr };
-
- p.start_group(group);
- let res = expr_prec(p, true, 0);
- if stmt && res.is_ok() && !p.eof() {
- p.expected_at("semicolon or line break");
- }
- p.end_group();
- }
+ | NodeKind::Include => markup_expr(p),
// Block and template.
NodeKind::LeftBrace => block(p),
@@ -222,6 +219,21 @@ fn enum_node(p: &mut Parser) {
});
}
+/// Parse an expression within markup mode.
+fn markup_expr(p: &mut Parser) {
+ if let Some(token) = p.peek() {
+ let stmt = matches!(token, NodeKind::Let | NodeKind::Set | NodeKind::Import);
+ let group = if stmt { Group::Stmt } else { Group::Expr };
+
+ p.start_group(group);
+ let res = expr_prec(p, true, 0);
+ if stmt && res.is_ok() && !p.eof() {
+ p.expected_at("semicolon or line break");
+ }
+ p.end_group();
+ }
+}
+
/// Parse an expression.
fn expr(p: &mut Parser) -> ParseResult {
expr_prec(p, false, 0)
diff --git a/src/parse/parser.rs b/src/parse/parser.rs
index a37cb9c6..06cb1578 100644
--- a/src/parse/parser.rs
+++ b/src/parse/parser.rs
@@ -48,7 +48,7 @@ impl<'s> Parser<'s> {
}
/// End the parsing process and return multiple children.
- pub fn eject(self) -> Option<(Vec<Green>, bool)>{
+ pub fn eject(self) -> Option<(Vec<Green>, bool)> {
if self.eof() && self.group_success() {
Some((self.children, self.tokens.was_unterminated()))
} else {
diff --git a/src/source.rs b/src/source.rs
index 7eb1d3a7..aaf009e0 100644
--- a/src/source.rs
+++ b/src/source.rs
@@ -12,7 +12,7 @@ use crate::diag::TypResult;
use crate::loading::{FileHash, Loader};
use crate::parse::{is_newline, parse, Scanner};
use crate::syntax::ast::Markup;
-use crate::syntax::{self, Category, GreenNode, RedNode, Span};
+use crate::syntax::{self, Category, GreenNode, RedNode, Reparser, Span};
use crate::util::PathExt;
#[cfg(feature = "codespan-reporting")]
@@ -285,9 +285,8 @@ impl SourceFile {
// Update the root node.
let span = Span::new(self.id, replace.start, replace.end);
- if let Ok(range) =
- Rc::make_mut(&mut self.root).incremental(&self.src, span, with.len())
- {
+ let reparser = Reparser::new(&self.src, span, with.len());
+ if let Ok(range) = reparser.incremental(Rc::make_mut(&mut self.root)) {
range
} else {
self.root = parse(&self.src);
@@ -502,6 +501,14 @@ mod tests {
test("= A heading", 3 .. 3, "n evocative", 2 .. 15);
test("your thing", 5 .. 5, "a", 4 .. 11);
test("a your thing a", 6 .. 7, "a", 2 .. 12);
+ test("{call(); abc}", 7 .. 7, "[]", 0 .. 15);
+ test("#call() abc", 7 .. 7, "[]", 0 .. 13);
+ // test(
+ // "hi\n- item\n- item 2\n - item 3",
+ // 10 .. 10,
+ // " ",
+ // 9 .. 33,
+ // );
test(
"#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])",
16 .. 20,
@@ -535,7 +542,7 @@ mod tests {
"C ",
14 .. 22,
);
- test("{ let x = g() }", 10 .. 12, "f(54", 2 .. 15);
+ test("{ let x = g() }", 10 .. 12, "f(54", 0 .. 17);
test(
"a #let rect with (fill: eastern)\nb",
16 .. 31,
diff --git a/src/syntax/incremental.rs b/src/syntax/incremental.rs
new file mode 100644
index 00000000..d7b5ca3c
--- /dev/null
+++ b/src/syntax/incremental.rs
@@ -0,0 +1,515 @@
+use std::ops::Range;
+use std::rc::Rc;
+
+use super::{Green, GreenNode, NodeKind, Span};
+
+use crate::parse::{
+ parse_atomic, parse_atomic_markup, parse_block, parse_comment, parse_markup,
+ parse_markup_elements, parse_template, TokenMode,
+};
+
+pub struct Reparser<'a> {
+ src: &'a str,
+ replace_range: Span,
+ replace_len: usize,
+}
+
+impl<'a> Reparser<'a> {
+ pub fn new(src: &'a str, replace_range: Span, replace_len: usize) -> Self {
+ Self { src, replace_range, replace_len }
+ }
+}
+
+impl Reparser<'_> {
+ /// Find the innermost child that is incremental safe.
+ pub fn incremental(&self, green: &mut GreenNode) -> Result<Range<usize>, ()> {
+ self.incremental_int(green, 0, TokenMode::Markup, true)
+ }
+
+ fn incremental_int(
+ &self,
+ green: &mut GreenNode,
+ mut offset: usize,
+ parent_mode: TokenMode,
+ outermost: bool,
+ ) -> Result<Range<usize>, ()> {
+ let kind = green.kind().clone();
+ let mode = kind.mode().contextualize(parent_mode);
+
+ let mut loop_result = None;
+ let mut child_at_start = true;
+ let last = green.children.len() - 1;
+ let mut start = None;
+ for (i, child) in green.children.iter_mut().enumerate() {
+ let child_span =
+ Span::new(self.replace_range.source, offset, offset + child.len());
+ if child_span.surrounds(self.replace_range)
+ && start.is_none()
+ && ((self.replace_range.start != child_span.end
+ && self.replace_range.end != child_span.start)
+ || mode == TokenMode::Code
+ || i == last)
+ {
+ let old_len = child.len();
+ // First, we try if the child has another, more specific applicable child.
+ if !kind.post().unsafe_interior() {
+ if let Ok(range) = match child {
+ Green::Node(n) => self.incremental_int(
+ Rc::make_mut(n),
+ offset,
+ kind.mode().child_mode(),
+ i == last && outermost,
+ ),
+ Green::Token(_) => Err(()),
+ } {
+ let new_len = child.len();
+ green.update_child_len(new_len, old_len);
+ return Ok(range);
+ }
+ }
+
+ // This didn't work, so we try to self.replace_range the child at this
+ // level.
+ loop_result =
+ Some((i .. i + 1, child_span, i == last && outermost, child.kind()));
+ break;
+ } else if start.is_none()
+ && child_span.contains(self.replace_range.start)
+ && mode == TokenMode::Markup
+ && child.kind().post().markup_safe()
+ {
+ start = Some((i, offset));
+ } else if child_span.contains(self.replace_range.end)
+ && (self.replace_range.end != child_span.end || i == last)
+ && mode == TokenMode::Markup
+ && child.kind().post().markup_safe()
+ {
+ if let Some((start, start_offset)) = start {
+ loop_result = Some((
+ start .. i + 1,
+ Span::new(
+ self.replace_range.source,
+ start_offset,
+ offset + child.len(),
+ ),
+ i == last && outermost,
+ child.kind(),
+ ));
+ }
+ break;
+ } else if start.is_some()
+ && (mode != TokenMode::Markup || !child.kind().post().markup_safe())
+ {
+ break;
+ }
+
+ offset += child.len();
+ child_at_start = child.kind().is_at_start(child_at_start);
+ }
+
+
+ // We now have a child that we can self.replace_range and a function to do so if
+ // the loop found any results at all.
+ let (child_idx_range, child_span, child_outermost, func, policy) =
+ loop_result.ok_or(()).and_then(|(a, b, c, child_kind)| {
+ let (func, policy) =
+ child_kind.reparsing_function(kind.mode().child_mode());
+ Ok((a, b, c, func?, policy))
+ })?;
+
+ let src_span = child_span.inserted(self.replace_range, self.replace_len);
+ let recompile_range = if policy == Postcondition::AtomicPrimary {
+ src_span.start .. self.src.len()
+ } else {
+ src_span.to_range()
+ };
+
+ let (mut new_children, unterminated) =
+ func(&self.src[recompile_range], child_at_start).ok_or(())?;
+
+ // Do not accept unclosed nodes if the old node did not use to be at the
+ // right edge of the tree.
+ if !child_outermost && unterminated {
+ return Err(());
+ }
+
+ let insertion = match check_invariants(
+ &new_children,
+ green.children(),
+ child_idx_range.clone(),
+ child_at_start,
+ mode,
+ src_span,
+ policy,
+ ) {
+ InvariantResult::Ok => Ok(new_children),
+ InvariantResult::UseFirst => Ok(vec![std::mem::take(&mut new_children[0])]),
+ InvariantResult::Error => Err(()),
+ }?;
+
+ green.replace_child_range(child_idx_range, insertion);
+
+ Ok(src_span.to_range())
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+enum InvariantResult {
+ Ok,
+ UseFirst,
+ Error,
+}
+
+fn check_invariants(
+ use_children: &[Green],
+ old_children: &[Green],
+ child_idx_range: Range<usize>,
+ child_at_start: bool,
+ mode: TokenMode,
+ src_span: Span,
+ policy: Postcondition,
+) -> InvariantResult {
+ let (new_children, ok) = if policy == Postcondition::AtomicPrimary {
+ if use_children.iter().map(Green::len).sum::<usize>() == src_span.len() {
+ (use_children, InvariantResult::Ok)
+ } else if use_children.len() == 1 && use_children[0].len() == src_span.len() {
+ (&use_children[0 .. 1], InvariantResult::UseFirst)
+ } else {
+ return InvariantResult::Error;
+ }
+ } else {
+ (use_children, InvariantResult::Ok)
+ };
+
+ let child_mode = old_children[child_idx_range.start].kind().mode().child_mode();
+
+ // Check if the children / child has the right type.
+ let same_kind = match policy {
+ Postcondition::SameKind(x) => x.map_or(true, |x| x == child_mode),
+ _ => false,
+ };
+
+ if same_kind || policy == Postcondition::AtomicPrimary {
+ if new_children.len() != 1 {
+ return InvariantResult::Error;
+ }
+
+ if same_kind {
+ if old_children[child_idx_range.start].kind() != new_children[0].kind() {
+ return InvariantResult::Error;
+ }
+ }
+ }
+
+ // Check if the neighbor invariants are still true.
+ if mode == TokenMode::Markup {
+ if child_idx_range.start > 0 {
+ if old_children[child_idx_range.start - 1].kind().pre()
+ == Precondition::RightWhitespace
+ && !new_children[0].kind().is_whitespace()
+ {
+ return InvariantResult::Error;
+ }
+ }
+
+ if new_children.last().map(|x| x.kind().pre())
+ == Some(Precondition::RightWhitespace)
+ && old_children.len() > child_idx_range.end
+ {
+ if !old_children[child_idx_range.end].kind().is_whitespace() {
+ return InvariantResult::Error;
+ }
+ }
+
+ let mut new_at_start = child_at_start;
+ for child in new_children {
+ new_at_start = child.kind().is_at_start(new_at_start);
+ }
+
+ for child in &old_children[child_idx_range.end ..] {
+ if child.kind().is_trivia() {
+ new_at_start = child.kind().is_at_start(new_at_start);
+ continue;
+ }
+
+ match child.kind().pre() {
+ Precondition::AtStart if !new_at_start => {
+ return InvariantResult::Error;
+ }
+ Precondition::NotAtStart if new_at_start => {
+ return InvariantResult::Error;
+ }
+ _ => {}
+ }
+ break;
+ }
+ }
+
+ ok
+}
+
+impl NodeKind {
+ pub fn reparsing_function(
+ &self,
+ parent_mode: TokenMode,
+ ) -> (
+ Result<fn(&str, bool) -> Option<(Vec<Green>, bool)>, ()>,
+ Postcondition,
+ ) {
+ let policy = self.post();
+ let mode = self.mode().contextualize(parent_mode);
+
+ match policy {
+ Postcondition::Unsafe | Postcondition::UnsafeLayer => (Err(()), policy),
+ Postcondition::AtomicPrimary if mode == TokenMode::Code => {
+ (Ok(parse_atomic), policy)
+ }
+ Postcondition::AtomicPrimary => (Ok(parse_atomic_markup), policy),
+ Postcondition::SameKind(x) if x == None || x == Some(mode) => {
+ let parser: fn(&str, bool) -> _ = match self {
+ NodeKind::Template => parse_template,
+ NodeKind::Block => parse_block,
+ NodeKind::LineComment | NodeKind::BlockComment => parse_comment,
+ _ => return (Err(()), policy),
+ };
+
+ (Ok(parser), policy)
+ }
+ _ => {
+ let parser: fn(&str, bool) -> _ = match mode {
+ TokenMode::Markup if self == &Self::Markup => parse_markup,
+ TokenMode::Markup => parse_markup_elements,
+ _ => return (Err(()), policy),
+ };
+
+ (Ok(parser), policy)
+ }
+ }
+ }
+
+ /// Whether it is safe to do incremental parsing on this node. Never allow
+ /// non-termination errors if this is not already the last leaf node.
+ pub fn post(&self) -> Postcondition {
+ match self {
+ // Replacing parenthesis changes if the expression is balanced and
+ // is therefore not safe.
+ Self::LeftBracket
+ | Self::RightBracket
+ | Self::LeftBrace
+ | Self::RightBrace
+ | Self::LeftParen
+ | Self::RightParen => Postcondition::Unsafe,
+
+ // Replacing an operator can change whether the parent is an
+ // operation which makes it unsafe. The star can appear in markup.
+ Self::Star
+ | Self::Comma
+ | Self::Semicolon
+ | Self::Colon
+ | Self::Plus
+ | Self::Minus
+ | Self::Slash
+ | Self::Eq
+ | Self::EqEq
+ | Self::ExclEq
+ | Self::Lt
+ | Self::LtEq
+ | Self::Gt
+ | Self::GtEq
+ | Self::PlusEq
+ | Self::HyphEq
+ | Self::StarEq
+ | Self::SlashEq
+ | Self::Not
+ | Self::And
+ | Self::Or
+ | Self::With
+ | Self::Dots
+ | Self::Arrow => Postcondition::Unsafe,
+
+ // These keywords are literals and can be safely be substituted with
+ // other expressions.
+ Self::None | Self::Auto => Postcondition::AtomicPrimary,
+
+ // These keywords change what kind of expression the parent is and
+ // how far the expression would go.
+ Self::Let
+ | Self::Set
+ | Self::If
+ | Self::Else
+ | Self::For
+ | Self::In
+ | Self::While
+ | Self::Break
+ | Self::Continue
+ | Self::Return
+ | Self::Import
+ | Self::Include
+ | Self::From => Postcondition::Unsafe,
+
+ Self::Markup => Postcondition::SameKind(None),
+
+ Self::Space(_) => Postcondition::SameKind(Some(TokenMode::Code)),
+
+ // These are all replaceable by other tokens.
+ Self::Parbreak
+ | Self::Linebreak
+ | Self::Text(_)
+ | Self::TextInLine(_)
+ | Self::NonBreakingSpace
+ | Self::EnDash
+ | Self::EmDash
+ | Self::Escape(_)
+ | Self::Strong
+ | Self::Emph
+ | Self::Heading
+ | Self::Enum
+ | Self::List
+ | Self::Raw(_)
+ | Self::Math(_) => Postcondition::Safe,
+
+ // Changing the heading level, enum numbering, or list bullet
+ // changes the next layer.
+ Self::EnumNumbering(_) => Postcondition::Unsafe,
+
+ // These are expressions that can be replaced by other expressions.
+ Self::Ident(_)
+ | Self::Bool(_)
+ | Self::Int(_)
+ | Self::Float(_)
+ | Self::Length(_, _)
+ | Self::Angle(_, _)
+ | Self::Percentage(_)
+ | Self::Str(_)
+ | Self::Fraction(_)
+ | Self::Array
+ | Self::Dict
+ | Self::Group => Postcondition::AtomicPrimary,
+
+ Self::Call
+ | Self::Unary
+ | Self::Binary
+ | Self::CallArgs
+ | Self::Named
+ | Self::Spread => Postcondition::UnsafeLayer,
+
+ // The closure is a bit magic with the let expression, and also it
+ // is not atomic.
+ Self::Closure | Self::ClosureParams => Postcondition::UnsafeLayer,
+
+ // These can appear as bodies and would trigger an error if they
+ // became something else.
+ Self::Template => Postcondition::SameKind(None),
+ Self::Block => Postcondition::SameKind(Some(TokenMode::Code)),
+
+ Self::ForExpr
+ | Self::WhileExpr
+ | Self::IfExpr
+ | Self::LetExpr
+ | Self::SetExpr
+ | Self::ImportExpr
+ | Self::IncludeExpr => Postcondition::AtomicPrimary,
+
+ Self::WithExpr | Self::ForPattern | Self::ImportItems => {
+ Postcondition::UnsafeLayer
+ }
+
+ // These can appear everywhere and must not change to other stuff
+ // because that could change the outer expression.
+ Self::LineComment | Self::BlockComment => Postcondition::SameKind(None),
+
+ Self::Error(_, _) | Self::Unknown(_) => Postcondition::Unsafe,
+ }
+ }
+
+ /// The appropriate precondition for the type.
+ pub fn pre(&self) -> Precondition {
+ match self {
+ Self::Heading | Self::Enum | Self::List => Precondition::AtStart,
+ Self::TextInLine(_) => Precondition::NotAtStart,
+ Self::Linebreak => Precondition::RightWhitespace,
+ _ => Precondition::None,
+ }
+ }
+}
+
+/// This enum describes what conditions a node has for being replaced by a new
+/// parse result.
+///
+/// Safe nodes are replaced by the new parse result from the respective mode.
+/// They can be replaced by multiple tokens. If a token is inserted in Markup
+/// mode and the next token would not be `at_start` there needs to be a forward
+/// check for a `EnsureAtStart` node. If this fails, the parent has to be
+/// reparsed. if the direct whitespace sibling of a `EnsureRightWhitespace` is
+/// `Unsafe`. Similarly, if a `EnsureRightWhitespace` token is one of the last
+/// tokens to be inserted, the edit is invalidated if there is no following
+/// whitespace. The atomic nodes may only be replaced by other atomic nodes. The
+/// unsafe layers cannot be used but allow children access, the unsafe nodes do
+/// neither.
+///
+/// *Procedure:*
+/// 1. Check if the node is safe - if unsafe layer recurse, if unsafe, return
+/// None.
+/// 2. Reparse with appropriate node kind and `at_start`.
+/// 3. Check whether the topmost group is terminated and the range was
+/// completely consumed, otherwise return None.
+/// 4. Check if the type criteria are met.
+/// 5. If the node is not at the end of the tree, check if Strings etc. are
+/// terminated.
+/// 6. If this is markup, check the following things:
+/// - The `at_start` conditions of the next non-comment and non-space(0) node
+/// are met.
+/// - The first node is whitespace or the previous siblings are not
+/// `EnsureRightWhitespace`.
+/// - If any of those fails, return None.
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+pub enum Postcondition {
+ /// Changing this node can never have an influence on the other nodes.
+ Safe,
+ /// This node has to be replaced with a single token of the same kind.
+ SameKind(Option<TokenMode>),
+ /// Changing this node into a single atomic expression is allowed if it
+ /// appears in code mode, otherwise it is safe.
+ AtomicPrimary,
+ /// Changing an unsafe layer node changes what the parents or the
+ /// surrounding nodes would be and is therefore disallowed. Change the
+ /// parents or children instead. If it appears in Markup, however, it is
+ /// safe to change.
+ UnsafeLayer,
+ /// Changing an unsafe node or any of its children will trigger undefined
+ /// behavior. Change the parents instead.
+ Unsafe,
+}
+
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+pub enum Precondition {
+ /// These nodes depend on being at the start of a line. Reparsing of safe
+ /// left neighbors has to check this invariant. Otherwise, this node is
+ /// safe.
+ AtStart,
+ /// These nodes depend on not being at the start of a line. Reparsing of
+ /// safe left neighbors has to check this invariant. Otherwise, this node is
+ /// safe.
+ NotAtStart,
+ /// These nodes must be followed by whitespace.
+ RightWhitespace,
+ /// No additional requirements.
+ None,
+}
+
+impl Postcondition {
+ pub fn unsafe_interior(&self) -> bool {
+ match self {
+ Self::Unsafe => true,
+ _ => false,
+ }
+ }
+
+ pub fn markup_safe(&self) -> bool {
+ match self {
+ Self::Safe | Self::UnsafeLayer => true,
+ Self::SameKind(tm) => tm.map_or(false, |tm| tm != TokenMode::Markup),
+ _ => false,
+ }
+ }
+}
diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs
index cfb44376..4d0ca026 100644
--- a/src/syntax/mod.rs
+++ b/src/syntax/mod.rs
@@ -2,6 +2,7 @@
pub mod ast;
mod highlight;
+mod incremental;
mod pretty;
mod span;
@@ -10,16 +11,14 @@ use std::ops::Range;
use std::rc::Rc;
pub use highlight::*;
+pub use incremental::*;
pub use pretty::*;
pub use span::*;
use self::ast::{MathNode, RawNode, TypedNode};
use crate::diag::Error;
use crate::geom::{AngularUnit, LengthUnit};
-use crate::parse::{
- parse_atomic, parse_block, parse_comment, parse_markup, parse_markup_elements,
- parse_template, TokenMode,
-};
+use crate::parse::TokenMode;
use crate::source::SourceId;
use crate::util::EcoString;
@@ -87,29 +86,6 @@ impl Green {
Self::Token(data) => data.kind = kind,
}
}
-
- /// Find the innermost child that is incremental safe.
- fn incremental(
- &mut self,
- edit: &str,
- replace: Span,
- replacement_len: usize,
- offset: usize,
- parent_mode: TokenMode,
- outermost: bool,
- ) -> Result<Range<usize>, ()> {
- match self {
- Green::Node(n) => Rc::make_mut(n).incremental_int(
- edit,
- replace,
- replacement_len,
- offset,
- parent_mode,
- outermost,
- ),
- Green::Token(_) => Err(()),
- }
- }
}
impl Default for Green {
@@ -194,8 +170,22 @@ impl GreenNode {
self.children[child_idx_range.clone()].iter().map(Green::len).sum();
let new_len: usize = replacement.iter().map(Green::len).sum();
+ if self.erroneous {
+ if self.children[child_idx_range.clone()].iter().any(Green::erroneous) {
+ // the old range was erroneous but we do not know if anywhere
+ // else was so we have to iterate over the whole thing.
+ self.erroneous = self.children[.. child_idx_range.start]
+ .iter()
+ .any(Green::erroneous)
+ || self.children[child_idx_range.end ..].iter().any(Green::erroneous);
+ }
+ // in this case nothing changes so we do not have to bother.
+ }
+
+ // the or assignment operator is not lazy.
+ self.erroneous = self.erroneous || replacement.iter().any(Green::erroneous);
+
self.children.splice(child_idx_range, replacement);
- self.erroneous = self.children.iter().any(|x| x.erroneous());
self.data.set_len(self.data.len + new_len - old_len);
}
@@ -203,250 +193,6 @@ impl GreenNode {
self.data.len = self.data.len() + new_len - old_len;
self.erroneous = self.children.iter().any(|x| x.erroneous());
}
-
- /// Find the innermost child that is incremental safe.
- pub fn incremental(
- &mut self,
- src: &str,
- replace: Span,
- replacement_len: usize,
- ) -> Result<Range<usize>, ()> {
- self.incremental_int(src, replace, replacement_len, 0, TokenMode::Markup, true)
- }
-
- fn incremental_int(
- &mut self,
- src: &str,
- replace: Span,
- replacement_len: usize,
- mut offset: usize,
- parent_mode: TokenMode,
- outermost: bool,
- ) -> Result<Range<usize>, ()> {
- let kind = self.kind().clone();
- let mode = kind.mode().contextualize(parent_mode);
-
- let mut loop_result = None;
- let mut child_at_start = true;
- let last = self.children.len() - 1;
- let mut start = None;
- for (i, child) in self.children.iter_mut().enumerate() {
- let child_span = Span::new(replace.source, offset, offset + child.len());
- if child_span.surrounds(replace)
- && start.is_none()
- && ((replace.start != child_span.end && replace.end != child_span.start)
- || mode == TokenMode::Code
- || i == last)
- {
- let old_len = child.len();
- // First, we try if the child has another, more specific applicable child.
- if !kind.incremental_safety().unsafe_interior() {
- if let Ok(range) = child.incremental(
- src,
- replace,
- replacement_len,
- offset,
- kind.mode().child_mode(),
- i == last && outermost,
- ) {
- let new_len = child.len();
- self.update_child_len(new_len, old_len);
- return Ok(range);
- }
- }
-
- // This didn't work, so we try to replace the child at this
- // level.
- let (function, policy) =
- child.kind().reparsing_function(kind.mode().child_mode());
- let function = function?;
- loop_result = Some((
- i .. i + 1,
- child_span,
- i == last && outermost,
- function,
- policy,
- ));
- break;
- } else if start.is_none()
- && child_span.contains(replace.start)
- && mode == TokenMode::Markup
- && child.kind().incremental_safety().markup_safe()
- {
- start = Some((i, offset));
- } else if child_span.contains(replace.end)
- && (replace.end != child_span.end || i == last)
- && mode == TokenMode::Markup
- && child.kind().incremental_safety().markup_safe()
- {
- if let Some((start, start_offset)) = start {
- let (function, policy) =
- child.kind().reparsing_function(kind.mode().child_mode());
- let function = function?;
- loop_result = Some((
- start .. i + 1,
- Span::new(replace.source, start_offset, offset + child.len()),
- i == last && outermost,
- function,
- policy,
- ));
- }
- break;
- } else if start.is_some()
- && (mode != TokenMode::Markup
- || !child.kind().incremental_safety().markup_safe())
- {
- break;
- }
-
- offset += child.len();
- child_at_start = child.kind().is_at_start(child_at_start);
- }
-
-
- // We now have a child that we can replace and a function to do so if
- // the loop found any results at all.
- let (child_idx_range, child_span, child_outermost, func, policy) =
- loop_result.ok_or(())?;
-
- let src_span = child_span.inserted(replace, replacement_len);
- let recompile_range = if policy == IncrementalSafety::AtomicPrimary {
- src_span.start .. src.len()
- } else {
- src_span.to_range()
- };
-
- let (mut new_children, unterminated) =
- func(&src[recompile_range], child_at_start).ok_or(())?;
-
- let insertion = match check_invariants(
- &new_children,
- self.children(),
- unterminated,
- child_idx_range.clone(),
- child_outermost,
- child_at_start,
- mode,
- src_span,
- policy,
- ) {
- InvariantResult::Ok => Ok(new_children),
- InvariantResult::UseFirst => Ok(vec![std::mem::take(&mut new_children[0])]),
- InvariantResult::Error => Err(()),
- }?;
-
- self.replace_child_range(child_idx_range, insertion);
-
- Ok(src_span.to_range())
- }
-}
-
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-enum InvariantResult {
- Ok,
- UseFirst,
- Error,
-}
-
-fn check_invariants(
- use_children: &[Green],
- old_children: &[Green],
- unterminated: bool,
- child_idx_range: Range<usize>,
- outermost: bool,
- child_at_start: bool,
- mode: TokenMode,
- src_span: Span,
- policy: IncrementalSafety,
-) -> InvariantResult {
- let (new_children, ok) = if policy == IncrementalSafety::AtomicPrimary {
- if use_children.iter().map(Green::len).sum::<usize>() == src_span.len() {
- (use_children, InvariantResult::Ok)
- } else if use_children[0].len() == src_span.len() {
- (&use_children[0 .. 1], InvariantResult::UseFirst)
- } else {
- return InvariantResult::Error;
- }
- } else {
- (use_children, InvariantResult::Ok)
- };
-
- let child_mode = old_children[child_idx_range.start].kind().mode().child_mode();
-
- // Check if the children / child has the right type.
- let require_single = match policy {
- IncrementalSafety::AtomicPrimary | IncrementalSafety::SameKind => true,
- IncrementalSafety::SameKindInCode if child_mode == TokenMode::Code => true,
- _ => false,
- };
-
- if require_single {
- if new_children.len() != 1 {
- return InvariantResult::Error;
- }
-
- if match policy {
- IncrementalSafety::SameKind => true,
- IncrementalSafety::SameKindInCode => child_mode == TokenMode::Code,
- _ => false,
- } {
- if old_children[child_idx_range.start].kind() != new_children[0].kind() {
- return InvariantResult::Error;
- }
- }
- }
-
- // Do not accept unclosed nodes if the old node did not use to be at the
- // right edge of the tree.
- if !outermost && unterminated {
- return InvariantResult::Error;
- }
-
- // Check if the neighbor invariants are still true.
- if mode == TokenMode::Markup {
- if child_idx_range.start > 0 {
- if old_children[child_idx_range.start - 1].kind().incremental_safety()
- == IncrementalSafety::EnsureRightWhitespace
- && !new_children[0].kind().is_whitespace()
- {
- return InvariantResult::Error;
- }
- }
-
- let mut new_at_start = child_at_start;
- for child in new_children {
- new_at_start = child.kind().is_at_start(new_at_start);
- }
-
- for child in &old_children[child_idx_range.end ..] {
- if child.kind().is_trivia() {
- new_at_start = child.kind().is_at_start(new_at_start);
- continue;
- }
-
- match child.kind().incremental_safety() {
- IncrementalSafety::EnsureAtStart if !new_at_start => {
- return InvariantResult::Error;
- }
- IncrementalSafety::EnsureNotAtStart if new_at_start => {
- return InvariantResult::Error;
- }
- _ => {}
- }
- break;
- }
-
- if new_children.last().map(|x| x.kind().incremental_safety())
- == Some(IncrementalSafety::EnsureRightWhitespace)
- && old_children.len() > child_idx_range.end
- {
- if !old_children[child_idx_range.end].kind().is_whitespace() {
- return InvariantResult::Error;
- }
- }
- }
-
- ok
}
impl From<GreenNode> for Green {
@@ -1053,190 +799,6 @@ impl NodeKind {
}
}
- pub fn reparsing_function(
- &self,
- parent_mode: TokenMode,
- ) -> (
- Result<fn(&str, bool) -> Option<(Vec<Green>, bool)>, ()>,
- IncrementalSafety,
- ) {
- let policy = self.incremental_safety();
- if policy.is_unsafe() {
- return (Err(()), policy);
- }
-
- let contextualized = self.mode().contextualize(parent_mode);
- let is_code = contextualized == TokenMode::Code;
-
- if is_code && policy == IncrementalSafety::UnsafeLayer {
- return (Err(()), policy);
- }
-
- if is_code && policy == IncrementalSafety::AtomicPrimary {
- return (Ok(parse_atomic), policy);
- }
-
- if policy == IncrementalSafety::SameKind
- || (policy == IncrementalSafety::SameKindInCode && is_code)
- {
- let parser: fn(&str, bool) -> _ = match self {
- NodeKind::Template => parse_template,
- NodeKind::Block => parse_block,
- NodeKind::LineComment | NodeKind::BlockComment => parse_comment,
- _ => return (Err(()), policy),
- };
-
- return (Ok(parser), policy);
- }
-
- let parser: fn(&str, bool) -> _ = match contextualized {
- TokenMode::Markup if self == &Self::Markup => parse_markup,
- TokenMode::Markup => parse_markup_elements,
- _ => return (Err(()), policy),
- };
-
- (Ok(parser), policy)
- }
-
- /// Whether it is safe to do incremental parsing on this node. Never allow
- /// non-termination errors if this is not already the last leaf node.
- pub fn incremental_safety(&self) -> IncrementalSafety {
- match self {
- // Replacing parenthesis changes if the expression is balanced and
- // is therefore not safe.
- Self::LeftBracket
- | Self::RightBracket
- | Self::LeftBrace
- | Self::RightBrace
- | Self::LeftParen
- | Self::RightParen => IncrementalSafety::Unsafe,
-
- // Replacing an operator can change whether the parent is an
- // operation which makes it unsafe. The star can appear in markup.
- Self::Star
- | Self::Comma
- | Self::Semicolon
- | Self::Colon
- | Self::Plus
- | Self::Minus
- | Self::Slash
- | Self::Eq
- | Self::EqEq
- | Self::ExclEq
- | Self::Lt
- | Self::LtEq
- | Self::Gt
- | Self::GtEq
- | Self::PlusEq
- | Self::HyphEq
- | Self::StarEq
- | Self::SlashEq
- | Self::Not
- | Self::And
- | Self::Or
- | Self::With
- | Self::Dots
- | Self::Arrow => IncrementalSafety::Unsafe,
-
- // These keywords are literals and can be safely be substituted with
- // other expressions.
- Self::None | Self::Auto => IncrementalSafety::AtomicPrimary,
-
- // These keywords change what kind of expression the parent is and
- // how far the expression would go.
- Self::Let
- | Self::If
- | Self::Else
- | Self::For
- | Self::In
- | Self::While
- | Self::Break
- | Self::Continue
- | Self::Return
- | Self::Set
- | Self::Import
- | Self::Include
- | Self::From => IncrementalSafety::Unsafe,
-
- // This is a backslash followed by a space. But changing it to
- // anything else is fair game.
- Self::Linebreak => IncrementalSafety::EnsureRightWhitespace,
-
- Self::Markup => IncrementalSafety::SameKind,
-
- Self::Space(_) => IncrementalSafety::SameKindInCode,
-
- // These are all replaceable by other tokens.
- Self::Parbreak
- | Self::Text(_)
- | Self::NonBreakingSpace
- | Self::EnDash
- | Self::EmDash
- | Self::Escape(_)
- | Self::Strong
- | Self::Emph => IncrementalSafety::Safe,
-
- // This is text that needs to be not `at_start`, otherwise it would
- // start one of the below items.
- Self::TextInLine(_) => IncrementalSafety::EnsureNotAtStart,
-
- // These have to be `at_start` so they must be preceeded with a
- // Space(n) with n > 0 or a Parbreak.
- Self::Heading | Self::Enum | Self::List => IncrementalSafety::EnsureAtStart,
-
- // Changing the heading level, enum numbering, or list bullet
- // changes the next layer.
- Self::EnumNumbering(_) => IncrementalSafety::Unsafe,
-
- Self::Raw(_) | Self::Math(_) => IncrementalSafety::Safe,
-
- // These are expressions that can be replaced by other expressions.
- Self::Ident(_)
- | Self::Bool(_)
- | Self::Int(_)
- | Self::Float(_)
- | Self::Length(_, _)
- | Self::Angle(_, _)
- | Self::Percentage(_)
- | Self::Str(_)
- | Self::Fraction(_)
- | Self::Array
- | Self::Dict
- | Self::Group => IncrementalSafety::AtomicPrimary,
-
- Self::Call | Self::Unary | Self::Binary | Self::SetExpr => {
- IncrementalSafety::UnsafeLayer
- }
-
- Self::CallArgs | Self::Named | Self::Spread => IncrementalSafety::UnsafeLayer,
-
- // The closure is a bit magic with the let expression, and also it
- // is not atomic.
- Self::Closure | Self::ClosureParams => IncrementalSafety::UnsafeLayer,
-
- // These can appear as bodies and would trigger an error if they
- // became something else.
- Self::Template | Self::Block => IncrementalSafety::SameKindInCode,
-
- Self::ForExpr
- | Self::WhileExpr
- | Self::IfExpr
- | Self::LetExpr
- | Self::ImportExpr
- | Self::IncludeExpr => IncrementalSafety::AtomicPrimary,
-
- Self::WithExpr | Self::ForPattern | Self::ImportItems => {
- IncrementalSafety::UnsafeLayer
- }
-
- // These can appear everywhere and must not change to other stuff
- // because that could change the outer expression.
- Self::LineComment | Self::BlockComment => IncrementalSafety::SameKind,
-
- Self::Error(_, _) | Self::Unknown(_) => IncrementalSafety::Unsafe,
- }
- }
-
/// A human-readable name for the kind.
pub fn as_str(&self) -> &'static str {
match self {
@@ -1351,95 +913,6 @@ impl Display for NodeKind {
}
}
-/// This enum describes what conditions a node has for being replaced by a new
-/// parse result.
-///
-/// Safe nodes are replaced by the new parse result from the respective mode.
-/// They can be replaced by multiple tokens. If a token is inserted in Markup
-/// mode and the next token would not be `at_start` there needs to be a forward
-/// check for a `EnsureAtStart` node. If this fails, the parent has to be
-/// reparsed. if the direct whitespace sibling of a `EnsureRightWhitespace` is
-/// `Unsafe`. Similarly, if a `EnsureRightWhitespace` token is one of the last
-/// tokens to be inserted, the edit is invalidated if there is no following
-/// whitespace. The atomic nodes may only be replaced by other atomic nodes. The
-/// unsafe layers cannot be used but allow children access, the unsafe nodes do
-/// neither.
-///
-/// *Procedure:*
-/// 1. Check if the node is safe - if unsafe layer recurse, if unsafe, return
-/// None.
-/// 2. Reparse with appropriate node kind and `at_start`.
-/// 3. Check whether the topmost group is terminated and the range was
-/// completely consumed, otherwise return None.
-/// 4. Check if the type criteria are met.
-/// 5. If the node is not at the end of the tree, check if Strings etc. are
-/// terminated.
-/// 6. If this is markup, check the following things:
-/// - The `at_start` conditions of the next non-comment and non-space(0) node
-/// are met.
-/// - The first node is whitespace or the previous siblings are not
-/// `EnsureRightWhitespace`.
-/// - If any of those fails, return None.
-#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
-pub enum IncrementalSafety {
- /// Changing this node can never have an influence on the other nodes.
- Safe,
- /// This node has to be replaced with a single token of the same kind.
- SameKind,
- /// This node has to be replaced with a single token of the same kind if in
- /// code mode.
- SameKindInCode,
- /// These nodes depend on being at the start of a line. Reparsing of safe
- /// left neighbors has to check this invariant. Otherwise, this node is
- /// safe.
- EnsureAtStart,
- /// These nodes depend on not being at the start of a line. Reparsing of
- /// safe left neighbors has to check this invariant. Otherwise, this node is
- /// safe.
- EnsureNotAtStart,
- /// These nodes must be followed by whitespace.
- EnsureRightWhitespace,
- /// Changing this node into a single atomic expression is allowed if it
- /// appears in code mode, otherwise it is safe.
- AtomicPrimary,
- /// Changing an unsafe layer node changes what the parents or the
- /// surrounding nodes would be and is therefore disallowed. Change the
- /// parents or children instead. If it appears in Markup, however, it is
- /// safe to change.
- UnsafeLayer,
- /// Changing an unsafe node or any of its children will trigger undefined
- /// behavior. Change the parents instead.
- Unsafe,
-}
-
-impl IncrementalSafety {
- pub fn unsafe_interior(&self) -> bool {
- match self {
- Self::Unsafe => true,
- _ => false,
- }
- }
-
- pub fn is_unsafe(&self) -> bool {
- match self {
- Self::UnsafeLayer | Self::Unsafe => true,
- _ => false,
- }
- }
-
- pub fn markup_safe(&self) -> bool {
- match self {
- Self::Safe
- | Self::SameKindInCode
- | Self::EnsureAtStart
- | Self::EnsureNotAtStart
- | Self::EnsureRightWhitespace
- | Self::UnsafeLayer => true,
- _ => false,
- }
- }
-}
-
/// This enum describes which mode a token of [`NodeKind`] can appear in.
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum NodeMode {