summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMartin Haug <mhaug@live.de>2021-11-28 18:18:45 +0100
committerMartin Haug <mhaug@live.de>2021-11-28 18:18:45 +0100
commite05eb5fda5d1dfeef168b6fc071b20fdbcce2dcd (patch)
tree6b4585c065ce21a76784cd0915d9b6e0a414f88d
parentedc686d7384470068858e16f2926cf50f31b2c90 (diff)
Code Review: Parser, I can't let you do this
-rw-r--r--Cargo.lock420
-rw-r--r--Cargo.toml6
-rw-r--r--benches/timed.rs98
-rw-r--r--src/parse/incremental.rs449
-rw-r--r--src/parse/mod.rs10
-rw-r--r--src/parse/parser.rs14
-rw-r--r--src/parse/tokens.rs28
-rw-r--r--src/source.rs16
-rw-r--r--src/syntax/mod.rs50
9 files changed, 228 insertions, 863 deletions
diff --git a/Cargo.lock b/Cargo.lock
index 98cb4d81..df3bf74a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -39,17 +39,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
[[package]]
-name = "atty"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
-dependencies = [
- "hermit-abi",
- "libc",
- "winapi",
-]
-
-[[package]]
name = "autocfg"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -68,24 +57,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
-name = "bstr"
-version = "0.2.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
-dependencies = [
- "lazy_static",
- "memchr",
- "regex-automata",
- "serde",
-]
-
-[[package]]
-name = "bumpalo"
-version = "3.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f1e260c3a9040a7c19a12468758f4c16f31a81a1fe087482be9570ec864bb6c"
-
-[[package]]
name = "bytemuck"
version = "1.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -98,32 +69,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
-name = "cast"
-version = "0.2.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c24dab4283a142afa2fdca129b80ad2c6284e073930f964c3a1293c225ee39a"
-dependencies = [
- "rustc_version",
-]
-
-[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
-name = "clap"
-version = "2.34.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
-dependencies = [
- "bitflags",
- "textwrap",
- "unicode-width",
-]
-
-[[package]]
name = "codespan-reporting"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -149,108 +100,6 @@ dependencies = [
]
[[package]]
-name = "criterion"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1604dafd25fba2fe2d5895a9da139f8dc9b319a5fe5354ca137cbbce4e178d10"
-dependencies = [
- "atty",
- "cast",
- "clap",
- "criterion-plot",
- "csv",
- "itertools",
- "lazy_static",
- "num-traits",
- "oorandom",
- "plotters",
- "rayon",
- "regex",
- "serde",
- "serde_cbor",
- "serde_derive",
- "serde_json",
- "tinytemplate",
- "walkdir",
-]
-
-[[package]]
-name = "criterion-plot"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d00996de9f2f7559f7f4dc286073197f83e92256a59ed395f9aac01fe717da57"
-dependencies = [
- "cast",
- "itertools",
-]
-
-[[package]]
-name = "crossbeam-channel"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4"
-dependencies = [
- "cfg-if",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-deque"
-version = "0.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
-dependencies = [
- "cfg-if",
- "crossbeam-epoch",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-epoch"
-version = "0.9.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd"
-dependencies = [
- "cfg-if",
- "crossbeam-utils",
- "lazy_static",
- "memoffset",
- "scopeguard",
-]
-
-[[package]]
-name = "crossbeam-utils"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db"
-dependencies = [
- "cfg-if",
- "lazy_static",
-]
-
-[[package]]
-name = "csv"
-version = "1.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1"
-dependencies = [
- "bstr",
- "csv-core",
- "itoa 0.4.8",
- "ryu",
- "serde",
-]
-
-[[package]]
-name = "csv-core"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90"
-dependencies = [
- "memchr",
-]
-
-[[package]]
name = "data-url"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -428,21 +277,6 @@ dependencies = [
]
[[package]]
-name = "half"
-version = "1.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
-
-[[package]]
-name = "hermit-abi"
-version = "0.1.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
-dependencies = [
- "libc",
-]
-
-[[package]]
name = "iai"
version = "0.1.1"
source = "git+https://github.com/reknih/iai#f57a4452e7ac3b118e695e1feddb6ea02c4015be"
@@ -482,27 +316,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
[[package]]
-name = "itoa"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
-
-[[package]]
name = "jpeg-decoder"
version = "0.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2"
[[package]]
-name = "js-sys"
-version = "0.3.55"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cc9ffccd38c451a86bf13657df244e9c3f37493cce8e5e21e940963777acc84"
-dependencies = [
- "wasm-bindgen",
-]
-
-[[package]]
name = "kurbo"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -512,12 +331,6 @@ dependencies = [
]
[[package]]
-name = "lazy_static"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
-
-[[package]]
name = "libc"
version = "0.2.112"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -539,12 +352,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
[[package]]
-name = "memchr"
-version = "2.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
-
-[[package]]
name = "memmap2"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -554,15 +361,6 @@ dependencies = [
]
[[package]]
-name = "memoffset"
-version = "0.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
-dependencies = [
- "autocfg",
-]
-
-[[package]]
name = "miniz_oxide"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -623,35 +421,19 @@ dependencies = [
]
[[package]]
-name = "num_cpus"
-version = "1.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
-dependencies = [
- "hermit-abi",
- "libc",
-]
-
-[[package]]
name = "once_cell"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5"
[[package]]
-name = "oorandom"
-version = "11.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
-
-[[package]]
name = "pdf-writer"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36d760a6f2ac90811cba1006a298e8a7e5ce2c922bb5dc7f7000911a4a6b60f4"
dependencies = [
"bitflags",
- "itoa 0.4.8",
+ "itoa",
"ryu",
]
@@ -662,34 +444,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db8bcd96cb740d03149cbad5518db9fd87126a10ab519c011893b1754134c468"
[[package]]
-name = "plotters"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32a3fd9ec30b9749ce28cd91f255d569591cdf937fe280c312143e3c4bad6f2a"
-dependencies = [
- "num-traits",
- "plotters-backend",
- "plotters-svg",
- "wasm-bindgen",
- "web-sys",
-]
-
-[[package]]
-name = "plotters-backend"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d88417318da0eaf0fdcdb51a0ee6c3bed624333bff8f946733049380be67ac1c"
-
-[[package]]
-name = "plotters-svg"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "521fa9638fa597e1dc53e9412a4f9cefb01187ee1f7413076f9e6749e2885ba9"
-dependencies = [
- "plotters-backend",
-]
-
-[[package]]
name = "png"
version = "0.16.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -779,31 +533,6 @@ dependencies = [
]
[[package]]
-name = "rayon"
-version = "1.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90"
-dependencies = [
- "autocfg",
- "crossbeam-deque",
- "either",
- "rayon-core",
-]
-
-[[package]]
-name = "rayon-core"
-version = "1.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e"
-dependencies = [
- "crossbeam-channel",
- "crossbeam-deque",
- "crossbeam-utils",
- "lazy_static",
- "num_cpus",
-]
-
-[[package]]
name = "rctree"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -829,27 +558,6 @@ dependencies = [
]
[[package]]
-name = "regex"
-version = "1.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461"
-dependencies = [
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-automata"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
-
-[[package]]
-name = "regex-syntax"
-version = "0.6.25"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
-
-[[package]]
name = "resvg"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -883,15 +591,6 @@ dependencies = [
]
[[package]]
-name = "rustc_version"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
-dependencies = [
- "semver",
-]
-
-[[package]]
name = "rustybuzz"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -932,18 +631,6 @@ dependencies = [
]
[[package]]
-name = "scopeguard"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
-
-[[package]]
-name = "semver"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012"
-
-[[package]]
name = "serde"
version = "1.0.132"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -953,16 +640,6 @@ dependencies = [
]
[[package]]
-name = "serde_cbor"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5"
-dependencies = [
- "half",
- "serde",
-]
-
-[[package]]
name = "serde_derive"
version = "1.0.132"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -974,17 +651,6 @@ dependencies = [
]
[[package]]
-name = "serde_json"
-version = "1.0.74"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee2bb9cd061c5865d345bb02ca49fcef1391741b672b54a0bf7b679badec3142"
-dependencies = [
- "itoa 1.0.1",
- "ryu",
- "serde",
-]
-
-[[package]]
name = "simplecss"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1047,15 +713,6 @@ dependencies = [
]
[[package]]
-name = "textwrap"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
-dependencies = [
- "unicode-width",
-]
-
-[[package]]
name = "thiserror"
version = "1.0.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1090,16 +747,6 @@ dependencies = [
]
[[package]]
-name = "tinytemplate"
-version = "1.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
-dependencies = [
- "serde",
- "serde_json",
-]
-
-[[package]]
name = "ttf-parser"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1111,7 +758,6 @@ version = "0.1.0"
dependencies = [
"anyhow",
"codespan-reporting",
- "criterion",
"dirs",
"filedescriptor",
"fxhash",
@@ -1247,70 +893,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
-name = "wasm-bindgen"
-version = "0.2.78"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "632f73e236b219150ea279196e54e610f5dbafa5d61786303d4da54f84e47fce"
-dependencies = [
- "cfg-if",
- "wasm-bindgen-macro",
-]
-
-[[package]]
-name = "wasm-bindgen-backend"
-version = "0.2.78"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a317bf8f9fba2476b4b2c85ef4c4af8ff39c3c7f0cdfeed4f82c34a880aa837b"
-dependencies = [
- "bumpalo",
- "lazy_static",
- "log",
- "proc-macro2",
- "quote",
- "syn",
- "wasm-bindgen-shared",
-]
-
-[[package]]
-name = "wasm-bindgen-macro"
-version = "0.2.78"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9"
-dependencies = [
- "quote",
- "wasm-bindgen-macro-support",
-]
-
-[[package]]
-name = "wasm-bindgen-macro-support"
-version = "0.2.78"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
- "wasm-bindgen-backend",
- "wasm-bindgen-shared",
-]
-
-[[package]]
-name = "wasm-bindgen-shared"
-version = "0.2.78"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0237232789cf037d5480773fe568aac745bfe2afbc11a863e97901780a6b47cc"
-
-[[package]]
-name = "web-sys"
-version = "0.3.55"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38eb105f1c59d9eaa6b5cdc92b859d85b926e82cb2e0945cd0c9259faa6fe9fb"
-dependencies = [
- "js-sys",
- "wasm-bindgen",
-]
-
-[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/Cargo.toml b/Cargo.toml
index 5c4dddcb..8251a7fa 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -61,7 +61,6 @@ walkdir = "2"
# Dependencies updates:
# - Bump ttf-parser when rustybuzz is updated
# - Bump usvg and resvg in conjunction with svg2pdf
-criterion = "0.3"
[[bin]]
name = "typst"
@@ -76,8 +75,3 @@ harness = false
name = "oneshot"
path = "benches/oneshot.rs"
harness = false
-
-[[bench]]
-name = "timed"
-path = "benches/timed.rs"
-harness = false
diff --git a/benches/timed.rs b/benches/timed.rs
deleted file mode 100644
index 83820af2..00000000
--- a/benches/timed.rs
+++ /dev/null
@@ -1,98 +0,0 @@
-use std::path::Path;
-
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-
-use typst::eval::eval;
-use typst::layout::layout;
-use typst::loading::MemLoader;
-use typst::parse::{parse, Scanner, TokenMode, Tokens};
-use typst::source::SourceId;
-use typst::Context;
-
-const SRC: &str = include_str!("bench.typ");
-const FONT: &[u8] = include_bytes!("../fonts/IBMPlexSans-Regular.ttf");
-
-fn context() -> (Context, SourceId) {
- let loader = MemLoader::new().with(Path::new("font.ttf"), FONT).wrap();
- let mut ctx = Context::new(loader);
- let id = ctx.sources.provide(Path::new("src.typ"), SRC.to_string());
- (ctx, id)
-}
-
-fn bench_decode(c: &mut Criterion) {
- c.bench_function("decode", |b| {
- b.iter(|| {
- // We don't use chars().count() because that has a special
- // superfast implementation.
- let mut count = 0;
- let mut chars = black_box(SRC).chars();
- while let Some(_) = chars.next() {
- count += 1;
- }
- count
- })
- });
-}
-
-fn bench_scan(c: &mut Criterion) {
- c.bench_function("scan", |b| {
- b.iter(|| {
- let mut count = 0;
- let mut scanner = Scanner::new(black_box(SRC));
- while let Some(_) = scanner.eat() {
- count += 1;
- }
- count
- })
- });
-}
-
-fn bench_tokenize(c: &mut Criterion) {
- c.bench_function("tokenize", |b| {
- b.iter(|| Tokens::new(black_box(SRC), black_box(TokenMode::Markup)).count())
- });
-}
-
-fn bench_parse(c: &mut Criterion) {
- c.bench_function("parse", |b| b.iter(|| parse(SRC)));
-}
-
-fn bench_edit(c: &mut Criterion) {
- let (mut ctx, id) = context();
- c.bench_function("edit", |b| {
- b.iter(|| black_box(ctx.sources.edit(id, 1168 .. 1171, "_Uhr_")))
- });
-}
-
-fn bench_eval(c: &mut Criterion) {
- let (mut ctx, id) = context();
- let ast = ctx.sources.get(id).ast().unwrap();
- c.bench_function("eval", |b| b.iter(|| eval(&mut ctx, id, &ast).unwrap()));
-}
-
-fn bench_to_tree(c: &mut Criterion) {
- let (mut ctx, id) = context();
- let module = ctx.evaluate(id).unwrap();
- c.bench_function("to_tree", |b| {
- b.iter(|| module.template.to_pages(ctx.style()))
- });
-}
-
-fn bench_layout(c: &mut Criterion) {
- let (mut ctx, id) = context();
- let tree = ctx.execute(id).unwrap();
- c.bench_function("layout", |b| b.iter(|| layout(&mut ctx, &tree)));
-}
-
-criterion_group!(
- benches,
- bench_decode,
- bench_scan,
- bench_tokenize,
- bench_parse,
- bench_edit,
- bench_eval,
- bench_to_tree,
- bench_layout
-);
-criterion_main!(benches);
diff --git a/src/parse/incremental.rs b/src/parse/incremental.rs
index 9c912aae..8e52c143 100644
--- a/src/parse/incremental.rs
+++ b/src/parse/incremental.rs
@@ -1,13 +1,57 @@
use std::ops::Range;
use std::rc::Rc;
-use crate::syntax::{Green, GreenNode, NodeKind, Span};
+use crate::syntax::{Green, GreenNode, NodeKind};
use super::{
parse_atomic, parse_atomic_markup, parse_block, parse_comment, parse_markup,
parse_markup_elements, parse_template, TokenMode,
};
+/// The conditions that a node has to fulfill in order to be replaced.
+///
+/// This can dictate if a node can be replaced at all and if yes, what can take
+/// its place.
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+pub enum Postcondition {
+ /// Changing this node can never have an influence on the other nodes.
+ Safe,
+ /// This node has to be replaced with a single token of the same kind.
+ SameKind(Option<TokenMode>),
+ /// Changing this node into a single atomic expression is allowed if it
+ /// appears in code mode, otherwise it is safe.
+ AtomicPrimary,
+ /// Changing an unsafe layer node changes what the parents or the
+ /// surrounding nodes would be and is therefore disallowed. Change the
+ /// parents or children instead. If it appears in Markup, however, it is
+ /// safe to change.
+ UnsafeLayer,
+ /// Changing an unsafe node or any of its children will trigger undefined
+ /// behavior. Change the parents instead.
+ Unsafe,
+}
+
+/// The conditions under which a node can be inserted or remain in a tree.
+///
+/// These conditions all search the neighbors of the node and see if its
+/// existence is plausible with them present. This can be used to encode some
+/// context-free language components for incremental parsing.
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+pub enum Precondition {
+ /// These nodes depend on being at the start of a line. Reparsing of safe
+ /// left neighbors has to check this invariant. Otherwise, this node is
+ /// safe.
+ AtStart,
+ /// These nodes depend on not being at the start of a line. Reparsing of
+ /// safe left neighbors has to check this invariant. Otherwise, this node is
+ /// safe.
+ NotAtStart,
+ /// These nodes must be followed by whitespace.
+ RightWhitespace,
+ /// No additional requirements.
+ None,
+}
+
/// Allows partial refreshs of the [`Green`] node tree.
///
/// This struct holds a description of a change. Its methods can be used to try
@@ -16,21 +60,21 @@ pub struct Reparser<'a> {
/// The new source code, with the change applied.
src: &'a str,
/// Which range in the old source file was changed.
- replace_range: Span,
- /// How many characters replaced the text in `replacement_range`.
+ replace_range: Range<usize>,
+ /// How many characters replaced the text in `replace_range`.
replace_len: usize,
}
impl<'a> Reparser<'a> {
/// Create a new reparser.
- pub fn new(src: &'a str, replace_range: Span, replace_len: usize) -> Self {
+ pub fn new(src: &'a str, replace_range: Range<usize>, replace_len: usize) -> Self {
Self { src, replace_range, replace_len }
}
}
impl Reparser<'_> {
/// Find the innermost child that is incremental safe.
- pub fn reparse(&self, green: &mut GreenNode) -> Result<Range<usize>, ()> {
+ pub fn reparse(&self, green: &mut GreenNode) -> Option<Range<usize>> {
self.reparse_step(green, 0, TokenMode::Markup, true)
}
@@ -39,30 +83,26 @@ impl Reparser<'_> {
green: &mut GreenNode,
mut offset: usize,
parent_mode: TokenMode,
- outermost: bool,
- ) -> Result<Range<usize>, ()> {
+ mut outermost: bool,
+ ) -> Option<Range<usize>> {
let kind = green.kind().clone();
let mode = kind.mode().unwrap_or(parent_mode);
- let mut loop_result = None;
let mut child_at_start = true;
- let last = green.children().len() - 1;
+ let last = green.children().len().saturating_sub(1);
let mut start = None;
for (i, child) in green.children_mut().iter_mut().enumerate() {
- let child_span =
- Span::new(self.replace_range.source, offset, offset + child.len());
+ let child_span = offset .. offset + child.len();
// We look for the start in the element but we only take a position
// at the right border if this is markup or the last element.
//
// This is because in Markup mode, we want to examine all nodes
// touching a replacement but in code we want to atomically replace.
- if child_span.contains(self.replace_range.start)
- && (mode == TokenMode::Markup
- || self.replace_range.start != child_span.end
- || self.replace_range.len() == 0
- || i == last)
+ if child_span.contains(&self.replace_range.start)
+ || (mode == TokenMode::Markup
+ && self.replace_range.start == child_span.end)
{
start = Some((i, offset));
break;
@@ -72,31 +112,21 @@ impl Reparser<'_> {
child_at_start = child.kind().is_at_start(child_at_start);
}
- let (start_idx, start_offset) = start.ok_or(())?;
+ let (start_idx, start_offset) = start?;
+ let mut end = None;
- for (i, child) in (green.children_mut()[start_idx ..]).iter_mut().enumerate() {
- let i = i + start_idx;
- let child_span =
- Span::new(self.replace_range.source, offset, offset + child.len());
+ for (i, child) in green.children_mut().iter_mut().enumerate().skip(start_idx) {
+ let child_span = offset .. offset + child.len();
// Similarly to above, the end of the edit must be in the node but
// if it is at the edge and we are in markup node, we also want its
// neighbor!
- if child_span.contains(self.replace_range.end)
- && (mode != TokenMode::Markup
- || self.replace_range.end != child_span.end
- || i == last)
+ if child_span.contains(&self.replace_range.end)
+ || self.replace_range.end == child_span.end
+ && (mode != TokenMode::Markup || i == last)
{
- loop_result = Some((
- start_idx .. i + 1,
- Span::new(
- self.replace_range.source,
- start_offset,
- offset + child.len(),
- ),
- i == last && outermost,
- child.kind().clone(),
- ));
+ outermost &= i == last;
+ end = Some(i);
break;
} else if mode != TokenMode::Markup || !child.kind().post().markup_safe() {
break;
@@ -105,28 +135,30 @@ impl Reparser<'_> {
offset += child.len();
}
- let (child_idx_range, child_span, child_outermost, child_kind) =
- loop_result.ok_or(())?;
+ let end = end?;
+ let child_idx_range = start_idx .. end + 1;
+ let child_span = start_offset .. offset + green.children()[end].len();
+ let child_kind = green.children()[end].kind().clone();
if child_idx_range.len() == 1 {
let idx = child_idx_range.start;
let child = &mut green.children_mut()[idx];
+ let prev_len = child.len();
- let old_len = child.len();
// First, we try if the child has another, more specific applicable child.
if !child_kind.post().unsafe_interior() {
- if let Ok(range) = match child {
+ if let Some(range) = match child {
Green::Node(n) => self.reparse_step(
Rc::make_mut(n),
start_offset,
kind.mode().unwrap_or(TokenMode::Code),
- child_outermost,
+ outermost,
),
- Green::Token(_) => Err(()),
+ Green::Token(_) => None,
} {
let new_len = child.len();
- green.update_child_len(new_len, old_len);
- return Ok(range);
+ green.update_child_len(new_len, prev_len);
+ return Some(range);
}
}
}
@@ -134,28 +166,31 @@ impl Reparser<'_> {
debug_assert_ne!(child_idx_range.len(), 0);
if mode == TokenMode::Code && child_idx_range.len() > 1 {
- return Err(());
+ return None;
}
// We now have a child that we can replace and a function to do so.
- let (func, policy) =
- child_kind.reparsing_function(kind.mode().unwrap_or(TokenMode::Code));
- let func = func?;
+ let func =
+ child_kind.reparsing_function(kind.mode().unwrap_or(TokenMode::Code))?;
+ let policy = child_kind.post();
+
+ let len_change = self.replace_len as isize - self.replace_range.len() as isize;
+ let mut src_span = child_span;
+ src_span.end = (src_span.end as isize + len_change) as usize;
- let src_span = inserted_span(child_span, self.replace_range, self.replace_len);
let recompile_range = if policy == Postcondition::AtomicPrimary {
src_span.start .. self.src.len()
} else {
- src_span.to_range()
+ src_span.clone()
};
- let (mut new_children, unterminated) =
- func(&self.src[recompile_range], child_at_start).ok_or(())?;
+ let (mut new_children, terminated) =
+ func(&self.src[recompile_range], child_at_start)?;
// Do not accept unclosed nodes if the old node did not use to be at the
// right edge of the tree.
- if !child_outermost && unterminated {
- return Err(());
+ if !outermost && !terminated {
+ return None;
}
let insertion = match check_invariants(
@@ -164,17 +199,17 @@ impl Reparser<'_> {
child_idx_range.clone(),
child_at_start,
mode,
- src_span,
+ src_span.clone(),
policy,
) {
- InvariantResult::Ok => Ok(new_children),
- InvariantResult::UseFirst => Ok(vec![std::mem::take(&mut new_children[0])]),
- InvariantResult::Error => Err(()),
+ InvariantResult::Ok => Some(new_children),
+ InvariantResult::UseFirst => Some(vec![std::mem::take(&mut new_children[0])]),
+ InvariantResult::Error => None,
}?;
green.replace_child_range(child_idx_range, insertion);
- Ok(src_span.to_range())
+ Some(src_span)
}
}
@@ -191,7 +226,7 @@ fn check_invariants(
child_idx_range: Range<usize>,
child_at_start: bool,
mode: TokenMode,
- src_span: Span,
+ src_span: Range<usize>,
policy: Postcondition,
) -> InvariantResult {
let (new_children, ok) = if policy == Postcondition::AtomicPrimary {
@@ -206,10 +241,7 @@ fn check_invariants(
(use_children, InvariantResult::Ok)
};
- let child_mode = old_children[child_idx_range.start]
- .kind()
- .mode()
- .unwrap_or(TokenMode::Code);
+ let child_mode = old_children[child_idx_range.start].kind().mode().unwrap_or(mode);
// Check if the children / child has the right type.
let same_kind = match policy {
@@ -249,25 +281,22 @@ fn check_invariants(
}
}
- let mut new_at_start = child_at_start;
+ let mut post_at_start = child_at_start;
for child in new_children {
- new_at_start = child.kind().is_at_start(new_at_start);
+ post_at_start = child.kind().is_at_start(post_at_start);
}
for child in &old_children[child_idx_range.end ..] {
if child.kind().is_trivia() {
- new_at_start = child.kind().is_at_start(new_at_start);
+ post_at_start = child.kind().is_at_start(post_at_start);
continue;
}
- match child.kind().pre() {
- Precondition::AtStart if !new_at_start => {
- return InvariantResult::Error;
- }
- Precondition::NotAtStart if new_at_start => {
- return InvariantResult::Error;
- }
- _ => {}
+ let pre = child.kind().pre();
+ if pre == Precondition::AtStart && !post_at_start
+ || pre == Precondition::NotAtStart && post_at_start
+ {
+ return InvariantResult::Error;
}
break;
}
@@ -276,56 +305,31 @@ fn check_invariants(
ok
}
-/// Create a new span by specifying a span in which a modification happened
-/// and how many characters are now in that span.
-fn inserted_span(mut source: Span, other: Span, n: usize) -> Span {
- if !source.surrounds(other) {
- panic!();
- }
-
- let len_change = n as i64 - other.len() as i64;
- source.end = (source.end as i64 + len_change) as usize;
- source
-}
-
impl NodeKind {
/// Return the correct reparsing function given the postconditions for the
/// type.
fn reparsing_function(
&self,
parent_mode: TokenMode,
- ) -> (
- Result<fn(&str, bool) -> Option<(Vec<Green>, bool)>, ()>,
- Postcondition,
- ) {
+ ) -> Option<fn(&str, bool) -> Option<(Vec<Green>, bool)>> {
let policy = self.post();
let mode = self.mode().unwrap_or(parent_mode);
match policy {
- Postcondition::Unsafe | Postcondition::UnsafeLayer => (Err(()), policy),
- Postcondition::AtomicPrimary if mode == TokenMode::Code => {
- (Ok(parse_atomic), policy)
- }
- Postcondition::AtomicPrimary => (Ok(parse_atomic_markup), policy),
- Postcondition::SameKind(x) if x == None || x == Some(mode) => {
- let parser: fn(&str, bool) -> _ = match self {
- NodeKind::Template => parse_template,
- NodeKind::Block => parse_block,
- NodeKind::LineComment | NodeKind::BlockComment => parse_comment,
- _ => return (Err(()), policy),
- };
-
- (Ok(parser), policy)
- }
- _ => {
- let parser: fn(&str, bool) -> _ = match mode {
- TokenMode::Markup if self == &Self::Markup => parse_markup,
- TokenMode::Markup => parse_markup_elements,
- _ => return (Err(()), policy),
- };
-
- (Ok(parser), policy)
- }
+ Postcondition::Unsafe | Postcondition::UnsafeLayer => None,
+ Postcondition::AtomicPrimary if mode == TokenMode::Code => Some(parse_atomic),
+ Postcondition::AtomicPrimary => Some(parse_atomic_markup),
+ Postcondition::SameKind(x) if x == None || x == Some(mode) => match self {
+ NodeKind::Template => Some(parse_template),
+ NodeKind::Block => Some(parse_block),
+ NodeKind::LineComment | NodeKind::BlockComment => Some(parse_comment),
+ _ => None,
+ },
+ _ => match mode {
+ TokenMode::Markup if self == &Self::Markup => Some(parse_markup),
+ TokenMode::Markup => Some(parse_markup_elements),
+ _ => return None,
+ },
}
}
@@ -369,10 +373,6 @@ impl NodeKind {
| Self::Dots
| Self::Arrow => Postcondition::Unsafe,
- // These keywords are literals and can be safely be substituted with
- // other expressions.
- Self::None | Self::Auto => Postcondition::AtomicPrimary,
-
// These keywords change what kind of expression the parent is and
// how far the expression would go.
Self::Let
@@ -389,45 +389,14 @@ impl NodeKind {
| Self::Include
| Self::From => Postcondition::Unsafe,
- Self::Markup => Postcondition::SameKind(None),
-
- Self::Space(_) => Postcondition::SameKind(Some(TokenMode::Code)),
-
- // These are all replaceable by other tokens.
- Self::Parbreak
- | Self::Linebreak
- | Self::Text(_)
- | Self::TextInLine(_)
- | Self::NonBreakingSpace
- | Self::EnDash
- | Self::EmDash
- | Self::Escape(_)
- | Self::Strong
- | Self::Emph
- | Self::Heading
- | Self::Enum
- | Self::List
- | Self::Raw(_)
- | Self::Math(_) => Postcondition::Safe,
-
// Changing the heading level, enum numbering, or list bullet
// changes the next layer.
Self::EnumNumbering(_) => Postcondition::Unsafe,
- // These are expressions that can be replaced by other expressions.
- Self::Ident(_)
- | Self::Bool(_)
- | Self::Int(_)
- | Self::Float(_)
- | Self::Length(_, _)
- | Self::Angle(_, _)
- | Self::Percentage(_)
- | Self::Str(_)
- | Self::Fraction(_)
- | Self::Array
- | Self::Dict
- | Self::Group => Postcondition::AtomicPrimary,
+ Self::Error(_, _) | Self::Unknown(_) => Postcondition::Unsafe,
+ // These are complex expressions which may screw with their
+ // environments.
Self::Call
| Self::Unary
| Self::Binary
@@ -439,11 +408,44 @@ impl NodeKind {
// is not atomic.
Self::Closure | Self::ClosureParams => Postcondition::UnsafeLayer,
+ // Missing these creates errors for the parents.
+ Self::WithExpr | Self::ForPattern | Self::ImportItems => {
+ Postcondition::UnsafeLayer
+ }
+
+ // Only markup is expected at the points where it does occur.
+ Self::Markup => Postcondition::SameKind(None),
+
+ // These can appear everywhere and must not change to other stuff
+ // because that could change the outer expression.
+ Self::LineComment | Self::BlockComment => Postcondition::SameKind(None),
+
// These can appear as bodies and would trigger an error if they
// became something else.
Self::Template => Postcondition::SameKind(None),
Self::Block => Postcondition::SameKind(Some(TokenMode::Code)),
+ // Whitespace in code mode has to remain whitespace or else the type
+ // of things would change.
+ Self::Space(_) => Postcondition::SameKind(Some(TokenMode::Code)),
+
+ // These are expressions that can be replaced by other expressions.
+ Self::Ident(_)
+ | Self::Bool(_)
+ | Self::Int(_)
+ | Self::Float(_)
+ | Self::Length(_, _)
+ | Self::Angle(_, _)
+ | Self::Percentage(_)
+ | Self::Str(_)
+ | Self::Fraction(_)
+ | Self::Array
+ | Self::Dict
+ | Self::Group
+ | Self::None
+ | Self::Auto => Postcondition::AtomicPrimary,
+
+ // More complex, but still an expression.
Self::ForExpr
| Self::WhileExpr
| Self::IfExpr
@@ -452,15 +454,22 @@ impl NodeKind {
| Self::ImportExpr
| Self::IncludeExpr => Postcondition::AtomicPrimary,
- Self::WithExpr | Self::ForPattern | Self::ImportItems => {
- Postcondition::UnsafeLayer
- }
-
- // These can appear everywhere and must not change to other stuff
- // because that could change the outer expression.
- Self::LineComment | Self::BlockComment => Postcondition::SameKind(None),
-
- Self::Error(_, _) | Self::Unknown(_) => Postcondition::Unsafe,
+ // These are all replaceable by other tokens.
+ Self::Parbreak
+ | Self::Linebreak
+ | Self::Text(_)
+ | Self::TextInLine(_)
+ | Self::NonBreakingSpace
+ | Self::EnDash
+ | Self::EmDash
+ | Self::Escape(_)
+ | Self::Strong
+ | Self::Emph
+ | Self::Heading
+ | Self::Enum
+ | Self::List
+ | Self::Raw(_)
+ | Self::Math(_) => Postcondition::Safe,
}
}
@@ -475,50 +484,6 @@ impl NodeKind {
}
}
-/// The conditions that a node has to fulfill in order to be replaced.
-///
-/// This can dictate if a node can be replaced at all and if yes, what can take
-/// its place.
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
-pub enum Postcondition {
- /// Changing this node can never have an influence on the other nodes.
- Safe,
- /// This node has to be replaced with a single token of the same kind.
- SameKind(Option<TokenMode>),
- /// Changing this node into a single atomic expression is allowed if it
- /// appears in code mode, otherwise it is safe.
- AtomicPrimary,
- /// Changing an unsafe layer node changes what the parents or the
- /// surrounding nodes would be and is therefore disallowed. Change the
- /// parents or children instead. If it appears in Markup, however, it is
- /// safe to change.
- UnsafeLayer,
- /// Changing an unsafe node or any of its children will trigger undefined
- /// behavior. Change the parents instead.
- Unsafe,
-}
-
-/// The conditions under which a node can be inserted or remain in a tree.
-///
-/// These conditions all search the neighbors of the node and see if its
-/// existence is plausible with them present. This can be used to encode some
-/// context-free language components for incremental parsing.
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
-pub enum Precondition {
- /// These nodes depend on being at the start of a line. Reparsing of safe
- /// left neighbors has to check this invariant. Otherwise, this node is
- /// safe.
- AtStart,
- /// These nodes depend on not being at the start of a line. Reparsing of
- /// safe left neighbors has to check this invariant. Otherwise, this node is
- /// safe.
- NotAtStart,
- /// These nodes must be followed by whitespace.
- RightWhitespace,
- /// No additional requirements.
- None,
-}
-
impl Postcondition {
pub fn unsafe_interior(&self) -> bool {
match self {
@@ -544,6 +509,7 @@ mod tests {
use super::*;
#[test]
+ #[rustfmt::skip]
fn test_incremental_parse() {
#[track_caller]
fn test(prev: &str, range: Range<usize>, with: &str, incr: Range<usize>) {
@@ -551,12 +517,14 @@ mod tests {
let range = source.edit(range, with);
assert_eq!(range, incr);
- let incr_tree = source.root();
+ let incr_tree = source.root().clone();
assert_eq!(parse(source.src()), incr_tree);
}
// Test simple replacements.
- test("hello world", 6 .. 11, "wankers", 5 .. 13);
+ test("hello world", 6 .. 11, "walkers", 5 .. 13);
+ test("some content", 0..12, "", 0..0);
+ test("", 0..0, "do it", 0..5);
test("a d e", 1 .. 3, " b c d", 0 .. 8);
test("a #f() e", 1 .. 6, " b c d", 0 .. 8);
test("{(0, 1, 2)}", 5 .. 6, "11pt", 5 .. 9);
@@ -564,53 +532,18 @@ mod tests {
test("your thing", 5 .. 5, "a", 4 .. 11);
test("a your thing a", 6 .. 7, "a", 2 .. 12);
test("{call(); abc}", 7 .. 7, "[]", 0 .. 15);
- test("#call() abc", 7 .. 7, "[]", 0 .. 13);
- // test(
- // "hi\n- item\n- item 2\n - item 3",
- // 10 .. 10,
- // " ",
- // 9 .. 33,
- // );
- test(
- "#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])",
- 16 .. 20,
- "none",
- 16 .. 20,
- );
- test(
- "#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])",
- 33 .. 42,
- "[_gronk_]",
- 33 .. 42,
- );
- test(
- "#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])",
- 34 .. 41,
- "_bar_",
- 34 .. 39,
- );
+ test("#call() abc", 7 .. 7, "[]", 0 .. 10);
+ // test("hi\n- item\n- item 2\n - item 3", 10 .. 10, " ", 9 .. 33);
+ test("#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", 16 .. 20, "none", 16 .. 20);
+ test("#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", 33 .. 42, "[_gronk_]", 33 .. 42);
+ test("#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", 34 .. 41, "_bar_", 34 .. 39);
test("{let i=1; for x in range(5) {i}}", 6 .. 6, " ", 1 .. 9);
- test("{let i=1; for x in range(5) {i}}", 13 .. 14, " ", 13 .. 15);
+ test("{let i=1; for x in range(5) {i}}", 13 .. 14, " ", 10 .. 32);
test("hello {x}", 6 .. 9, "#f()", 5 .. 10);
- test(
- "this is -- in my opinion -- spectacular",
- 8 .. 10,
- "---",
- 7 .. 12,
- );
- test(
- "understanding `code` is complicated",
- 15 .. 15,
- "C ",
- 14 .. 22,
- );
+ test("this is -- in my opinion -- spectacular", 8 .. 10, "---", 7 .. 12);
+ test("understanding `code` is complicated", 15 .. 15, "C ", 14 .. 22);
test("{ let x = g() }", 10 .. 12, "f(54", 0 .. 17);
- test(
- "a #let rect with (fill: eastern)\nb",
- 16 .. 31,
- " (stroke: conifer",
- 2 .. 34,
- );
+ test("a #let rect with (fill: eastern)\nb", 16 .. 31, " (stroke: conifer", 2 .. 34);
// Test the whitespace invariants.
test("hello \\ world", 7 .. 8, "a ", 6 .. 14);
@@ -642,18 +575,8 @@ mod tests {
test(r"{{let x = z}; a = 1} b", 6 .. 6, "//", 0 .. 24);
test("a b c", 1 .. 1, " /* letters */", 0 .. 16);
test("a b c", 1 .. 1, " /* letters", 0 .. 16);
- test(
- "{if i==1 {a} else [b]; b()}",
- 12 .. 12,
- " /* letters */",
- 1 .. 35,
- );
- test(
- "{if i==1 {a} else [b]; b()}",
- 12 .. 12,
- " /* letters",
- 0 .. 38,
- );
+ test("{if i==1 {a} else [b]; b()}", 12 .. 12, " /* letters */", 1 .. 35);
+ test("{if i==1 {a} else [b]; b()}", 12 .. 12, " /* letters", 0 .. 38);
test(r#"a ```typst hello``` b"#, 16 .. 17, "", 0 .. 20);
test(r#"a ```typst hello```"#, 16 .. 17, "", 2 .. 18);
diff --git a/src/parse/mod.rs b/src/parse/mod.rs
index 2c421374..2c5afb6b 100644
--- a/src/parse/mod.rs
+++ b/src/parse/mod.rs
@@ -64,7 +64,7 @@ pub fn parse_markup_elements(
/// Parse a template literal. Returns `Some` if all of the input was consumed.
pub fn parse_template(source: &str, _: bool) -> Option<(Vec<Green>, bool)> {
let mut p = Parser::new(source, TokenMode::Code);
- if !matches!(p.peek(), Some(NodeKind::LeftBracket)) {
+ if !p.at(&NodeKind::LeftBracket) {
return None;
}
@@ -75,7 +75,7 @@ pub fn parse_template(source: &str, _: bool) -> Option<(Vec<Green>, bool)> {
/// Parse a code block. Returns `Some` if all of the input was consumed.
pub fn parse_block(source: &str, _: bool) -> Option<(Vec<Green>, bool)> {
let mut p = Parser::new(source, TokenMode::Code);
- if !matches!(p.peek(), Some(NodeKind::LeftBrace)) {
+ if !p.at(&NodeKind::LeftBrace) {
return None;
}
@@ -252,14 +252,14 @@ fn expr_prec(p: &mut Parser, atomic: bool, min_prec: usize) -> ParseResult {
let marker = p.marker();
// Start the unary expression.
- match (!atomic).then(|| p.peek().and_then(UnOp::from_token)).flatten() {
- Some(op) => {
+ match p.peek().and_then(UnOp::from_token) {
+ Some(op) if !atomic => {
p.eat();
let prec = op.precedence();
expr_prec(p, atomic, prec)?;
marker.end(p, NodeKind::Unary);
}
- None => primary(p, atomic)?,
+ _ => primary(p, atomic)?,
};
loop {
diff --git a/src/parse/parser.rs b/src/parse/parser.rs
index 06cb1578..ade9b5df 100644
--- a/src/parse/parser.rs
+++ b/src/parse/parser.rs
@@ -22,7 +22,7 @@ pub struct Parser<'s> {
/// The children of the currently built node.
children: Vec<Green>,
/// Whether the last group was terminated.
- last_group_terminated: bool,
+ last_terminated: bool,
}
impl<'s> Parser<'s> {
@@ -38,7 +38,7 @@ impl<'s> Parser<'s> {
current_start: 0,
groups: vec![],
children: vec![],
- last_group_terminated: true,
+ last_terminated: true,
}
}
@@ -50,7 +50,7 @@ impl<'s> Parser<'s> {
/// End the parsing process and return multiple children.
pub fn eject(self) -> Option<(Vec<Green>, bool)> {
if self.eof() && self.group_success() {
- Some((self.children, self.tokens.was_unterminated()))
+ Some((self.children, self.tokens.was_terminated()))
} else {
None
}
@@ -99,7 +99,7 @@ impl<'s> Parser<'s> {
/// remains stuff in the string.
pub fn eject_partial(self) -> Option<(Vec<Green>, bool)> {
self.group_success()
- .then(|| (self.children, self.tokens.was_unterminated()))
+ .then(|| (self.children, self.tokens.was_terminated()))
}
/// Whether the end of the source string or group is reached.
@@ -244,7 +244,7 @@ impl<'s> Parser<'s> {
let group = self.groups.pop().expect("no started group");
self.tokens.set_mode(group.prev_mode);
self.repeek();
- self.last_group_terminated = true;
+ self.last_terminated = true;
let mut rescan = self.tokens.mode() != group_mode;
@@ -263,7 +263,7 @@ impl<'s> Parser<'s> {
rescan = false;
} else if required {
self.push_error(format_eco!("expected {}", end));
- self.last_group_terminated = false;
+ self.last_terminated = false;
}
}
@@ -283,7 +283,7 @@ impl<'s> Parser<'s> {
/// Check if the group processing was successfully terminated.
pub fn group_success(&self) -> bool {
- self.last_group_terminated && self.groups.is_empty()
+ self.last_terminated && self.groups.is_empty()
}
/// Low-level bump that consumes exactly one token without special trivia
diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs
index 7be31fe1..836e8cf1 100644
--- a/src/parse/tokens.rs
+++ b/src/parse/tokens.rs
@@ -13,7 +13,7 @@ use crate::util::EcoString;
pub struct Tokens<'s> {
s: Scanner<'s>,
mode: TokenMode,
- has_unterminated: bool,
+ was_terminated: bool,
}
/// What kind of tokens to emit.
@@ -32,7 +32,7 @@ impl<'s> Tokens<'s> {
Self {
s: Scanner::new(src),
mode,
- has_unterminated: false,
+ was_terminated: true,
}
}
@@ -69,10 +69,10 @@ impl<'s> Tokens<'s> {
self.s
}
- /// Whether the last token was unterminated.
+ /// Whether the last token was terminated.
#[inline]
- pub fn was_unterminated(&self) -> bool {
- self.has_unterminated
+ pub fn was_terminated(&self) -> bool {
+ self.was_terminated
}
}
@@ -259,7 +259,7 @@ impl<'s> Tokens<'s> {
)
}
} else {
- self.has_unterminated = true;
+ self.was_terminated = false;
NodeKind::Error(
ErrorPos::End,
"expected closing brace".into(),
@@ -358,7 +358,7 @@ impl<'s> Tokens<'s> {
let remaining = backticks - found;
let noun = if remaining == 1 { "backtick" } else { "backticks" };
- self.has_unterminated = true;
+ self.was_terminated = false;
NodeKind::Error(
ErrorPos::End,
if found == 0 {
@@ -406,7 +406,7 @@ impl<'s> Tokens<'s> {
display,
}))
} else {
- self.has_unterminated = true;
+ self.was_terminated = false;
NodeKind::Error(
ErrorPos::End,
if !display || (!escaped && dollar) {
@@ -495,7 +495,7 @@ impl<'s> Tokens<'s> {
if self.s.eat_if('"') {
NodeKind::Str(string)
} else {
- self.has_unterminated = true;
+ self.was_terminated = false;
NodeKind::Error(ErrorPos::End, "expected quote".into())
}
}
@@ -503,7 +503,7 @@ impl<'s> Tokens<'s> {
fn line_comment(&mut self) -> NodeKind {
self.s.eat_until(is_newline);
if self.s.peek().is_none() {
- self.has_unterminated = true;
+ self.was_terminated = false;
}
NodeKind::LineComment
}
@@ -511,7 +511,7 @@ impl<'s> Tokens<'s> {
fn block_comment(&mut self) -> NodeKind {
let mut state = '_';
let mut depth = 1;
- let mut terminated = false;
+ self.was_terminated = false;
// Find the first `*/` that does not correspond to a nested `/*`.
while let Some(c) = self.s.eat() {
@@ -519,7 +519,7 @@ impl<'s> Tokens<'s> {
('*', '/') => {
depth -= 1;
if depth == 0 {
- terminated = true;
+ self.was_terminated = true;
break;
}
'_'
@@ -532,10 +532,6 @@ impl<'s> Tokens<'s> {
}
}
- if !terminated {
- self.has_unterminated = true;
- }
-
NodeKind::BlockComment
}
diff --git a/src/source.rs b/src/source.rs
index 421412ee..6cca9f75 100644
--- a/src/source.rs
+++ b/src/source.rs
@@ -12,7 +12,7 @@ use crate::diag::TypResult;
use crate::loading::{FileHash, Loader};
use crate::parse::{is_newline, parse, Reparser, Scanner};
use crate::syntax::ast::Markup;
-use crate::syntax::{self, Category, GreenNode, RedNode, Span};
+use crate::syntax::{self, Category, GreenNode, RedNode};
use crate::util::PathExt;
#[cfg(feature = "codespan-reporting")]
@@ -265,7 +265,8 @@ impl SourceFile {
/// Edit the source file by replacing the given range.
///
- /// This panics if the `replace` range is out of bounds.
+ /// Returns the range of the section in the new source that was ultimately
+ /// reparsed. The method panics if the `replace` range is out of bounds.
pub fn edit(&mut self, replace: Range<usize>, with: &str) -> Range<usize> {
let start = replace.start;
self.src.replace_range(replace.clone(), with);
@@ -284,9 +285,8 @@ impl SourceFile {
.extend(newlines(&self.src[start ..]).map(|idx| start + idx));
// Update the root node.
- let span = Span::new(self.id, replace.start, replace.end);
- let reparser = Reparser::new(&self.src, span, with.len());
- if let Ok(range) = reparser.reparse(Rc::make_mut(&mut self.root)) {
+ let reparser = Reparser::new(&self.src, replace, with.len());
+ if let Some(range) = reparser.reparse(Rc::make_mut(&mut self.root)) {
range
} else {
self.root = parse(&self.src);
@@ -302,12 +302,6 @@ impl SourceFile {
let red = RedNode::from_root(self.root.clone(), self.id);
syntax::highlight(red.as_ref(), range, &mut f)
}
-
- /// Obtain a reference to the source's root green node.
- #[cfg(test)]
- pub(crate) fn root(&self) -> Rc<GreenNode> {
- self.root.clone()
- }
}
/// The indices at which lines start (right behind newlines).
diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs
index 9ab530d8..b72e5843 100644
--- a/src/syntax/mod.rs
+++ b/src/syntax/mod.rs
@@ -48,15 +48,6 @@ impl Green {
self.data().len()
}
- /// Set the length of the node.
- pub fn set_len(&mut self, len: usize) {
- let data = match self {
- Self::Node(node) => &mut Rc::make_mut(node).data,
- Self::Token(data) => data,
- };
- data.set_len(len);
- }
-
/// Whether the node or its children contain an error.
pub fn erroneous(&self) -> bool {
match self {
@@ -139,11 +130,6 @@ impl GreenNode {
&self.children
}
- /// The node's children, mutably.
- pub fn children_mut(&mut self) -> &mut [Green] {
- &mut self.children
- }
-
/// The node's metadata.
pub fn data(&self) -> &GreenData {
&self.data
@@ -159,10 +145,15 @@ impl GreenNode {
self.data().len()
}
+ /// The node's children, mutably.
+ pub(crate) fn children_mut(&mut self) -> &mut [Green] {
+ &mut self.children
+ }
+
/// Replaces a range of children with some replacement.
///
/// This method updates the `erroneous` and `data.len` fields.
- pub fn replace_child_range(
+ pub(crate) fn replace_child_range(
&mut self,
child_idx_range: Range<usize>,
replacement: Vec<Green>,
@@ -187,12 +178,12 @@ impl GreenNode {
self.erroneous = self.erroneous || replacement.iter().any(Green::erroneous);
self.children.splice(child_idx_range, replacement);
- self.data.set_len(self.data.len + new_len - old_len);
+ self.data.len = self.data.len + new_len - old_len;
}
/// Update the length of this node given the old and new length of a
/// replaced child.
- pub fn update_child_len(&mut self, new_len: usize, old_len: usize) {
+ pub(crate) fn update_child_len(&mut self, new_len: usize, old_len: usize) {
self.data.len = self.data.len() + new_len - old_len;
self.erroneous = self.children.iter().any(|x| x.erroneous());
}
@@ -246,11 +237,6 @@ impl GreenData {
pub fn len(&self) -> usize {
self.len
}
-
- /// Set the length of the node.
- pub fn set_len(&mut self, len: usize) {
- self.len = len;
- }
}
impl From<GreenData> for Green {
@@ -261,7 +247,7 @@ impl From<GreenData> for Green {
impl Debug for GreenData {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
- write!(f, "{:?}: {}", self.kind, self.len)
+ write!(f, "{:?}: {}", &self.kind, self.len)
}
}
@@ -375,11 +361,6 @@ impl<'a> RedRef<'a> {
Span::new(self.id, self.offset, self.offset + self.green.len())
}
- /// Whether the node or its children contain an error.
- pub fn erroneous(self) -> bool {
- self.green.erroneous()
- }
-
/// The error messages for this node and its descendants.
pub fn errors(self) -> Vec<Error> {
if !self.green.erroneous() {
@@ -731,19 +712,12 @@ impl NodeKind {
/// Whether this is whitespace.
pub fn is_whitespace(&self) -> bool {
- match self {
- Self::Space(_) | Self::Parbreak => true,
- _ => false,
- }
+ matches!(self, Self::Space(_) | Self::Parbreak)
}
/// Whether this is trivia.
pub fn is_trivia(&self) -> bool {
- match self {
- _ if self.is_whitespace() => true,
- Self::LineComment | Self::BlockComment => true,
- _ => false,
- }
+ self.is_whitespace() || matches!(self, Self::LineComment | Self::BlockComment)
}
/// Whether this is some kind of error.
@@ -765,7 +739,6 @@ impl NodeKind {
pub fn mode(&self) -> Option<TokenMode> {
match self {
Self::Markup
- | Self::Space(_)
| Self::Linebreak
| Self::Parbreak
| Self::Text(_)
@@ -783,6 +756,7 @@ impl NodeKind {
| Self::Raw(_)
| Self::Math(_) => Some(TokenMode::Markup),
Self::Template
+ | Self::Space(_)
| Self::Block
| Self::Ident(_)
| Self::LetExpr