Support syntax highlighting for more languages (#771)

Co-authored-by: Laurenz <laurmaedje@gmail.com>

6 files changed, 68 insertions, 37 deletions

diff --git a/NOTICE b/NOTICE
index 46834822..f1dc8950 100644
--- a/NOTICE
+++ b/NOTICE
@@ -63,6 +63,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ================================================================================
 
 ================================================================================
+Syntaxes in `assets/data/syntect.bin` are generated from syntax definitions
+which can be found here:
+https://github.com/sharkdp/bat/tree/master/assets/syntaxes
+
+The syntax definitions are used under their respective licenses.
+================================================================================
+
+================================================================================
 The SIL Open Font License Version 1.1 applies to:
 
 * IBM Plex fonts in assets/fonts/IBMPlex*.ttf
diff --git a/assets/data/syntect.bin b/assets/data/syntect.bin
new file mode 100644
index 00000000..748cdfe6
--- /dev/null
+++ b/assets/data/syntect.bin
diff --git a/library/Cargo.toml b/library/Cargo.toml
index 02dedd3e..b6e06c8e 100644
--- a/library/Cargo.toml
+++ b/library/Cargo.toml
@@ -34,7 +34,7 @@ rustybuzz = "0.7"
 serde_json = "1"
 serde_yaml = "0.8"
 smallvec = "1.10"
-syntect = { version = "5", default-features = false, features = ["default-syntaxes", "regex-fancy"] }
+syntect = { version = "5", default-features = false, features = ["parsing", "regex-fancy"] }
 time = { version = "0.3.20", features = ["formatting"] }
 toml = { version = "0.7.3", default-features = false, features = ["parse"] }
 tracing = "0.1.37"
diff --git a/library/src/text/raw.rs b/library/src/text/raw.rs
index 5929c04c..75e8ddc7 100644
--- a/library/src/text/raw.rs
+++ b/library/src/text/raw.rs
@@ -313,8 +313,33 @@ fn to_syn(RgbaColor { r, g, b, a }: RgbaColor) -> synt::Color {
 }
 
 /// The syntect syntax definitions.
-static SYNTAXES: Lazy<syntect::parsing::SyntaxSet> =
-    Lazy::new(syntect::parsing::SyntaxSet::load_defaults_nonewlines);
+///
+/// Code for syntax set generation is below. The `syntaxes` directory is from
+/// <https://github.com/sharkdp/bat/tree/master/assets/syntaxes>
+///
+/// ```ignore
+/// fn main() {
+///     let mut builder = syntect::parsing::SyntaxSet::load_defaults_nonewlines().into_builder();
+///     builder.add_from_folder("syntaxes/02_Extra", false).unwrap();
+///     syntect::dumps::dump_to_file(&builder.build(), "syntect.bin").unwrap();
+/// }
+/// ```
+///
+/// The following syntaxes are disabled due to compatibility issues:
+/// ```text
+/// syntaxes/02_Extra/Assembly (ARM).sublime-syntax
+/// syntaxes/02_Extra/Elixir/Regular Expressions (Elixir).sublime-syntax
+/// syntaxes/02_Extra/JavaScript (Babel).sublime-syntax
+/// syntaxes/02_Extra/LiveScript.sublime-syntax
+/// syntaxes/02_Extra/PowerShell.sublime-syntax
+/// syntaxes/02_Extra/SCSS_Sass/Syntaxes/Sass.sublime-syntax
+/// syntaxes/02_Extra/SLS/SLS.sublime-syntax
+/// syntaxes/02_Extra/VimHelp.sublime-syntax
+/// syntaxes/02_Extra/cmd-help/syntaxes/cmd-help.sublime-syntax
+/// ```
+static SYNTAXES: Lazy<syntect::parsing::SyntaxSet> = Lazy::new(|| {
+    syntect::dumps::from_binary(include_bytes!("../../../assets/data/syntect.bin"))
+});
 
 /// The default theme used for syntax highlighting.
 pub static THEME: Lazy<synt::Theme> = Lazy::new(|| synt::Theme {
diff --git a/tests/ref/text/raw-code.png b/tests/ref/text/raw-code.png
index 25735e9d..f772ba41 100644
--- a/tests/ref/text/raw-code.png
+++ b/tests/ref/text/raw-code.png
diff --git a/tests/typ/text/raw-code.typ b/tests/typ/text/raw-code.typ
index 41d20ead..ca7b247d 100644
--- a/tests/typ/text/raw-code.typ
+++ b/tests/typ/text/raw-code.typ
@@ -3,6 +3,14 @@
 ---
 #set page(width: 180pt)
 #set text(6pt)
+#show raw: it => rect(
+  width: 100%,
+  inset: (x: 4pt, y: 5pt),
+  radius: 4pt,
+  fill: rgb(239, 241, 243),
+  place(right, text(luma(110), it.lang)) + it,
+)
+
 ```typ
 = Chapter 1
 #lorem(100)
@@ -11,10 +19,6 @@
 #show heading: emph
 ```
 
----
-#set page(width: 180pt)
-#set text(6pt)
-
 ```rust
 /// A carefully designed state machine.
 #[derive(Debug)]
@@ -25,10 +29,6 @@ fn advance(state: State<'_>) -> State<'_> {
 }
 ```
 
----
-#set page(width: 180pt)
-#set text(6pt)
-
 ```py
 import this
 
@@ -36,10 +36,6 @@ def hi():
   print("Hi!")
 ```
 
----
-#set page(width: 180pt)
-#set text(6pt)
-
 ```cpp
 #include <iostream>
 
@@ -48,26 +44,28 @@ int main() {
 }
 ```
 
----
-#set page(width: 180pt)
-#set text(6pt)
+```julia
+# Add two numbers
+function add(x, y)
+    return x * y
+end
+```
 
-#rect(inset: (x: 4pt, y: 5pt), radius: 4pt, fill: rgb(239, 241, 243))[
-  ```html
-  <!DOCTYPE html>
-  <html>
-    <head>
-      <meta charset="utf-8">
-    </head>
-    <body>
-      <h1>Topic</h1>
-      <p>The Hypertext Markup Language.</p>
-      <script>
-        function foo(a, b) {
-          return a + b + "string";
-        }
-      </script>
-    </body>
-  </html>
-  ```
-]
+    // Try with some indent.
+    ```html
+    <!DOCTYPE html>
+    <html>
+      <head>
+        <meta charset="utf-8">
+      </head>
+      <body>
+        <h1>Topic</h1>
+        <p>The Hypertext Markup Language.</p>
+        <script>
+          function foo(a, b) {
+            return a + b + "string";
+          }
+        </script>
+      </body>
+    </html>
+    ```