diff options
| -rw-r--r-- | src/parse/tokens.rs | 23 | ||||
| -rw-r--r-- | src/syntax/mod.rs | 5 | ||||
| -rw-r--r-- | src/syntax/node.rs | 53 | ||||
| -rw-r--r-- | tests/lang/ref/raw.png | bin | 4071 -> 4459 bytes | |||
| -rw-r--r-- | tests/lang/typ/raw.typ | 15 |
5 files changed, 54 insertions, 42 deletions
diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs index 71e71e3e..405352c3 100644 --- a/src/parse/tokens.rs +++ b/src/parse/tokens.rs @@ -227,6 +227,11 @@ impl<'s> Tokens<'s> { backticks += 1; } + // Special case for empty inline block. + if backticks == 2 { + return Token::Raw(TokenRaw { text: "", backticks: 1, terminated: true }); + } + let start = self.s.index(); let mut found = 0; @@ -723,21 +728,25 @@ mod tests { #[test] fn test_tokenize_raw_blocks() { + let empty = Raw("", 1, true); + // Test basic raw block. + t!(Markup: "``" => empty); t!(Markup: "`raw`" => Raw("raw", 1, true)); t!(Markup[""]: "`]" => Raw("]", 1, false)); // Test special symbols in raw block. - t!(Markup: "`[func]`" => Raw("[func]", 1, true)); - t!(Markup[""]: r"`\`` " => Raw(r"\", 1, true), Raw(" ", 1, false)); - - // Test more backticks. - t!(Markup: "````🚀````" => Raw("🚀", 4, true)); - t!(Markup[""]: "````👩🚀``noend" => Raw("👩🚀``noend", 4, false)); - t!(Markup[""]: "````raw``````new" => Raw("raw", 4, true), Raw("new", 2, false)); + t!(Markup: "`[brackets]`" => Raw("[brackets]", 1, true)); + t!(Markup[""]: r"`\`` " => Raw(r"\", 1, true), Raw(" ", 1, false)); // Test separated closing backticks. t!(Markup: "```not `y`e`t```" => Raw("not `y`e`t", 3, true)); + + // Test more backticks. + t!(Markup: "``nope``" => empty, Text("nope"), empty); + t!(Markup: "````🚀````" => Raw("🚀", 4, true)); + t!(Markup[""]: "`````👩🚀````noend" => Raw("👩🚀````noend", 5, false)); + t!(Markup[""]: "````raw``````" => Raw("raw", 4, true), empty); } #[test] diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs index 41fba134..409e8cbf 100644 --- a/src/syntax/mod.rs +++ b/src/syntax/mod.rs @@ -63,9 +63,10 @@ mod tests { roundtrip("= *Ok*"); // Raw. + roundtrip("``"); roundtrip("`lang 1`"); - test("`` hi``", "`hi`"); - test("`` ` ``", "```"); + test("``` hi```", "`hi`"); + test("``` ` ```", "```"); } #[test] diff --git a/src/syntax/node.rs b/src/syntax/node.rs index d45e5952..f7625036 100644 --- a/src/syntax/node.rs +++ b/src/syntax/node.rs @@ -66,11 +66,11 @@ impl Pretty for NodeHeading { /// A raw block with optional syntax highlighting: `` `raw` ``. /// -/// Raw blocks start with an arbitrary number of backticks and end with the same -/// number of backticks. If you want to include a sequence of backticks in a raw -/// block, simply surround the block with more backticks. +/// Raw blocks start with 1 or 3+ backticks and end with the same number of +/// backticks. If you want to include a sequence of backticks in a raw block, +/// simply surround the block with more backticks. /// -/// When using at least two backticks, an optional language tag may follow +/// When using at least three backticks, an optional language tag may follow /// directly after the backticks. This tag defines which language to /// syntax-highlight the text in. Apart from the language tag and some /// whitespace trimming discussed below, everything inside a raw block is @@ -82,22 +82,21 @@ impl Pretty for NodeHeading { /// `raw` /// ``` /// - An optional language tag may follow directly at the start when the block -/// is surrounded by at least two backticks. -/// ```typst -/// ``rust println!("hello!")``; -/// ``` -/// - Blocks can span multiple lines. Two backticks suffice to be able to -/// specify the language tag, but three are fine, too. -/// ```typst -/// ``rust +/// is surrounded by at least three backticks. +/// ````typst +/// ```rust println!("hello!")```; +/// ```` +/// - Blocks can span multiple lines. +/// ````typst +/// ```rust /// loop { /// find_yak().shave(); /// } -/// `` /// ``` -/// - Start with a space to omit the language tag (the space will be trimmed -/// from the output) and use more backticks to allow backticks in the raw -/// text. +/// ```` +/// - Start with a space to omit the language tag (the space will be trimmed +/// from the output) and use more backticks to allow backticks in the raw +/// text. /// `````typst /// ```` This contains ```backticks``` and has no leading & trailing spaces. ```` /// ````` @@ -107,24 +106,22 @@ impl Pretty for NodeHeading { /// given, a few things would become problematic or even impossible: /// - Typical multiline code blocks (like in the example above) would have an /// additional newline before and after the code. -/// - Raw text wrapped in more than one backtick could not exist without -/// leading whitespace since the first word would be interpreted as a -/// language tag. +/// - The first word of text wrapped in more than three backticks would always +/// be interpreted as a language tag which means that text without leading +/// space would be impossible. /// - A single backtick without surrounding spaces could not exist as raw text /// since it would be interpreted as belonging to the opening or closing /// backticks. /// -/// To fix these problems, we trim text in multi-backtick blocks as follows: -/// - We trim a single space or a sequence of whitespace followed by a newline -/// at the start. -/// - We trim a single space or a newline followed by a sequence of whitespace -/// at the end. +/// To fix these problems, we trim blocks with 3+ backticks as follows: +/// - A single space or a sequence of whitespace followed by a newline at the start. +/// - A single space or a newline followed by a sequence of whitespace at the end. /// /// With these rules, a single raw backtick can be produced by the sequence -/// ``` `` ` `` ```, ``` `` unhighlighted text `` ``` has no surrounding -/// spaces and multiline code blocks don't have extra empty lines. Note that -/// you can always force leading or trailing whitespace simply by adding more -/// spaces. +/// ```` ``` ` ``` ````, ```` ``` unhighlighted text ``` ```` has no +/// surrounding spaces and multiline code blocks don't have extra empty lines. +/// Note that you can always force leading or trailing whitespace simply by +/// adding more spaces. #[derive(Debug, Clone, PartialEq)] pub struct NodeRaw { /// An optional identifier specifying the language to syntax-highlight in. diff --git a/tests/lang/ref/raw.png b/tests/lang/ref/raw.png Binary files differindex 8b68922a..2a5b083d 100644 --- a/tests/lang/ref/raw.png +++ b/tests/lang/ref/raw.png diff --git a/tests/lang/typ/raw.typ b/tests/lang/typ/raw.typ index a1f796b5..36149dc8 100644 --- a/tests/lang/typ/raw.typ +++ b/tests/lang/typ/raw.typ @@ -1,22 +1,27 @@ #[font 8pt] +// No extra space. +`A``B` + // Typst syntax inside. -`#let x = 1``[f 1]` +`#let x = 1` \ +`#[f 1]` // Space between "rust" and "let" is trimmed. -The keyword ``rust let``. +The keyword ```rust let```. // Trimming depends on number backticks. +<``> \ <` untrimmed `> \ -<`` trimmed ``> +<``` trimmed ```> // Multiline trimming. -``py +```py import this def say_hi(): print("Hi!") -`` +``` // Lots of backticks inside. ```` |
