chore(tools/cheddar): bump to syntect 5.0 r/5622

Upgrade to syntect 5.0 and load the new kind of syntax set
serialisation with the new helper function for that purpose.

Includes other minor API fixes as well, note that the things that are
now calling `expect` previously failed internally at those points and
we're reasonably confident they don't fail in production.

This has been waiting for a long time ...

Change-Id: I8af4fef995ff64bfbe24e1f13917fa50ecb6e4ad
Reviewed-on: https://cl.tvl.fyi/c/depot/+/7787
Reviewed-by: sterni <sternenseemann@systemli.org>
Tested-by: BuildkiteCI

4 files changed, 22 insertions, 37 deletions

diff --git a/tools/cheddar/Cargo.lock b/tools/cheddar/Cargo.lock
index 71eb0a293e..2787d79cb7 100644
--- a/tools/cheddar/Cargo.lock
+++ b/tools/cheddar/Cargo.lock
@@ -184,7 +184,7 @@ dependencies = [
  "rouille",
  "serde",
  "serde_json",
- "syntect 4.6.0",
+ "syntect",
 ]
 
 [[package]]
@@ -282,7 +282,7 @@ dependencies = [
  "pest_derive",
  "regex",
  "shell-words",
- "syntect 5.0.0",
+ "syntect",
  "typed-arena",
  "unicode_categories",
  "xdg",
@@ -658,12 +658,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
-name = "lazycell"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
-
-[[package]]
 name = "libc"
 version = "0.2.139"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1168,28 +1162,6 @@ dependencies = [
 
 [[package]]
 name = "syntect"
-version = "4.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b20815bbe80ee0be06e6957450a841185fcf690fe0178f14d77a05ce2caa031"
-dependencies = [
- "bincode",
- "bitflags",
- "flate2",
- "fnv",
- "lazy_static",
- "lazycell",
- "onig",
- "plist",
- "regex-syntax",
- "serde",
- "serde_derive",
- "serde_json",
- "walkdir",
- "yaml-rust",
-]
-
-[[package]]
-name = "syntect"
 version = "5.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c6c454c27d9d7d9a84c7803aaa3c50cd088d2906fe3c6e42da3209aa623576a8"
diff --git a/tools/cheddar/Cargo.toml b/tools/cheddar/Cargo.toml
index a1447f7b91..d911b7c446 100644
--- a/tools/cheddar/Cargo.toml
+++ b/tools/cheddar/Cargo.toml
@@ -9,7 +9,7 @@ clap = "2.33"
 comrak = "0.15"
 lazy_static = "1.4"
 rouille = "3.6"
-syntect = "4.5.0"
+syntect = "5.0"
 serde_json = "1.0"
 regex = "1.7"
 
diff --git a/tools/cheddar/src/lib.rs b/tools/cheddar/src/lib.rs
index 851bd743db..c56789eca9 100644
--- a/tools/cheddar/src/lib.rs
+++ b/tools/cheddar/src/lib.rs
@@ -12,7 +12,7 @@ use std::ffi::OsStr;
 use std::io::{BufRead, Write};
 use std::path::Path;
 use std::{env, io};
-use syntect::dumps::from_binary;
+use syntect::dumps::from_uncompressed_data;
 use syntect::easy::HighlightLines;
 use syntect::highlighting::{Theme, ThemeSet};
 use syntect::parsing::{SyntaxReference, SyntaxSet};
@@ -33,7 +33,9 @@ lazy_static! {
     // Note that the syntax set is included from the path pointed to
     // by the BAT_SYNTAXES environment variable at compile time. This
     // variable is populated by Nix and points to TVL's syntax set.
-    static ref SYNTAXES: SyntaxSet = from_binary(include_bytes!(env!("BAT_SYNTAXES")));
+    static ref SYNTAXES: SyntaxSet = from_uncompressed_data(include_bytes!(env!("BAT_SYNTAXES")))
+            .expect("failed to deserialise SyntaxSet");
+
     pub static ref THEMES: ThemeSet = ThemeSet::load_defaults();
 
     // Configure Comrak's Markdown rendering with all the bells &
@@ -153,8 +155,11 @@ fn highlight_code_block(code_block: &NodeCodeBlock) -> NodeValue {
         let mut buf = BLOCK_PRE.to_string();
 
         for line in LinesWithEndings::from(&code) {
-            let regions = hl.highlight(line, &SYNTAXES);
-            append_highlighted_html_for_styled_line(&regions[..], IncludeBackground::No, &mut buf);
+            let regions = hl
+                .highlight_line(line, &SYNTAXES)
+                .expect("highlight_line failed");
+            append_highlighted_html_for_styled_line(&regions[..], IncludeBackground::No, &mut buf)
+                .expect("appending HTML failed");
         }
 
         buf.push_str("</pre>");
@@ -317,13 +322,16 @@ pub fn format_code<R: BufRead, W: Write>(
     // newlines to be efficient, and those are stripped in the lines
     // iterator.
     while should_continue(&read_result) {
-        let regions = hl.highlight(&linebuf, &SYNTAXES);
+        let regions = hl
+            .highlight_line(&linebuf, &SYNTAXES)
+            .expect("highlight_line failed");
 
         append_highlighted_html_for_styled_line(
             &regions[..],
             IncludeBackground::IfDifferent(bg),
             &mut outbuf,
-        );
+        )
+        .expect("appending highlighted HTML failed");
 
         // immediately output the current state to avoid keeping
         // things in memory
diff --git a/tools/cheddar/src/tests.rs b/tools/cheddar/src/tests.rs
index c82bba6767..4d22a1cf05 100644
--- a/tools/cheddar/src/tests.rs
+++ b/tools/cheddar/src/tests.rs
@@ -103,3 +103,8 @@ fn highlights_multiple_shortlinks() {
 fn ignores_invalid_shortlinks() {
     expect_markdown("b/abc is not a real bug", "<p>b/abc is not a real bug</p>");
 }
+
+#[test]
+fn syntax_set_loaded() {
+    assert!(SYNTAXES.syntaxes().len() > 0)
+}