chore(tools/cheddar): bump to syntect 5.0 r/5622

Upgrade to syntect 5.0 and load the new kind of syntax set
serialisation with the new helper function for that purpose.

Includes other minor API fixes as well, note that the things that are
now calling `expect` previously failed internally at those points and
we're reasonably confident they don't fail in production.

This has been waiting for a long time ...

Change-Id: I8af4fef995ff64bfbe24e1f13917fa50ecb6e4ad
Reviewed-on: https://cl.tvl.fyi/c/depot/+/7787
Reviewed-by: sterni <sternenseemann@systemli.org>
Tested-by: BuildkiteCI

5 files changed, 22 insertions, 41 deletions

diff --git a/third_party/nixpkgs/default.nix b/third_party/nixpkgs/default.nix
index 051a961c68aa..02d5e84ff507 100644
--- a/third_party/nixpkgs/default.nix
+++ b/third_party/nixpkgs/default.nix
@@ -54,10 +54,6 @@ let
   # Use `stableNixpkgs` from above.
   stableOverlay = _unstableSelf: _unstableSuper: {
     inherit (stableNixpkgs)
-      # bat syntaxes changed with syntect 5.0, but cheddar is still on 4.x
-      # TODO(tazjin): upgrade cheddar to syntect 5.0
-      bat
-
       # ntfy does not build on unstable as of 2022-08-02
       ntfy
 
diff --git a/tools/cheddar/Cargo.lock b/tools/cheddar/Cargo.lock
index 71eb0a293e87..2787d79cb782 100644
--- a/tools/cheddar/Cargo.lock
+++ b/tools/cheddar/Cargo.lock
@@ -184,7 +184,7 @@ dependencies = [
  "rouille",
  "serde",
  "serde_json",
- "syntect 4.6.0",
+ "syntect",
 ]
 
 [[package]]
@@ -282,7 +282,7 @@ dependencies = [
  "pest_derive",
  "regex",
  "shell-words",
- "syntect 5.0.0",
+ "syntect",
  "typed-arena",
  "unicode_categories",
  "xdg",
@@ -658,12 +658,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
-name = "lazycell"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
-
-[[package]]
 name = "libc"
 version = "0.2.139"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1168,28 +1162,6 @@ dependencies = [
 
 [[package]]
 name = "syntect"
-version = "4.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b20815bbe80ee0be06e6957450a841185fcf690fe0178f14d77a05ce2caa031"
-dependencies = [
- "bincode",
- "bitflags",
- "flate2",
- "fnv",
- "lazy_static",
- "lazycell",
- "onig",
- "plist",
- "regex-syntax",
- "serde",
- "serde_derive",
- "serde_json",
- "walkdir",
- "yaml-rust",
-]
-
-[[package]]
-name = "syntect"
 version = "5.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c6c454c27d9d7d9a84c7803aaa3c50cd088d2906fe3c6e42da3209aa623576a8"
diff --git a/tools/cheddar/Cargo.toml b/tools/cheddar/Cargo.toml
index a1447f7b91a8..d911b7c44682 100644
--- a/tools/cheddar/Cargo.toml
+++ b/tools/cheddar/Cargo.toml
@@ -9,7 +9,7 @@ clap = "2.33"
 comrak = "0.15"
 lazy_static = "1.4"
 rouille = "3.6"
-syntect = "4.5.0"
+syntect = "5.0"
 serde_json = "1.0"
 regex = "1.7"
 
diff --git a/tools/cheddar/src/lib.rs b/tools/cheddar/src/lib.rs
index 851bd743db2e..c56789eca914 100644
--- a/tools/cheddar/src/lib.rs
+++ b/tools/cheddar/src/lib.rs
@@ -12,7 +12,7 @@ use std::ffi::OsStr;
 use std::io::{BufRead, Write};
 use std::path::Path;
 use std::{env, io};
-use syntect::dumps::from_binary;
+use syntect::dumps::from_uncompressed_data;
 use syntect::easy::HighlightLines;
 use syntect::highlighting::{Theme, ThemeSet};
 use syntect::parsing::{SyntaxReference, SyntaxSet};
@@ -33,7 +33,9 @@ lazy_static! {
     // Note that the syntax set is included from the path pointed to
     // by the BAT_SYNTAXES environment variable at compile time. This
     // variable is populated by Nix and points to TVL's syntax set.
-    static ref SYNTAXES: SyntaxSet = from_binary(include_bytes!(env!("BAT_SYNTAXES")));
+    static ref SYNTAXES: SyntaxSet = from_uncompressed_data(include_bytes!(env!("BAT_SYNTAXES")))
+            .expect("failed to deserialise SyntaxSet");
+
     pub static ref THEMES: ThemeSet = ThemeSet::load_defaults();
 
     // Configure Comrak's Markdown rendering with all the bells &
@@ -153,8 +155,11 @@ fn highlight_code_block(code_block: &NodeCodeBlock) -> NodeValue {
         let mut buf = BLOCK_PRE.to_string();
 
         for line in LinesWithEndings::from(&code) {
-            let regions = hl.highlight(line, &SYNTAXES);
-            append_highlighted_html_for_styled_line(&regions[..], IncludeBackground::No, &mut buf);
+            let regions = hl
+                .highlight_line(line, &SYNTAXES)
+                .expect("highlight_line failed");
+            append_highlighted_html_for_styled_line(&regions[..], IncludeBackground::No, &mut buf)
+                .expect("appending HTML failed");
         }
 
         buf.push_str("</pre>");
@@ -317,13 +322,16 @@ pub fn format_code<R: BufRead, W: Write>(
     // newlines to be efficient, and those are stripped in the lines
     // iterator.
     while should_continue(&read_result) {
-        let regions = hl.highlight(&linebuf, &SYNTAXES);
+        let regions = hl
+            .highlight_line(&linebuf, &SYNTAXES)
+            .expect("highlight_line failed");
 
         append_highlighted_html_for_styled_line(
             &regions[..],
             IncludeBackground::IfDifferent(bg),
             &mut outbuf,
-        );
+        )
+        .expect("appending highlighted HTML failed");
 
         // immediately output the current state to avoid keeping
         // things in memory
diff --git a/tools/cheddar/src/tests.rs b/tools/cheddar/src/tests.rs
index c82bba676746..4d22a1cf05e0 100644
--- a/tools/cheddar/src/tests.rs
+++ b/tools/cheddar/src/tests.rs
@@ -103,3 +103,8 @@ fn highlights_multiple_shortlinks() {
 fn ignores_invalid_shortlinks() {
     expect_markdown("b/abc is not a real bug", "<p>b/abc is not a real bug</p>");
 }
+
+#[test]
+fn syntax_set_loaded() {
+    assert!(SYNTAXES.syntaxes().len() > 0)
+}