feat(tvix/nar-bridge): support zstd content-encoding r/8961

We previously didn't handle compression in nar-bridge, and left it up to
a fronting reverse proxy.

However, at least nginx with http2 enabled pins each connection to a
single core, causing compression to be limited by the throughput of a
single CPU.

Change-Id: Ia11c2ff5c012192b25eb8ad05dae5542a2d2f777
Reviewed-on: https://cl.tvl.fyi/c/depot/+/12834
Tested-by: BuildkiteCI
Reviewed-by: flokli <flokli@flokli.de>
Autosubmit: flokli <flokli@flokli.de>
Reviewed-by: Ilan Joselevich <personal@ilanjoselevich.com>
Reviewed-by: Jörg Thalheim <joerg@thalheim.io>

4 files changed, 52 insertions, 16 deletions

diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock
index de2c76ee6a02..4b4f45cbe630 100644
--- a/tvix/Cargo.lock
+++ b/tvix/Cargo.lock
@@ -517,9 +517,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.7.2"
+version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3"
+checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da"
 
 [[package]]
 name = "bzip2"
@@ -1655,9 +1655,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 
 [[package]]
 name = "hyper"
-version = "1.4.1"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05"
+checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f"
 dependencies = [
  "bytes",
  "futures-channel",
@@ -4046,9 +4046,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.40.0"
+version = "1.41.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998"
+checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33"
 dependencies = [
  "backtrace",
  "bytes",
@@ -4354,12 +4354,16 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
 dependencies = [
+ "async-compression",
  "bitflags 2.6.0",
  "bytes",
+ "futures-core",
  "http",
  "http-body",
  "http-body-util",
  "pin-project-lite",
+ "tokio",
+ "tokio-util",
  "tower-layer",
  "tower-service",
  "tracing",
diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix
index 103305ced89b..313ec256c491 100644
--- a/tvix/Cargo.nix
+++ b/tvix/Cargo.nix
@@ -1860,9 +1860,9 @@ rec {
       };
       "bytes" = rec {
         crateName = "bytes";
-        version = "1.7.2";
+        version = "1.8.0";
         edition = "2018";
-        sha256 = "1wzs7l57iwqmrszdpr2mmqf1b1hgvpxafc30imxhnry0zfl9m3a2";
+        sha256 = "1nnhpb7jlpj393qnjr1n9n6sgpl3w5ymrwl3pnjmrriam861bh4s";
         authors = [
           "Carl Lerche <me@carllerche.com>"
           "Sean McArthur <sean@seanmonstar.com>"
@@ -5170,9 +5170,9 @@ rec {
       };
       "hyper" = rec {
         crateName = "hyper";
-        version = "1.4.1";
+        version = "1.5.1";
         edition = "2021";
-        sha256 = "01ds8i3q6hw5kw56mavy544m11gkr87zi999siigdl3n1qpd5psh";
+        sha256 = "07s87id0566m2p5dc5q6nqmxz5r8drqd81b7w4q44djgxwkqi0cp";
         authors = [
           "Sean McArthur <sean@seanmonstar.com>"
         ];
@@ -5262,7 +5262,7 @@ rec {
         ];
         features = {
           "client" = [ "dep:want" "dep:pin-project-lite" "dep:smallvec" ];
-          "ffi" = [ "dep:libc" "dep:http-body-util" "futures-util?/alloc" ];
+          "ffi" = [ "dep:http-body-util" "futures-util?/alloc" ];
           "full" = [ "client" "http1" "http2" "server" ];
           "http1" = [ "dep:futures-channel" "dep:futures-util" "dep:httparse" "dep:itoa" ];
           "http2" = [ "dep:futures-channel" "dep:futures-util" "dep:h2" ];
@@ -6916,7 +6916,7 @@ rec {
           {
             name = "tower-http";
             packageId = "tower-http";
-            features = [ "trace" ];
+            features = [ "compression-zstd" "trace" ];
           }
           {
             name = "tower-otel-http-metrics";
@@ -12954,9 +12954,9 @@ rec {
       };
       "tokio" = rec {
         crateName = "tokio";
-        version = "1.40.0";
+        version = "1.41.1";
         edition = "2021";
-        sha256 = "166rllhfkyqp0fs7sxn6crv74iizi4wzd3cvxkcpmlk52qip1c72";
+        sha256 = "0csdvrlpz2b0amrsinkq809nkdkvi6ndc94jr8wjk9d6wyzbbkr2";
         authors = [
           "Tokio Contributors <team@tokio.rs>"
         ];
@@ -14253,6 +14253,12 @@ rec {
         ];
         dependencies = [
           {
+            name = "async-compression";
+            packageId = "async-compression";
+            optional = true;
+            features = [ "tokio" ];
+          }
+          {
             name = "bitflags";
             packageId = "bitflags 2.6.0";
           }
@@ -14261,6 +14267,12 @@ rec {
             packageId = "bytes";
           }
           {
+            name = "futures-core";
+            packageId = "futures-core";
+            optional = true;
+            usesDefaultFeatures = false;
+          }
+          {
             name = "http";
             packageId = "http";
           }
@@ -14277,6 +14289,19 @@ rec {
             packageId = "pin-project-lite";
           }
           {
+            name = "tokio";
+            packageId = "tokio";
+            optional = true;
+            usesDefaultFeatures = false;
+          }
+          {
+            name = "tokio-util";
+            packageId = "tokio-util";
+            optional = true;
+            usesDefaultFeatures = false;
+            features = [ "io" ];
+          }
+          {
             name = "tower-layer";
             packageId = "tower-layer";
           }
@@ -14296,6 +14321,11 @@ rec {
             name = "bytes";
             packageId = "bytes";
           }
+          {
+            name = "tokio";
+            packageId = "tokio";
+            features = [ "full" ];
+          }
         ];
         features = {
           "async-compression" = [ "dep:async-compression" ];
@@ -14334,7 +14364,7 @@ rec {
           "uuid" = [ "dep:uuid" ];
           "validate-request" = [ "mime" ];
         };
-        resolvedDefaultFeatures = [ "default" "trace" "tracing" ];
+        resolvedDefaultFeatures = [ "async-compression" "compression-zstd" "default" "futures-core" "tokio" "tokio-util" "trace" "tracing" ];
       };
       "tower-layer" = rec {
         crateName = "tower-layer";
diff --git a/tvix/nar-bridge/Cargo.toml b/tvix/nar-bridge/Cargo.toml
index e2d50befb68e..e9f8182960af 100644
--- a/tvix/nar-bridge/Cargo.toml
+++ b/tvix/nar-bridge/Cargo.toml
@@ -8,7 +8,7 @@ axum = { workspace = true, features = ["http2"] }
 axum-extra = { workspace = true }
 axum-range = { workspace = true }
 tower = { workspace = true }
-tower-http = { workspace = true, features = ["trace"] }
+tower-http = { workspace = true, features = ["compression-zstd", "trace"] }
 bytes = { workspace = true }
 clap = { workspace = true, features = ["derive", "env"] }
 data-encoding = { workspace = true }
diff --git a/tvix/nar-bridge/src/lib.rs b/tvix/nar-bridge/src/lib.rs
index 0b1e865d5553..68f587d58b18 100644
--- a/tvix/nar-bridge/src/lib.rs
+++ b/tvix/nar-bridge/src/lib.rs
@@ -64,6 +64,8 @@ pub fn gen_router(priority: u64) -> Router<AppState> {
         .route("/:narinfo_str", put(narinfo::put))
         .route("/nix-cache-info", get(move || nix_cache_info(priority)));
 
+    let router = router.layer(tower_http::compression::CompressionLayer::new());
+
     #[cfg(feature = "otlp")]
     return router.layer(metrics_layer);
     #[cfg(not(feature = "otlp"))]