From 1d678bab5f760b9141feefeacf4a27f72ae1adbd Mon Sep 17 00:00:00 2001
From: Aspen Smith <root@gws.fyi>
Date: Thu, 14 Dec 2023 14:21:39 -0500
Subject: fix(tvix): Fix tvix nixpkgs eval benchmark

Fix the nixpkgs eval hyperfine benchmark in tvix/scripts.bench.sh to:

- Eval both a pinned (never-changing) nixpkgs commit and the nixpkgs
  commit used by the depot
- Not depend on fetchTarball, which is not yet implemented.
- Be called bench-windtunnel.sh

Change-Id: Id2ae18f983ab7327625320b5b16c082ecc369a49
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10364
Reviewed-by: flokli <flokli@flokli.de>
Autosubmit: grfn <grfn@gws.fyi>
Tested-by: BuildkiteCI
---
 tvix/scripts/bench-windtunnel.sh | 27 +++++++++++++++++++++++++++
 tvix/scripts/bench.sh            | 21 ---------------------
 2 files changed, 27 insertions(+), 21 deletions(-)
 create mode 100755 tvix/scripts/bench-windtunnel.sh
 delete mode 100755 tvix/scripts/bench.sh

(limited to 'tvix')

diff --git a/tvix/scripts/bench-windtunnel.sh b/tvix/scripts/bench-windtunnel.sh
new file mode 100755
index 000000000000..894d5ba97e0d
--- /dev/null
+++ b/tvix/scripts/bench-windtunnel.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash ../.. -A tvix.shell
+
+# Benchmark script that runs inside the Windtunnel build agent
+
+set -euo pipefail
+
+echo "Running benchmarks for tvix/eval..."
+pushd "$(dirname "$(dirname "$0")")/eval"
+cargo bench
+windtunnel-cli report -f criterion-rust .
+popd
+
+echo "Running tvix macrobenchmarks..."
+pushd "$(dirname "$(dirname "$0")")"
+
+depot_nixpkgs_path="$(nix eval --raw '("${((import ../third_party/sources {}).nixpkgs)}")')"
+pinned_nixpkgs_path="$(nix eval --raw '(builtins.fetchTarball {url = "https://github.com/NixOS/nixpkgs/archive/91050ea1e57e50388fa87a3302ba12d188ef723a.tar.gz"; sha256 = "1hf6cgaci1n186kkkjq106ryf8mmlq9vnwgfwh625wa8hfgdn4dm";})')"
+
+cargo build --release --bin tvix
+hyperfine --export-json ./results.json \
+    -n 'tvix-eval-depot-nixpkgs-hello' "target/release/tvix -E '(import ${depot_nixpkgs_path} {}).hello.outPath'" \
+    -n 'tvix-eval-depot-nixpkgs-cross-hello' "target/release/tvix -E '(import ${depot_nixpkgs_path} {}).pkgsCross.aarch64-multiplatform.hello.outPath'" \
+    -n 'tvix-eval-pinned-nixpkgs-hello' "target/release/tvix -E '(import ${pinned_nixpkgs_path} {}).hello.outPath'" \
+    -n 'tvix-eval-pinned-nixpkgs-cross-hello' "target/release/tvix -E '(import ${pinned_nixpkgs_path} {}).pkgsCross.aarch64-multiplatform.hello.outPath'" \
+windtunnel-cli report -f hyperfine-json ./results.json
+popd
diff --git a/tvix/scripts/bench.sh b/tvix/scripts/bench.sh
deleted file mode 100755
index 548a9d2058d3..000000000000
--- a/tvix/scripts/bench.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env nix-shell
-#!nix-shell -i bash ../.. -A tvix.shell
-
-# Benchmark script that runs inside the Windtunnel build agent
-
-set -euo pipefail
-
-echo "Running benchmarks for tvix/eval..."
-pushd "$(dirname "$(dirname "$0")")/eval"
-cargo bench
-windtunnel-cli report -f criterion-rust .
-popd
-
-echo "Running tvix macrobenchmarks..."
-pushd "$(dirname "$(dirname "$0")")"
-cargo build --release --bin tvix
-hyperfine --export-json ./results.json \
-    -n 'eval-nixpkgs-hello' "target/release/tvix -E '(import ../../nixpkgs {}).hello.outPath'" \
-    -n 'eval-nixpkgs-cross-hello' "target/release/tvix -E '(import ../../nixpkgs {}).pkgsCross.aarch64-multiplatform.hello.outPath'"
-windtunnel-cli report -f hyperfine-json ./results.json
-popd
-- 
cgit 1.4.1