From 0c15a09b1589de8bbd7f843ec213147d5f089d73 Mon Sep 17 00:00:00 2001
From: Adam Joseph <adam@westernsemico.com>
Date: Fri, 8 Dec 2023 20:15:31 -0800
Subject: feat(tvix/cli): add macrobenchmark

This commit adds a simple MVP benchmark, built on our nix
infrastructure instead of cargo.  It simply runs `tvix-eval` inside
of GNU time, and prints the three essential statistics in a short
JSON blob.

You can run the benchmark with a simple `nix run`, like:

  nix run -f . tvix.cli.benchmark-hello
  nix run -f . tvix.cli.benchmark-firefox
  nix run -f . tvix.cli.benchmark-cross-firefox

Currently these blobs are stored only in the CI logs, which I'm sure
get garbage-collected at some point.  We should be putting them in
the git trailers, but that can wait for a future CL.

I tried using `cargo bench` for this but found it incredibly
frustrating.  Maybe I'm doing it wrong.  It seems to be designed for
microbenchmarks only, and very hard to control.  It kept building
all sorts of unnecessary stuff (like the tests), and unlike
crate2nix it was doing all the builds on only a single machine
instead of using more than one machine.  Worse, for that single
machine it kept picking my laptop instead of my fast servers!  It
seems excessively cargo-flavored for such a straightforward task.

Benchmark: {"hello.outPath":{"kbytes":"244736","system":"0.36","user":"2.76"}}
Benchmark: {"firefox.outPath":{"kbytes":"1506736","system":"2.38","user":"32.01"}}
Benchmark: {"pkgsCross.aarch64-multiplatform.firefox.outPath":{"kbytes":"11334548","system":"10.70","user":"107.07"}}

Change-Id: I85bc046ec551360284d7ecfc81a03914f0085909
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10216
Autosubmit: Adam Joseph <adam@westernsemico.com>
Reviewed-by: grfn <grfn@gws.fyi>
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI
---
 tvix/cli/default.nix | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 61 insertions(+), 1 deletion(-)

(limited to 'tvix/cli/default.nix')

diff --git a/tvix/cli/default.nix b/tvix/cli/default.nix
index 0b73fa1aa5..4c4980be76 100644
--- a/tvix/cli/default.nix
+++ b/tvix/cli/default.nix
@@ -18,11 +18,67 @@ let
       echo "Output was correct."
     '';
   };
+
+  benchmark-gnutime-format-string =
+    description:
+    "Benchmark: " +
+    (builtins.toJSON {
+      "${description}" = {
+        kbytes = "%M";
+        system = "%S";
+        user = "%U";
+      };
+    });
+
 in
 
 (depot.tvix.crates.workspaceMembers.tvix-cli.build.override {
   runTests = true;
-}).overrideAttrs (_: {
+}).overrideAttrs (finalAttrs: previousAttrs:
+
+let
+  tvix-cli = finalAttrs.finalPackage;
+
+  # You can run the benchmark with a simple `nix run`, like:
+  #
+  #  nix run -f . tvix.cli.meta.ci.extraSteps.benchmark-nixpkgs-cross-hello-outpath
+  #
+  # TODO(amjoseph): store these results someplace more durable, like git trailers
+  #
+  mkExprBenchmark = { expr, description }:
+    let name = "tvix-cli-benchmark-${description}"; in
+    (pkgs.writeShellScriptBin name ''
+      ${lib.escapeShellArgs [
+        "${pkgs.time}/bin/time"
+        "--format" "${benchmark-gnutime-format-string description}"
+        "${tvix-cli}/bin/tvix"
+        "--no-warnings"
+        "-E" expr
+      ]}
+    '').overrideAttrs (finalAttrs: previousAttrs: {
+      passthru = (previousAttrs.passthru or { }) // {
+        ci = {
+          label = ":nix: benchmark ${description} in tvix";
+          needsOutput = true;
+          command = "${finalAttrs.finalPackage}/bin/${finalAttrs.meta.mainProgram}";
+        };
+      };
+    });
+
+  mkNixpkgsBenchmark = attrpath:
+    mkExprBenchmark {
+      description = builtins.replaceStrings [ ".drv" ] [ "-drv" ] attrpath;
+      expr = "(import ${pkgs.path} {}).${attrpath}";
+    };
+
+  benchmarks = {
+    benchmark-hello = (mkNixpkgsBenchmark "hello.outPath");
+    benchmark-cross-hello = (mkNixpkgsBenchmark "pkgsCross.aarch64-multiplatform.hello.outPath");
+    benchmark-firefox = (mkNixpkgsBenchmark "firefox.outPath");
+    benchmark-cross-firefox = (mkNixpkgsBenchmark "pkgsCross.aarch64-multiplatform.firefox.outPath");
+  };
+in
+{
   meta = {
     ci.extraSteps = {
       eval-nixpkgs-stdenv-drvpath = (mkNixpkgsEvalCheck "stdenv.drvPath" pkgs.stdenv.drvPath);
@@ -31,5 +87,9 @@ in
       eval-nixpkgs-cross-stdenv-outpath = (mkNixpkgsEvalCheck "pkgsCross.aarch64-multiplatform.stdenv.outPath" pkgs.pkgsCross.aarch64-multiplatform.stdenv.outPath);
       eval-nixpkgs-cross-hello-outpath = (mkNixpkgsEvalCheck "pkgsCross.aarch64-multiplatform.hello.outPath" pkgs.pkgsCross.aarch64-multiplatform.hello.outPath);
     };
+    ci.targets = builtins.attrNames benchmarks;
   };
+
+  # Expose benchmarks as standard CI targets.
+  passthru = benchmarks;
 })
-- 
cgit 1.4.1