about summary refs log tree commit diff
diff options
context:
space:
mode:
authorAdam Joseph <adam@westernsemico.com>2023-12-09T04·15-0800
committerclbot <clbot@tvl.fyi>2023-12-12T11·56+0000
commit0c15a09b1589de8bbd7f843ec213147d5f089d73 (patch)
treeb46cfb51127a4afe8312c25d163eae0e3973ffcd
parent02c66218138f6cf8eeb1437687c5cfddbe27fdd9 (diff)
feat(tvix/cli): add macrobenchmark r/7163
This commit adds a simple MVP benchmark, built on our nix
infrastructure instead of cargo.  It simply runs `tvix-eval` inside
of GNU time, and prints the three essential statistics in a short
JSON blob.

You can run the benchmark with a simple `nix run`, like:

  nix run -f . tvix.cli.benchmark-hello
  nix run -f . tvix.cli.benchmark-firefox
  nix run -f . tvix.cli.benchmark-cross-firefox

Currently these blobs are stored only in the CI logs, which I'm sure
get garbage-collected at some point.  We should be putting them in
the git trailers, but that can wait for a future CL.

I tried using `cargo bench` for this but found it incredibly
frustrating.  Maybe I'm doing it wrong.  It seems to be designed for
microbenchmarks only, and very hard to control.  It kept building
all sorts of unnecessary stuff (like the tests), and unlike
crate2nix it was doing all the builds on only a single machine
instead of using more than one machine.  Worse, for that single
machine it kept picking my laptop instead of my fast servers!  It
seems excessively cargo-flavored for such a straightforward task.

Benchmark: {"hello.outPath":{"kbytes":"244736","system":"0.36","user":"2.76"}}
Benchmark: {"firefox.outPath":{"kbytes":"1506736","system":"2.38","user":"32.01"}}
Benchmark: {"pkgsCross.aarch64-multiplatform.firefox.outPath":{"kbytes":"11334548","system":"10.70","user":"107.07"}}

Change-Id: I85bc046ec551360284d7ecfc81a03914f0085909
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10216
Autosubmit: Adam Joseph <adam@westernsemico.com>
Reviewed-by: grfn <grfn@gws.fyi>
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI
-rw-r--r--tvix/cli/default.nix62
1 files changed, 61 insertions, 1 deletions
diff --git a/tvix/cli/default.nix b/tvix/cli/default.nix
index 0b73fa1aa59e..4c4980be76a7 100644
--- a/tvix/cli/default.nix
+++ b/tvix/cli/default.nix
@@ -18,11 +18,67 @@ let
       echo "Output was correct."
     '';
   };
+
+  benchmark-gnutime-format-string =
+    description:
+    "Benchmark: " +
+    (builtins.toJSON {
+      "${description}" = {
+        kbytes = "%M";
+        system = "%S";
+        user = "%U";
+      };
+    });
+
 in
 
 (depot.tvix.crates.workspaceMembers.tvix-cli.build.override {
   runTests = true;
-}).overrideAttrs (_: {
+}).overrideAttrs (finalAttrs: previousAttrs:
+
+let
+  tvix-cli = finalAttrs.finalPackage;
+
+  # You can run the benchmark with a simple `nix run`, like:
+  #
+  #  nix run -f . tvix.cli.meta.ci.extraSteps.benchmark-nixpkgs-cross-hello-outpath
+  #
+  # TODO(amjoseph): store these results someplace more durable, like git trailers
+  #
+  mkExprBenchmark = { expr, description }:
+    let name = "tvix-cli-benchmark-${description}"; in
+    (pkgs.writeShellScriptBin name ''
+      ${lib.escapeShellArgs [
+        "${pkgs.time}/bin/time"
+        "--format" "${benchmark-gnutime-format-string description}"
+        "${tvix-cli}/bin/tvix"
+        "--no-warnings"
+        "-E" expr
+      ]}
+    '').overrideAttrs (finalAttrs: previousAttrs: {
+      passthru = (previousAttrs.passthru or { }) // {
+        ci = {
+          label = ":nix: benchmark ${description} in tvix";
+          needsOutput = true;
+          command = "${finalAttrs.finalPackage}/bin/${finalAttrs.meta.mainProgram}";
+        };
+      };
+    });
+
+  mkNixpkgsBenchmark = attrpath:
+    mkExprBenchmark {
+      description = builtins.replaceStrings [ ".drv" ] [ "-drv" ] attrpath;
+      expr = "(import ${pkgs.path} {}).${attrpath}";
+    };
+
+  benchmarks = {
+    benchmark-hello = (mkNixpkgsBenchmark "hello.outPath");
+    benchmark-cross-hello = (mkNixpkgsBenchmark "pkgsCross.aarch64-multiplatform.hello.outPath");
+    benchmark-firefox = (mkNixpkgsBenchmark "firefox.outPath");
+    benchmark-cross-firefox = (mkNixpkgsBenchmark "pkgsCross.aarch64-multiplatform.firefox.outPath");
+  };
+in
+{
   meta = {
     ci.extraSteps = {
       eval-nixpkgs-stdenv-drvpath = (mkNixpkgsEvalCheck "stdenv.drvPath" pkgs.stdenv.drvPath);
@@ -31,5 +87,9 @@ in
       eval-nixpkgs-cross-stdenv-outpath = (mkNixpkgsEvalCheck "pkgsCross.aarch64-multiplatform.stdenv.outPath" pkgs.pkgsCross.aarch64-multiplatform.stdenv.outPath);
       eval-nixpkgs-cross-hello-outpath = (mkNixpkgsEvalCheck "pkgsCross.aarch64-multiplatform.hello.outPath" pkgs.pkgsCross.aarch64-multiplatform.hello.outPath);
     };
+    ci.targets = builtins.attrNames benchmarks;
   };
+
+  # Expose benchmarks as standard CI targets.
+  passthru = benchmarks;
 })