feat(nix/runTestsuite): rebuild if test case list changes r/2857

If the result of the assertions changes for a successful test
suite (this happens if tests are reworded, added or removed), this
makes sure the no-op derivation is rebuilt.

This makes sure that test suites show up in buildkite on ocassions other
than channel bumps, since they are only added to the job list if their
`outPath` is missing nowadays (see cl/3427).

Change-Id: Ia1050cca5eeed8b7da84c40f6154b40760a3047f
Reviewed-on: https://cl.tvl.fyi/c/depot/+/3536
Tested-by: BuildkiteCI
Reviewed-by: Profpatsch <mail@profpatsch.de>

1 files changed, 5 insertions, 2 deletions

diff --git a/nix/runTestsuite/default.nix b/nix/runTestsuite/default.nix
index 78cf5389e4..dc8e40f8ad 100644
--- a/nix/runTestsuite/default.nix
+++ b/nix/runTestsuite/default.nix
@@ -38,7 +38,7 @@ let
     ;
 
   bins = depot.nix.getBins pkgs.coreutils [ "printf" ]
-      // depot.nix.getBins pkgs.s6-portable-utils [ "s6-touch" ];
+      // depot.nix.getBins pkgs.s6-portable-utils [ "s6-touch" "s6-false" ];
 
   # Returns true if the given expression throws when `deepSeq`-ed
   throws = expr:
@@ -157,10 +157,13 @@ let
         };
         goodIts = partitionTests (it: (goodIt it).asserts.err == []);
         res = goodIts itResults;
+        prettyRes = lib.generators.toPretty {} res;
       in
         if res.err == []
         then depot.nix.runExecline.local "testsuite-${name}-successful" {} [
           "importas" "out" "out"
+          # force derivation to rebuild if test case list changes
+          "ifelse" [ bins.s6-false ] [ bins.printf "" prettyRes ]
           "if" [ bins.printf "%s\n" "testsuite ${name} successful!" ]
           bins.s6-touch "$out"
         ]
@@ -169,7 +172,7 @@ let
           "if" [
             bins.printf "%s\n%s\n"
               "testsuite ${name} failed!"
-              (lib.generators.toPretty {} res)
+              prettyRes
           ]
           "exit" "1"
         ]);