feat(nix/runExecline): add runExecline r/1132

runExecline is a primitive that just does not care.

It’s similar to `runCommand`, but instead of concatenating bash
scripts left and right, it actually *uses* the features of
`derivation`, passing things to `args` and making it possible to
overwrite the `builder` in a sensible manner.

Additionally, it provides a way to pass a nix string to `stdin` of the
build script.

Similar to `writeExecline`, the passed script is not a string, but a
nested list of nix lists representing execline blocks. Escaping is
done by the implementation, the user can just use normal nix strings.

Change-Id: I890d9e5d921207751cdc8cc4309381395d92742f
Reviewed-on: https://cl.tvl.fyi/c/depot/+/701
Reviewed-by: BuildkiteCI
Reviewed-by: isomer <isomer@tvl.fyi>
Reviewed-by: tazjin <mail@tazj.in>
Tested-by: BuildkiteCI

5 files changed, 254 insertions, 1 deletions

diff --git a/ci-builds.nix b/ci-builds.nix
index b36b293266..7af5282f48 100644
--- a/ci-builds.nix
+++ b/ci-builds.nix
@@ -73,7 +73,9 @@ in lib.fix (self: {
     tools.cheddar
     tools.nsfv-setup
     (drvify "getBins-tests" nix.getBins.tests)
-  ];
+  ]
+  ++ nix.runExecline.tests
+  ;
 
   # Haskell packages we've patched locally
   haskellPackages = with depot.third_party.haskellPackages; [
diff --git a/nix/runExecline/OWNERS b/nix/runExecline/OWNERS
new file mode 100644
index 0000000000..a742d0d22b
--- /dev/null
+++ b/nix/runExecline/OWNERS
@@ -0,0 +1,3 @@
+inherited: true
+owners:
+  - Profpatsch
diff --git a/nix/runExecline/default.nix b/nix/runExecline/default.nix
new file mode 100644
index 0000000000..0b87b1366d
--- /dev/null
+++ b/nix/runExecline/default.nix
@@ -0,0 +1,19 @@
+{ depot, pkgs, lib, ... }:
+let
+  runExecline = import ./runExecline.nix {
+    inherit (pkgs) stdenv;
+    inherit (depot.nix) escapeExecline getBins;
+    inherit pkgs lib;
+  };
+
+  tests = import ./tests.nix {
+    inherit runExecline;
+    inherit (depot.nix) getBins;
+    inherit (pkgs) stdenv coreutils;
+    inherit pkgs;
+  };
+
+in {
+  __functor = _: runExecline;
+  inherit tests;
+}
diff --git a/nix/runExecline/runExecline.nix b/nix/runExecline/runExecline.nix
new file mode 100644
index 0000000000..498e26e576
--- /dev/null
+++ b/nix/runExecline/runExecline.nix
@@ -0,0 +1,121 @@
+{ pkgs, stdenv, lib, getBins, escapeExecline }:
+
+# runExecline is a primitive building block
+# for writing non-kitchen sink builders.
+#
+# It’s conceptually similar to `runCommand`,
+# but instead of concatenating bash scripts left
+# and right, it actually *uses* the features of
+# `derivation`, passing things to `args`
+# and making it possible to overwrite the `builder`
+# in a sensible manner.
+#
+# Additionally, it provides a way to pass a nix string
+# to `stdin` of the build script.
+#
+# Similar to //nix/writeExecline, the passed script is
+# not a string, but a nested list of nix lists
+# representing execline blocks. Escaping is
+# done by the implementation, the user can just use
+# normal nix strings.
+#
+# Example:
+#
+#  runExecline "my-drv" { stdin = "hi!"; } [
+#    "importas" "out" "out"
+#    # this pipes stdout of s6-cat to $out
+#    # and s6-cat redirects from stdin to stdout
+#    "redirfd" "-w" "1" "$out" bins.s6-cat
+#  ]
+#
+# which creates a derivation with "hi!" in $out.
+#
+# See ./tests.nix for more examples.
+
+
+let
+  bins = getBins pkgs.execline [
+           "execlineb"
+           { use = "if"; as = "execlineIf"; }
+           "redirfd"
+           "importas"
+           "exec"
+         ]
+      // getBins pkgs.s6-portable-utils [
+           "s6-cat"
+           "s6-grep"
+           "s6-touch"
+           "s6-test"
+           "s6-chmod"
+         ];
+
+in
+
+name:
+{
+# a string to pass as stdin to the execline script
+stdin ? ""
+# a program wrapping the acutal execline invocation;
+# should be in Bernstein-chaining style
+, builderWrapper ? bins.exec
+# additional arguments to pass to the derivation
+, derivationArgs ? {}
+}:
+# the execline script as a nested list of string,
+# representing the blocks;
+# see docs of `escapeExecline`.
+execline:
+
+# those arguments can’t be overwritten
+assert !derivationArgs ? system;
+assert !derivationArgs ? name;
+assert !derivationArgs ? builder;
+assert !derivationArgs ? args;
+
+derivation (derivationArgs // {
+  # TODO(Profpatsch): what about cross?
+  inherit (stdenv) system;
+  inherit name;
+
+  # okay, `builtins.toFile` does not accept strings
+  # that reference drv outputs. This means we need
+  # to pass the script and stdin as envvar;
+  # this might clash with another passed envar,
+  # so we give it a long & unique name
+  _runExeclineScript =
+    let
+    in escapeExecline execline;
+  _runExeclineStdin = stdin;
+  passAsFile = [
+    "_runExeclineScript"
+    "_runExeclineStdin"
+  ] ++ derivationArgs.passAsFile or [];
+
+  # the default, exec acts as identity executable
+  builder = builderWrapper;
+
+  args = [
+    bins.importas            # import script file as $script
+    "-ui"                    # drop the envvar afterwards
+    "script"                 # substitution name
+    "_runExeclineScriptPath" # passed script file
+
+    bins.importas            # do the same for $stdin
+    "-ui"
+    "stdin"
+    "_runExeclineStdinPath"
+
+    bins.redirfd             # now we
+    "-r"                     # read the file
+    "0"                      # into the stdin of execlineb
+    "$stdin"                 # that was given via stdin
+
+    bins.execlineb           # the actual invocation
+    # TODO(Profpatsch): depending on the use-case, -S0 might not be enough
+    # in all use-cases, then a wrapper for execlineb arguments
+    # should be added (-P, -S, -s).
+    "-S0"                    # set $@ inside the execline script
+    "-W"                     # die on syntax error
+    "$script"                # substituted by importas
+  ];
+})
diff --git a/nix/runExecline/tests.nix b/nix/runExecline/tests.nix
new file mode 100644
index 0000000000..cd7f4220b0
--- /dev/null
+++ b/nix/runExecline/tests.nix
@@ -0,0 +1,108 @@
+{ stdenv, pkgs, runExecline, getBins
+# https://www.mail-archive.com/skaware@list.skarnet.org/msg01256.html
+, coreutils }:
+
+let
+
+  bins = getBins coreutils [ "mv" ]
+      // getBins pkgs.execline [
+           "execlineb"
+           { use = "if"; as = "execlineIf"; }
+           "redirfd"
+           "importas"
+         ]
+      // getBins pkgs.s6-portable-utils [
+           "s6-chmod"
+           "s6-grep"
+           "s6-touch"
+           "s6-cat"
+           "s6-test"
+         ];
+
+  # lol
+  writeScript = name: script: runExecline name {
+    derivationArgs = {
+      inherit script;
+      passAsFile = [ "script" ];
+      preferLocalBuild = true;
+      allowSubstitutes = false;
+    };
+  } [
+      "importas" "-ui" "s" "scriptPath"
+      "importas" "-ui" "out" "out"
+      "foreground" [
+        bins.mv "$s" "$out"
+      ]
+      bins.s6-chmod "0755" "$out"
+  ];
+
+  # execline block of depth 1
+  block = args: builtins.map (arg: " ${arg}") args ++ [ "" ];
+
+  # derivation that tests whether a given line exists
+  # in the given file. Does not use runExecline, because
+  # that should be tested after all.
+  fileHasLine = line: file: derivation {
+    name = "run-execline-test-file-${file.name}-has-line";
+    inherit (stdenv) system;
+    builder = bins.execlineIf;
+    args =
+      (block [
+        bins.redirfd "-r" "0" file   # read file to stdin
+        bins.s6-grep "-F" "-q" line   # and grep for the line
+      ])
+      ++ [
+        # if the block succeeded, touch $out
+        bins.importas "-ui" "out" "out"
+        bins.s6-touch "$out"
+      ];
+    preferLocalBuild = true;
+    allowSubstitutes = false;
+  };
+
+  # basic test that touches out
+  basic = runExecline "run-execline-test-basic" {
+    derivationArgs = {
+      preferLocalBuild = true;
+      allowSubstitutes = false;
+    };
+  } [
+      "importas" "-ui" "out" "out"
+      "${bins.s6-touch}" "$out"
+  ];
+
+  # whether the stdin argument works as intended
+  stdin = fileHasLine "foo" (runExecline "run-execline-test-stdin" {
+    stdin = "foo\nbar\nfoo";
+    derivationArgs = {
+      preferLocalBuild = true;
+      allowSubstitutes = false;
+    };
+  } [
+      "importas" "-ui" "out" "out"
+      # this pipes stdout of s6-cat to $out
+      # and s6-cat redirects from stdin to stdout
+      "redirfd" "-w" "1" "$out" bins.s6-cat
+  ]);
+
+  wrapWithVar = runExecline "run-execline-test-wrap-with-var" {
+    builderWrapper = writeScript "var-wrapper" ''
+      #!${bins.execlineb} -S0
+      export myvar myvalue $@
+    '';
+    derivationArgs = {
+      preferLocalBuild = true;
+      allowSubstitutes = false;
+    };
+  } [
+    "importas" "-ui" "v" "myvar"
+    "if" [ bins.s6-test "myvalue" "=" "$v" ]
+      "importas" "out" "out"
+      bins.s6-touch "$out"
+  ];
+
+in [
+  basic
+  stdin
+  wrapWithVar
+]