From 4402c363b6137112716e82b93e946634aefbb600 Mon Sep 17 00:00:00 2001 From: Profpatsch Date: Sun, 28 Jun 2020 03:14:11 +0200 Subject: feat(nix/runExecline): add runExecline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runExecline is a primitive that just does not care. It’s similar to `runCommand`, but instead of concatenating bash scripts left and right, it actually *uses* the features of `derivation`, passing things to `args` and making it possible to overwrite the `builder` in a sensible manner. Additionally, it provides a way to pass a nix string to `stdin` of the build script. Similar to `writeExecline`, the passed script is not a string, but a nested list of nix lists representing execline blocks. Escaping is done by the implementation, the user can just use normal nix strings. Change-Id: I890d9e5d921207751cdc8cc4309381395d92742f Reviewed-on: https://cl.tvl.fyi/c/depot/+/701 Reviewed-by: BuildkiteCI Reviewed-by: isomer Reviewed-by: tazjin Tested-by: BuildkiteCI --- ci-builds.nix | 4 +- nix/runExecline/OWNERS | 3 + nix/runExecline/default.nix | 19 +++++++ nix/runExecline/runExecline.nix | 121 ++++++++++++++++++++++++++++++++++++++++ nix/runExecline/tests.nix | 108 +++++++++++++++++++++++++++++++++++ 5 files changed, 254 insertions(+), 1 deletion(-) create mode 100644 nix/runExecline/OWNERS create mode 100644 nix/runExecline/default.nix create mode 100644 nix/runExecline/runExecline.nix create mode 100644 nix/runExecline/tests.nix diff --git a/ci-builds.nix b/ci-builds.nix index b36b293266..7af5282f48 100644 --- a/ci-builds.nix +++ b/ci-builds.nix @@ -73,7 +73,9 @@ in lib.fix (self: { tools.cheddar tools.nsfv-setup (drvify "getBins-tests" nix.getBins.tests) - ]; + ] + ++ nix.runExecline.tests + ; # Haskell packages we've patched locally haskellPackages = with depot.third_party.haskellPackages; [ diff --git a/nix/runExecline/OWNERS b/nix/runExecline/OWNERS new file mode 100644 index 0000000000..a742d0d22b --- /dev/null +++ b/nix/runExecline/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - Profpatsch diff --git a/nix/runExecline/default.nix b/nix/runExecline/default.nix new file mode 100644 index 0000000000..0b87b1366d --- /dev/null +++ b/nix/runExecline/default.nix @@ -0,0 +1,19 @@ +{ depot, pkgs, lib, ... }: +let + runExecline = import ./runExecline.nix { + inherit (pkgs) stdenv; + inherit (depot.nix) escapeExecline getBins; + inherit pkgs lib; + }; + + tests = import ./tests.nix { + inherit runExecline; + inherit (depot.nix) getBins; + inherit (pkgs) stdenv coreutils; + inherit pkgs; + }; + +in { + __functor = _: runExecline; + inherit tests; +} diff --git a/nix/runExecline/runExecline.nix b/nix/runExecline/runExecline.nix new file mode 100644 index 0000000000..498e26e576 --- /dev/null +++ b/nix/runExecline/runExecline.nix @@ -0,0 +1,121 @@ +{ pkgs, stdenv, lib, getBins, escapeExecline }: + +# runExecline is a primitive building block +# for writing non-kitchen sink builders. +# +# It’s conceptually similar to `runCommand`, +# but instead of concatenating bash scripts left +# and right, it actually *uses* the features of +# `derivation`, passing things to `args` +# and making it possible to overwrite the `builder` +# in a sensible manner. +# +# Additionally, it provides a way to pass a nix string +# to `stdin` of the build script. +# +# Similar to //nix/writeExecline, the passed script is +# not a string, but a nested list of nix lists +# representing execline blocks. Escaping is +# done by the implementation, the user can just use +# normal nix strings. +# +# Example: +# +# runExecline "my-drv" { stdin = "hi!"; } [ +# "importas" "out" "out" +# # this pipes stdout of s6-cat to $out +# # and s6-cat redirects from stdin to stdout +# "redirfd" "-w" "1" "$out" bins.s6-cat +# ] +# +# which creates a derivation with "hi!" in $out. +# +# See ./tests.nix for more examples. + + +let + bins = getBins pkgs.execline [ + "execlineb" + { use = "if"; as = "execlineIf"; } + "redirfd" + "importas" + "exec" + ] + // getBins pkgs.s6-portable-utils [ + "s6-cat" + "s6-grep" + "s6-touch" + "s6-test" + "s6-chmod" + ]; + +in + +name: +{ +# a string to pass as stdin to the execline script +stdin ? "" +# a program wrapping the acutal execline invocation; +# should be in Bernstein-chaining style +, builderWrapper ? bins.exec +# additional arguments to pass to the derivation +, derivationArgs ? {} +}: +# the execline script as a nested list of string, +# representing the blocks; +# see docs of `escapeExecline`. +execline: + +# those arguments can’t be overwritten +assert !derivationArgs ? system; +assert !derivationArgs ? name; +assert !derivationArgs ? builder; +assert !derivationArgs ? args; + +derivation (derivationArgs // { + # TODO(Profpatsch): what about cross? + inherit (stdenv) system; + inherit name; + + # okay, `builtins.toFile` does not accept strings + # that reference drv outputs. This means we need + # to pass the script and stdin as envvar; + # this might clash with another passed envar, + # so we give it a long & unique name + _runExeclineScript = + let + in escapeExecline execline; + _runExeclineStdin = stdin; + passAsFile = [ + "_runExeclineScript" + "_runExeclineStdin" + ] ++ derivationArgs.passAsFile or []; + + # the default, exec acts as identity executable + builder = builderWrapper; + + args = [ + bins.importas # import script file as $script + "-ui" # drop the envvar afterwards + "script" # substitution name + "_runExeclineScriptPath" # passed script file + + bins.importas # do the same for $stdin + "-ui" + "stdin" + "_runExeclineStdinPath" + + bins.redirfd # now we + "-r" # read the file + "0" # into the stdin of execlineb + "$stdin" # that was given via stdin + + bins.execlineb # the actual invocation + # TODO(Profpatsch): depending on the use-case, -S0 might not be enough + # in all use-cases, then a wrapper for execlineb arguments + # should be added (-P, -S, -s). + "-S0" # set $@ inside the execline script + "-W" # die on syntax error + "$script" # substituted by importas + ]; +}) diff --git a/nix/runExecline/tests.nix b/nix/runExecline/tests.nix new file mode 100644 index 0000000000..cd7f4220b0 --- /dev/null +++ b/nix/runExecline/tests.nix @@ -0,0 +1,108 @@ +{ stdenv, pkgs, runExecline, getBins +# https://www.mail-archive.com/skaware@list.skarnet.org/msg01256.html +, coreutils }: + +let + + bins = getBins coreutils [ "mv" ] + // getBins pkgs.execline [ + "execlineb" + { use = "if"; as = "execlineIf"; } + "redirfd" + "importas" + ] + // getBins pkgs.s6-portable-utils [ + "s6-chmod" + "s6-grep" + "s6-touch" + "s6-cat" + "s6-test" + ]; + + # lol + writeScript = name: script: runExecline name { + derivationArgs = { + inherit script; + passAsFile = [ "script" ]; + preferLocalBuild = true; + allowSubstitutes = false; + }; + } [ + "importas" "-ui" "s" "scriptPath" + "importas" "-ui" "out" "out" + "foreground" [ + bins.mv "$s" "$out" + ] + bins.s6-chmod "0755" "$out" + ]; + + # execline block of depth 1 + block = args: builtins.map (arg: " ${arg}") args ++ [ "" ]; + + # derivation that tests whether a given line exists + # in the given file. Does not use runExecline, because + # that should be tested after all. + fileHasLine = line: file: derivation { + name = "run-execline-test-file-${file.name}-has-line"; + inherit (stdenv) system; + builder = bins.execlineIf; + args = + (block [ + bins.redirfd "-r" "0" file # read file to stdin + bins.s6-grep "-F" "-q" line # and grep for the line + ]) + ++ [ + # if the block succeeded, touch $out + bins.importas "-ui" "out" "out" + bins.s6-touch "$out" + ]; + preferLocalBuild = true; + allowSubstitutes = false; + }; + + # basic test that touches out + basic = runExecline "run-execline-test-basic" { + derivationArgs = { + preferLocalBuild = true; + allowSubstitutes = false; + }; + } [ + "importas" "-ui" "out" "out" + "${bins.s6-touch}" "$out" + ]; + + # whether the stdin argument works as intended + stdin = fileHasLine "foo" (runExecline "run-execline-test-stdin" { + stdin = "foo\nbar\nfoo"; + derivationArgs = { + preferLocalBuild = true; + allowSubstitutes = false; + }; + } [ + "importas" "-ui" "out" "out" + # this pipes stdout of s6-cat to $out + # and s6-cat redirects from stdin to stdout + "redirfd" "-w" "1" "$out" bins.s6-cat + ]); + + wrapWithVar = runExecline "run-execline-test-wrap-with-var" { + builderWrapper = writeScript "var-wrapper" '' + #!${bins.execlineb} -S0 + export myvar myvalue $@ + ''; + derivationArgs = { + preferLocalBuild = true; + allowSubstitutes = false; + }; + } [ + "importas" "-ui" "v" "myvar" + "if" [ bins.s6-test "myvalue" "=" "$v" ] + "importas" "out" "out" + bins.s6-touch "$out" + ]; + +in [ + basic + stdin + wrapWithVar +] -- cgit 1.4.1