Add a post-build-hook

Passing `--post-build-hook /foo/bar` to a nix-* command will cause
`/foo/bar` to be executed after each build with the following
environment variables set:

    DRV_PATH=/nix/store/drv-that-has-been-built.drv
    OUT_PATHS=/nix/store/...build /nix/store/...build-bin /nix/store/...build-dev

This can be useful in particular to upload all the builded artifacts to
the cache (including the ones that don't appear in the runtime closure
of the final derivation or are built because of IFD).

This new feature prints the stderr/stdout output to the `nix-build`
and `nix build` client, and the output is printed in a Nix 2
compatible format:

    [nix]$ ./inst/bin/nix-build ./test.nix
    these derivations will be built:
      /nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv
    building '/nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv'...
    hello!
    bye!
    running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'...
    post-build-hook: + sleep 1
    post-build-hook: + echo 'Signing paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation
    post-build-hook: Signing paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation
    post-build-hook: + sleep 1
    post-build-hook: + echo 'Uploading paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation
    post-build-hook: Uploading paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation
    post-build-hook: + sleep 1
    post-build-hook: + printf 'very important stuff'
    /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation

    [nix-shell:~/projects/github.com/NixOS/nix]$ ./inst/bin/nix build -L -f ./test.nix
    my-example-derivation> hello!
    my-example-derivation> bye!
    my-example-derivation (post)> + sleep 1
    my-example-derivation (post)> + echo 'Signing paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation
    my-example-derivation (post)> Signing paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation
    my-example-derivation (post)> + sleep 1
    my-example-derivation (post)> + echo 'Uploading paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation
    my-example-derivation (post)> Uploading paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation
    my-example-derivation (post)> + sleep 1
    my-example-derivation (post)> + printf 'very important stuff'
    [1 built, 0.0 MiB DL]

Co-authored-by: Graham Christensen <graham@grahamc.com>
Co-authored-by: Eelco Dolstra <edolstra@gmail.com>

6 files changed, 98 insertions, 3 deletions

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index cf6428e12467..7494eec419e0 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1629,6 +1629,61 @@ void DerivationGoal::buildDone()
            being valid. */
         registerOutputs();
 
+        if (settings.postBuildHook != "") {
+            Activity act(*logger, lvlInfo, actPostBuildHook,
+                fmt("running post-build-hook '%s'", settings.postBuildHook),
+                Logger::Fields{drvPath});
+            PushActivity pact(act.id);
+            auto outputPaths = drv->outputPaths();
+            std::map<std::string, std::string> hookEnvironment = getEnv();
+
+            hookEnvironment.emplace("DRV_PATH", drvPath);
+            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", outputPaths)));
+
+            RunOptions opts(settings.postBuildHook, {});
+            opts.environment = hookEnvironment;
+
+            struct LogSink : Sink {
+                Activity & act;
+                std::string currentLine;
+
+                LogSink(Activity & act) : act(act) { }
+
+                void operator() (const unsigned char * data, size_t len) override {
+                    for (size_t i = 0; i < len; i++) {
+                        auto c = data[i];
+
+                        if (c == '\n') {
+                            flushLine();
+                        } else {
+                            currentLine += c;
+                        }
+                    }
+                }
+
+                void flushLine() {
+                    if (settings.verboseBuild) {
+                        printError("post-build-hook: " + currentLine);
+                    } else {
+                        act.result(resPostBuildLogLine, currentLine);
+                    }
+                    currentLine.clear();
+                }
+
+                ~LogSink() {
+                    if (currentLine != "") {
+                        currentLine += '\n';
+                        flushLine();
+                    }
+                }
+            };
+            LogSink sink(act);
+
+            opts.standardOut = &sink;
+            opts.mergeStderrToStdout = true;
+            runProgram2(opts);
+        }
+
         if (buildMode == bmCheck) {
             done(BuildResult::Built);
             return;
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 0af8215d1fd8..7dea6892143e 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -315,6 +315,9 @@ public:
         "pre-build-hook",
         "A program to run just before a build to set derivation-specific build settings."};
 
+    Setting<std::string> postBuildHook{this, "", "post-build-hook",
+        "A program to run just after each succesful build."};
+
     Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file",
         "Path to the netrc file used to obtain usernames/passwords for downloads."};
 
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 5f221944594b..5df03da74e00 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -26,6 +26,7 @@ typedef enum {
     actVerifyPaths = 107,
     actSubstitute = 108,
     actQueryPathInfo = 109,
+    actPostBuildHook = 110,
 } ActivityType;
 
 typedef enum {
@@ -36,6 +37,7 @@ typedef enum {
     resSetPhase = 104,
     resProgress = 105,
     resSetExpected = 106,
+    resPostBuildLogLine = 107,
 } ResultType;
 
 typedef uint64_t ActivityId;
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 17aee2d5c3d0..44fa72482552 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -84,6 +84,15 @@ void clearEnv()
         unsetenv(name.first.c_str());
 }
 
+void replaceEnv(std::map<std::string, std::string> newEnv)
+{
+    clearEnv();
+    for (auto newEnvVar : newEnv)
+    {
+        setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1);
+    }
+}
+
 
 Path absPath(Path path, Path dir)
 {
@@ -1019,10 +1028,22 @@ void runProgram2(const RunOptions & options)
     if (options.standardOut) out.create();
     if (source) in.create();
 
+    ProcessOptions processOptions;
+    // vfork implies that the environment of the main process and the fork will
+    // be shared (technically this is undefined, but in practice that's the
+    // case), so we can't use it if we alter the environment
+    if (options.environment)
+        processOptions.allowVfork = false;
+
     /* Fork. */
     Pid pid = startProcess([&]() {
+        if (options.environment)
+            replaceEnv(*options.environment);
         if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
             throw SysError("dupping stdout");
+        if (options.mergeStderrToStdout)
+            if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1)
+                throw SysError("cannot dup stdout into stderr");
         if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1)
             throw SysError("dupping stdin");
 
@@ -1047,7 +1068,7 @@ void runProgram2(const RunOptions & options)
             execv(options.program.c_str(), stringsToCharPtrs(args_).data());
 
         throw SysError("executing '%1%'", options.program);
-    });
+    }, processOptions);
 
     out.writeSide = -1;
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index fce3cab8def5..b538a0b41ce8 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -270,12 +270,14 @@ struct RunOptions
     std::optional<uid_t> uid;
     std::optional<uid_t> gid;
     std::optional<Path> chdir;
+    std::optional<std::map<std::string, std::string>> environment;
     Path program;
     bool searchPath = true;
     Strings args;
     std::optional<std::string> input;
     Source * standardIn = nullptr;
     Sink * standardOut = nullptr;
+    bool mergeStderrToStdout = false;
     bool _killStderr = false;
 
     RunOptions(const Path & program, const Strings & args)
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index b1c1d87de1a2..c0bcfb0c91bc 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -170,6 +170,14 @@ public:
                 name, sub);
         }
 
+        if (type == actPostBuildHook) {
+            auto name = storePathToName(getS(fields, 0));
+            if (hasSuffix(name, ".drv"))
+                name.resize(name.size() - 4);
+            i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name);
+            i->name = DrvName(name).name;
+        }
+
         if (type == actQueryPathInfo) {
             auto name = storePathToName(getS(fields, 0));
             i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1));
@@ -228,14 +236,18 @@ public:
             update(*state);
         }
 
-        else if (type == resBuildLogLine) {
+        else if (type == resBuildLogLine || type == resPostBuildLogLine) {
             auto lastLine = trim(getS(fields, 0));
             if (!lastLine.empty()) {
                 auto i = state->its.find(act);
                 assert(i != state->its.end());
                 ActInfo info = *i->second;
                 if (printBuildLogs) {
-                    log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + "> " + ANSI_NORMAL + lastLine);
+                    auto suffix = "> ";
+                    if (type == resPostBuildLogLine) {
+                        suffix = " (post)> ";
+                    }
+                    log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine);
                 } else {
                     state->activities.erase(i->second);
                     info.lastLine = lastLine;