Add a mechanism for derivation attributes to reference the derivation's outputs

For example, you can now say:

  configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"}";

The strings returned by the ‘placeholder’ builtin are replaced at
build time by the actual store paths corresponding to the specified
outputs.

Previously, you had to work around the inability to self-reference by doing stuff like:

  preConfigure = ''
    configureFlags+=" --prefix $out --includedir=$dev"
  '';

or rely on ad-hoc variable interpolation semantics in Autoconf or Make
(e.g. --prefix=\$(out)), which doesn't always work.

7 files changed, 63 insertions, 16 deletions

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 6a4a7a0356fd..51ee16353ef6 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -673,6 +673,19 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 }
 
 
+/* Return a placeholder string for the specified output that will be
+   substituted by the corresponding output path at build time. For
+   example, ‘placeholder "out"’ returns the string
+   /1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9. At build
+   time, any occurence of this string in an derivation attribute will
+   be replaced with the concrete path in the Nix store of the output
+   ‘out’. */
+static void prim_placeholder(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    mkString(v, hashPlaceholder(state.forceStringNoCtx(*args[0], pos)));
+}
+
+
 /*************************************************************
  * Paths
  *************************************************************/
@@ -1893,6 +1906,7 @@ void EvalState::createBaseEnv()
 
     // Derivations
     addPrimOp("derivationStrict", 1, prim_derivationStrict);
+    addPrimOp("placeholder", 1, prim_placeholder);
 
     // Networking
     addPrimOp("__fetchurl", 1, prim_fetchurl);
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index cfab0b0dc32c..0c687dfc1386 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -652,18 +652,15 @@ HookInstance::~HookInstance()
 //////////////////////////////////////////////////////////////////////
 
 
-typedef map<string, string> HashRewrites;
+typedef map<std::string, std::string> StringRewrites;
 
 
-string rewriteHashes(string s, const HashRewrites & rewrites)
+std::string rewriteStrings(std::string s, const StringRewrites & rewrites)
 {
     for (auto & i : rewrites) {
-        assert(i.first.size() == i.second.size());
         size_t j = 0;
-        while ((j = s.find(i.first, j)) != string::npos) {
-            debug(format("rewriting @ %1%") % j);
-            s.replace(j, i.second.size(), i.second);
-        }
+        while ((j = s.find(i.first, j)) != string::npos)
+            s.replace(j, i.first.size(), i.second);
     }
     return s;
 }
@@ -782,7 +779,7 @@ private:
 #endif
 
     /* Hash rewriting. */
-    HashRewrites rewritesToTmp, rewritesFromTmp;
+    StringRewrites inputRewrites, outputRewrites;
     typedef map<Path, Path> RedirectedOutputs;
     RedirectedOutputs redirectedOutputs;
 
@@ -1774,6 +1771,10 @@ void DerivationGoal::startBuilder()
         for (auto & i : varNames) env[i] = getEnv(i);
     }
 
+    /* Substitute output placeholders with the actual output paths. */
+    for (auto & output : drv->outputs)
+        inputRewrites[hashPlaceholder(output.first)] = output.second.path;
+
     /* The `exportReferencesGraph' feature allows the references graph
        to be passed to a builder.  This attribute should be a list of
        pairs [name1 path1 name2 path2 ...].  The references graph of
@@ -2418,7 +2419,7 @@ void DerivationGoal::runChild()
         /* Fill in the environment. */
         Strings envStrs;
         for (auto & i : env)
-            envStrs.push_back(rewriteHashes(i.first + "=" + i.second, rewritesToTmp));
+            envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites));
 
         /* If we are running in `build-users' mode, then switch to the
            user we allocated above.  Make sure that we drop all root
@@ -2560,7 +2561,7 @@ void DerivationGoal::runChild()
         }
 
         for (auto & i : drv->args)
-            args.push_back(rewriteHashes(i, rewritesToTmp));
+            args.push_back(rewriteStrings(i, inputRewrites));
 
         restoreSIGPIPE();
 
@@ -2682,7 +2683,7 @@ void DerivationGoal::registerOutputs()
 
         /* Apply hash rewriting if necessary. */
         bool rewritten = false;
-        if (!rewritesFromTmp.empty()) {
+        if (!outputRewrites.empty()) {
             printMsg(lvlError, format("warning: rewriting hashes in ‘%1%’; cross fingers") % path);
 
             /* Canonicalise first.  This ensures that the path we're
@@ -2694,7 +2695,7 @@ void DerivationGoal::registerOutputs()
             StringSink sink;
             dumpPath(actualPath, sink);
             deletePath(actualPath);
-            sink.s = make_ref<std::string>(rewriteHashes(*sink.s, rewritesFromTmp));
+            sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
             StringSource source(*sink.s);
             restorePath(actualPath, source);
 
@@ -3033,8 +3034,8 @@ Path DerivationGoal::addHashRewrite(const Path & path)
     Path p = worker.store.storeDir + "/" + h2 + string(path, worker.store.storeDir.size() + 33);
     deletePath(p);
     assert(path.size() == p.size());
-    rewritesToTmp[h1] = h2;
-    rewritesFromTmp[h2] = h1;
+    inputRewrites[h1] = h2;
+    outputRewrites[h2] = h1;
     redirectedOutputs[path] = p;
     return p;
 }
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 7dcf71d468df..f051f10bd018 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -390,4 +390,11 @@ Sink & operator << (Sink & out, const BasicDerivation & drv)
 }
 
 
+std::string hashPlaceholder(const std::string & outputName)
+{
+    // FIXME: memoize?
+    return "/" + printHash32(hashString(htSHA256, "nix-output:" + outputName));
+}
+
+
 }
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 974de78c58d5..9717a81e469c 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -117,4 +117,6 @@ struct Sink;
 Source & readDerivation(Source & in, Store & store, BasicDerivation & drv);
 Sink & operator << (Sink & out, const BasicDerivation & drv);
 
+std::string hashPlaceholder(const std::string & outputName);
+
 }
diff --git a/tests/config.nix b/tests/config.nix
index 6244a15fa48a..76388fdd5b95 100644
--- a/tests/config.nix
+++ b/tests/config.nix
@@ -13,7 +13,7 @@ rec {
     derivation ({
       inherit system;
       builder = shell;
-      args = ["-e" args.builder];
+      args = ["-e" args.builder or (builtins.toFile "builder.sh" "eval \"$buildCommand\"")];
       PATH = path;
     } // removeAttrs args ["builder" "meta"])
     // { meta = args.meta or {}; };
diff --git a/tests/local.mk b/tests/local.mk
index 1e5439f061e2..2ca52144baee 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -10,7 +10,8 @@ nix_tests = \
   timeout.sh secure-drv-outputs.sh nix-channel.sh \
   multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
   binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \
-  check-reqs.sh pass-as-file.sh tarball.sh restricted.sh
+  check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \
+  placeholders.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
diff --git a/tests/placeholders.sh b/tests/placeholders.sh
new file mode 100644
index 000000000000..071cfe2dc893
--- /dev/null
+++ b/tests/placeholders.sh
@@ -0,0 +1,22 @@
+source common.sh
+
+clearStore
+
+nix-build --no-out-link -E '
+  with import ./config.nix;
+
+  mkDerivation {
+    name = "placeholders";
+    outputs = [ "out" "bin" "dev" ];
+    buildCommand = "
+      echo foo1 > $out
+      echo foo2 > $bin
+      echo foo3 > $dev
+      [[ $(cat ${placeholder "out"}) = foo1 ]]
+      [[ $(cat ${placeholder "bin"}) = foo2 ]]
+      [[ $(cat ${placeholder "dev"}) = foo3 ]]
+    ";
+  }
+'
+
+echo XYZZY