Add support for passing structured data to builders

Previously, all derivation attributes had to be coerced into strings
so that they could be passed via the environment. This is lossy
(e.g. lists get flattened, necessitating configureFlags
vs. configureFlagsArray, of which the latter cannot be specified as an
attribute), doesn't support attribute sets at all, and has size
limitations (necessitating hacks like passAsFile).

This patch adds a new mode for passing attributes to builders, namely
encoded as a JSON file ".attrs.json" in the current directory of the
builder. This mode is activated via the special attribute

  __structuredAttrs = true;

(The idea is that one day we can set this in stdenv.mkDerivation.)

For example,

  stdenv.mkDerivation {
    __structuredAttrs = true;
    name = "foo";
    buildInputs = [ pkgs.hello pkgs.cowsay ];
    doCheck = true;
    hardening.format = false;
  }

results in a ".attrs.json" file containing (sans the indentation):

  {
    "buildInputs": [],
    "builder": "/nix/store/ygl61ycpr2vjqrx775l1r2mw1g2rb754-bash-4.3-p48/bin/bash",
    "configureFlags": [
      "--with-foo",
      "--with-bar=1 2"
    ],
    "doCheck": true,
    "hardening": {
      "format": false
    },
    "name": "foo",
    "nativeBuildInputs": [
      "/nix/store/10h6li26i7g6z3mdpvra09yyf10mmzdr-hello-2.10",
      "/nix/store/4jnvjin0r6wp6cv1hdm5jbkx3vinlcvk-cowsay-3.03"
    ],
    "propagatedBuildInputs": [],
    "propagatedNativeBuildInputs": [],
    "stdenv": "/nix/store/f3hw3p8armnzy6xhd4h8s7anfjrs15n2-stdenv",
    "system": "x86_64-linux"
  }

"passAsFile" is ignored in this mode because it's not needed - large
strings are included directly in the JSON representation.

It is up to the builder to do something with the JSON
representation. For example, in bash-based builders, lists/attrsets of
string values could be mapped to bash (associative) arrays.

1 files changed, 39 insertions, 18 deletions

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 6250de13cb..d76c8d1727 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -865,6 +865,9 @@ private:
     /* Fill in the environment for the builder. */
     void initEnv();
 
+    /* Write a JSON file containing the derivation attributes. */
+    void writeStructuredAttrs();
+
     /* Make a file owned by the builder. */
     void chownToBuilder(const Path & path);
 
@@ -1726,13 +1729,15 @@ void DerivationGoal::startBuilder()
     tmpDirInSandbox = useChroot ? canonPath("/tmp", true) + "/nix-build-" + drvName + "-0" : tmpDir;
     chownToBuilder(tmpDir);
 
-    /* Construct the environment passed to the builder. */
-    initEnv();
-
     /* Substitute output placeholders with the actual output paths. */
     for (auto & output : drv->outputs)
         inputRewrites[hashPlaceholder(output.first)] = output.second.path;
 
+    /* Construct the environment passed to the builder. */
+    initEnv();
+
+    writeStructuredAttrs();
+
     /* Handle exportReferencesGraph(), if set. */
     doExportReferencesGraph();
 
@@ -2148,22 +2153,29 @@ void DerivationGoal::initEnv()
     /* The maximum number of cores to utilize for parallel building. */
     env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
 
-    /* Add all bindings specified in the derivation via the
-       environments, except those listed in the passAsFile
-       attribute. Those are passed as file names pointing to
-       temporary files containing the contents. */
-    StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile"));
-    int fileNr = 0;
-    for (auto & i : drv->env) {
-        if (passAsFile.find(i.first) == passAsFile.end()) {
-            env[i.first] = i.second;
-        } else {
-            string fn = ".attr-" + std::to_string(fileNr++);
-            Path p = tmpDir + "/" + fn;
-            writeFile(p, i.second);
-            chownToBuilder(p);
-            env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
+    /* In non-structured mode, add all bindings specified in the
+       derivation via the environments, except those listed in the
+       passAsFile attribute. Those are passed as file names pointing
+       to temporary files containing the contents. Note that
+       passAsFile is ignored in structure mode because it's not
+       needed (attributes are not passed through the environment, so
+       there is no size constraint). */
+    if (!drv->env.count("__json")) {
+
+        StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile"));
+        int fileNr = 0;
+        for (auto & i : drv->env) {
+            if (passAsFile.find(i.first) == passAsFile.end()) {
+                env[i.first] = i.second;
+            } else {
+                string fn = ".attr-" + std::to_string(fileNr++);
+                Path p = tmpDir + "/" + fn;
+                writeFile(p, i.second);
+                chownToBuilder(p);
+                env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
+            }
         }
+
     }
 
     /* For convenience, set an environment pointing to the top build
@@ -2201,6 +2213,15 @@ void DerivationGoal::initEnv()
 }
 
 
+void DerivationGoal::writeStructuredAttrs()
+{
+    auto json = drv->env.find("__json");
+    if (json == drv->env.end()) return;
+
+    writeFile(tmpDir + "/.attrs.json", rewriteStrings(json->second, inputRewrites));
+}
+
+
 void DerivationGoal::chownToBuilder(const Path & path)
 {
     if (!buildUser) return;