From 6de33a9c675b187437a2e1abbcb290981a89ecb1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Jan 2017 16:42:07 +0100 Subject: Add support for passing structured data to builders Previously, all derivation attributes had to be coerced into strings so that they could be passed via the environment. This is lossy (e.g. lists get flattened, necessitating configureFlags vs. configureFlagsArray, of which the latter cannot be specified as an attribute), doesn't support attribute sets at all, and has size limitations (necessitating hacks like passAsFile). This patch adds a new mode for passing attributes to builders, namely encoded as a JSON file ".attrs.json" in the current directory of the builder. This mode is activated via the special attribute __structuredAttrs = true; (The idea is that one day we can set this in stdenv.mkDerivation.) For example, stdenv.mkDerivation { __structuredAttrs = true; name = "foo"; buildInputs = [ pkgs.hello pkgs.cowsay ]; doCheck = true; hardening.format = false; } results in a ".attrs.json" file containing (sans the indentation): { "buildInputs": [], "builder": "/nix/store/ygl61ycpr2vjqrx775l1r2mw1g2rb754-bash-4.3-p48/bin/bash", "configureFlags": [ "--with-foo", "--with-bar=1 2" ], "doCheck": true, "hardening": { "format": false }, "name": "foo", "nativeBuildInputs": [ "/nix/store/10h6li26i7g6z3mdpvra09yyf10mmzdr-hello-2.10", "/nix/store/4jnvjin0r6wp6cv1hdm5jbkx3vinlcvk-cowsay-3.03" ], "propagatedBuildInputs": [], "propagatedNativeBuildInputs": [], "stdenv": "/nix/store/f3hw3p8armnzy6xhd4h8s7anfjrs15n2-stdenv", "system": "x86_64-linux" } "passAsFile" is ignored in this mode because it's not needed - large strings are included directly in the JSON representation. It is up to the builder to do something with the JSON representation. For example, in bash-based builders, lists/attrsets of string values could be mapped to bash (associative) arrays. --- src/libexpr/eval.cc | 2 + src/libexpr/eval.hh | 2 +- src/libexpr/primops.cc | 127 ++++++++++++++++++++++++++++++++++--------------- 3 files changed, 91 insertions(+), 40 deletions(-) (limited to 'src/libexpr') diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 5f9f5bac1ed6..d418ab4e43aa 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -291,6 +291,8 @@ EvalState::EvalState(const Strings & _searchPath, ref store) , sToString(symbols.create("__toString")) , sRight(symbols.create("right")) , sWrong(symbols.create("wrong")) + , sStructuredAttrs(symbols.create("__structuredAttrs")) + , sBuilder(symbols.create("builder")) , store(store) , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 195cb0db3acc..46d5a1cc866c 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -68,7 +68,7 @@ public: const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, - sRight, sWrong; + sRight, sWrong, sStructuredAttrs, sBuilder; Symbol sDerivationNix; /* If set, force copying files to the Nix store even if they diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d8dbedcaf89e..5a570cefb2fa 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -8,6 +8,7 @@ #include "names.hh" #include "store-api.hh" #include "util.hh" +#include "json.hh" #include "value-to-json.hh" #include "value-to-xml.hh" #include "primops.hh" @@ -474,6 +475,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * throw; } + /* Check whether attributes should be passed as a JSON file. */ + std::ostringstream jsonBuf; + std::unique_ptr jsonObject; + attr = args[0]->attrs->find(state.sStructuredAttrs); + if (attr != args[0]->attrs->end() && state.forceBool(*attr->value, pos)) + jsonObject = std::make_unique(jsonBuf); + /* Check whether null attributes should be ignored. */ bool ignoreNulls = false; attr = args[0]->attrs->find(state.sIgnoreNulls); @@ -491,24 +499,48 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * StringSet outputs; outputs.insert("out"); - for (auto & i : *args[0]->attrs) { - if (i.name == state.sIgnoreNulls) continue; - string key = i.name; + for (auto & i : args[0]->attrs->lexicographicOrder()) { + if (i->name == state.sIgnoreNulls) continue; + string key = i->name; Activity act(*logger, lvlVomit, format("processing attribute ‘%1%’") % key); + auto handleHashMode = [&](const std::string & s) { + if (s == "recursive") outputHashRecursive = true; + else if (s == "flat") outputHashRecursive = false; + else throw EvalError("invalid value ‘%s’ for ‘outputHashMode’ attribute, at %s", s, posDrvName); + }; + + auto handleOutputs = [&](const Strings & ss) { + outputs.clear(); + for (auto & j : ss) { + if (outputs.find(j) != outputs.end()) + throw EvalError(format("duplicate derivation output ‘%1%’, at %2%") % j % posDrvName); + /* !!! Check whether j is a valid attribute + name. */ + /* Derivations cannot be named ‘drv’, because + then we'd have an attribute ‘drvPath’ in + the resulting set. */ + if (j == "drv") + throw EvalError(format("invalid derivation output name ‘drv’, at %1%") % posDrvName); + outputs.insert(j); + } + if (outputs.empty()) + throw EvalError(format("derivation cannot have an empty set of outputs, at %1%") % posDrvName); + }; + try { if (ignoreNulls) { - state.forceValue(*i.value); - if (i.value->type == tNull) continue; + state.forceValue(*i->value); + if (i->value->type == tNull) continue; } /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ if (key == "args") { - state.forceList(*i.value, pos); - for (unsigned int n = 0; n < i.value->listSize(); ++n) { - string s = state.coerceToString(posDrvName, *i.value->listElems()[n], context, true); + state.forceList(*i->value, pos); + for (unsigned int n = 0; n < i->value->listSize(); ++n) { + string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true); drv.args.push_back(s); } } @@ -516,39 +548,51 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* All other attributes are passed to the builder through the environment. */ else { - string s = state.coerceToString(posDrvName, *i.value, context, true); - drv.env[key] = s; - if (key == "builder") drv.builder = s; - else if (i.name == state.sSystem) drv.platform = s; - else if (i.name == state.sName) { - drvName = s; - printMsg(lvlVomit, format("derivation name is ‘%1%’") % drvName); - } - else if (key == "outputHash") outputHash = s; - else if (key == "outputHashAlgo") outputHashAlgo = s; - else if (key == "outputHashMode") { - if (s == "recursive") outputHashRecursive = true; - else if (s == "flat") outputHashRecursive = false; - else throw EvalError(format("invalid value ‘%1%’ for ‘outputHashMode’ attribute, at %2%") % s % posDrvName); - } - else if (key == "outputs") { - Strings tmp = tokenizeString(s); - outputs.clear(); - for (auto & j : tmp) { - if (outputs.find(j) != outputs.end()) - throw EvalError(format("duplicate derivation output ‘%1%’, at %2%") % j % posDrvName); - /* !!! Check whether j is a valid attribute - name. */ - /* Derivations cannot be named ‘drv’, because - then we'd have an attribute ‘drvPath’ in - the resulting set. */ - if (j == "drv") - throw EvalError(format("invalid derivation output name ‘drv’, at %1%") % posDrvName); - outputs.insert(j); + + if (jsonObject) { + + if (i->name == state.sStructuredAttrs) continue; + + auto placeholder(jsonObject->placeholder(key)); + printValueAsJSON(state, true, *i->value, placeholder, context); + + if (i->name == state.sBuilder) + drv.builder = state.forceString(*i->value, context, posDrvName); + else if (i->name == state.sSystem) + drv.platform = state.forceStringNoCtx(*i->value, posDrvName); + else if (i->name == state.sName) + drvName = state.forceStringNoCtx(*i->value, posDrvName); + else if (key == "outputHash") + outputHash = state.forceStringNoCtx(*i->value, posDrvName); + else if (key == "outputHashAlgo") + outputHashAlgo = state.forceStringNoCtx(*i->value, posDrvName); + else if (key == "outputHashMode") + handleHashMode(state.forceStringNoCtx(*i->value, posDrvName)); + else if (key == "outputs") { + /* Require ‘outputs’ to be a list of strings. */ + state.forceList(*i->value, posDrvName); + Strings ss; + for (unsigned int n = 0; n < i->value->listSize(); ++n) + ss.emplace_back(state.forceStringNoCtx(*i->value->listElems()[n], posDrvName)); + handleOutputs(ss); + } + + } else { + auto s = state.coerceToString(posDrvName, *i->value, context, true); + drv.env.emplace(key, s); + if (i->name == state.sBuilder) drv.builder = s; + else if (i->name == state.sSystem) drv.platform = s; + else if (i->name == state.sName) { + drvName = s; + printMsg(lvlVomit, format("derivation name is ‘%1%’") % drvName); } - if (outputs.empty()) - throw EvalError(format("derivation cannot have an empty set of outputs, at %1%") % posDrvName); + else if (key == "outputHash") outputHash = s; + else if (key == "outputHashAlgo") outputHashAlgo = s; + else if (key == "outputHashMode") handleHashMode(s); + else if (key == "outputs") + handleOutputs(tokenizeString(s)); } + } } catch (Error & e) { @@ -558,6 +602,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * } } + if (jsonObject) { + jsonObject.reset(); + drv.env.emplace("__json", jsonBuf.str()); + } + /* Everything in the context of the strings in the derivation attributes should be added as dependencies of the resulting derivation. */ -- cgit 1.4.1