diff options
| -rw-r--r-- | doc/manual/command-ref/nix-store.xml | 12 | ||||
| -rw-r--r-- | src/libstore/build.cc | 219 | ||||
| -rw-r--r-- | src/nix-store/graphml.cc | 90 | ||||
| -rw-r--r-- | src/nix-store/graphml.hh (renamed from src/nix-store/xmlgraph.hh) | 2 | ||||
| -rw-r--r-- | src/nix-store/nix-store.cc | 10 | ||||
| -rw-r--r-- | src/nix-store/xmlgraph.cc | 66 | ||||
| -rw-r--r-- | tests/check-reqs.nix | 2 |
7 files changed, 277 insertions, 124 deletions
diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index c827d85b3815..41a04f265d7c 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -679,6 +679,18 @@ query is applied to the target of the symlink.</para> </varlistentry> + <varlistentry><term><option>--graphml</option></term> + + <listitem><para>Prints the references graph of the store paths + <replaceable>paths</replaceable> in the <link + xlink:href="http://graphml.graphdrawing.org/">GraphML</link> file format. + This can be used to visualise dependency graphs. To obtain a + build-time dependency graph, apply this to a store derivation. To + obtain a runtime dependency graph, apply it to an output + path.</para></listitem> + + </varlistentry> + <varlistentry><term><option>--binding</option> <replaceable>name</replaceable></term> <term><option>-b</option> <replaceable>name</replaceable></term> diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0073b9b727ec..cf4218a261fa 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -21,6 +21,7 @@ #include <future> #include <chrono> #include <regex> +#include <queue> #include <limits.h> #include <sys/time.h> @@ -857,7 +858,7 @@ private: building multiple times. Since this contains the hash, it allows us to compare whether two rounds produced the same result. */ - ValidPathInfos prevInfos; + std::map<Path, ValidPathInfo> prevInfos; const uid_t sandboxUid = 1000; const gid_t sandboxGid = 100; @@ -938,6 +939,11 @@ private: as valid. */ void registerOutputs(); + /* Check that an output meets the requirements specified by the + 'outputChecks' attribute (or the legacy + '{allowed,disallowed}{References,Requisites}' attributes). */ + void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs); + /* Open a log file and a pipe to it. */ Path openLogFile(); @@ -3010,7 +3016,7 @@ void DerivationGoal::registerOutputs() if (allValid) return; } - ValidPathInfos infos; + std::map<std::string, ValidPathInfo> infos; /* Set of inodes seen during calls to canonicalisePathMetaData() for this build's outputs. This needs to be shared between @@ -3195,49 +3201,6 @@ void DerivationGoal::registerOutputs() debug(format("referenced input: '%1%'") % i); } - /* Enforce `allowedReferences' and friends. */ - auto checkRefs = [&](const string & attrName, bool allowed, bool recursive) { - auto value = parsedDrv->getStringsAttr(attrName); - if (!value) return; - - PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value); - - PathSet used; - if (recursive) { - /* Our requisites are the union of the closures of our references. */ - for (auto & i : references) - /* Don't call computeFSClosure on ourselves. */ - if (path != i) - worker.store.computeFSClosure(i, used); - } else - used = references; - - PathSet badPaths; - - for (auto & i : used) - if (allowed) { - if (spec.find(i) == spec.end()) - badPaths.insert(i); - } else { - if (spec.find(i) != spec.end()) - badPaths.insert(i); - } - - if (!badPaths.empty()) { - string badPathsStr; - for (auto & i : badPaths) { - badPathsStr += "\n\t"; - badPathsStr += i; - } - throw BuildError(format("output '%1%' is not allowed to refer to the following paths:%2%") % actualPath % badPathsStr); - } - }; - - checkRefs("allowedReferences", true, false); - checkRefs("allowedRequisites", true, true); - checkRefs("disallowedReferences", false, false); - checkRefs("disallowedRequisites", false, true); - if (curRound == nrRounds) { worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() worker.markContentsGood(path); @@ -3253,11 +3216,14 @@ void DerivationGoal::registerOutputs() if (!info.references.empty()) info.ca.clear(); - infos.push_back(info); + infos[i.first] = info; } if (buildMode == bmCheck) return; + /* Apply output checks. */ + checkOutputs(infos); + /* Compare the result with the previous round, and report which path is different, if any.*/ if (curRound > 1 && prevInfos != infos) { @@ -3265,16 +3231,16 @@ void DerivationGoal::registerOutputs() for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) if (!(*i == *j)) { result.isNonDeterministic = true; - Path prev = i->path + checkSuffix; + Path prev = i->second.path + checkSuffix; bool prevExists = keepPreviousRound && pathExists(prev); auto msg = prevExists - ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->path, drvPath, prev) - : fmt("output '%1%' of '%2%' differs from previous round", i->path, drvPath); + ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->second.path, drvPath, prev) + : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath); auto diffHook = settings.diffHook; if (prevExists && diffHook != "" && runDiffHook) { try { - auto diff = runProgram(diffHook, true, {prev, i->path}); + auto diff = runProgram(diffHook, true, {prev, i->second.path}); if (diff != "") printError(chomp(diff)); } catch (Error & error) { @@ -3319,7 +3285,11 @@ void DerivationGoal::registerOutputs() /* Register each output path as valid, and register the sets of paths referenced by each of them. If there are cycles in the outputs, this will fail. */ - worker.store.registerValidPaths(infos); + { + ValidPathInfos infos2; + for (auto & i : infos) infos2.push_back(i.second); + worker.store.registerValidPaths(infos2); + } /* In case of a fixed-output derivation hash mismatch, throw an exception now that we have registered the output as valid. */ @@ -3328,6 +3298,153 @@ void DerivationGoal::registerOutputs() } +void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) +{ + std::map<Path, const ValidPathInfo &> outputsByPath; + for (auto & output : outputs) + outputsByPath.emplace(output.second.path, output.second); + + for (auto & output : outputs) { + auto & outputName = output.first; + auto & info = output.second; + + struct Checks + { + std::experimental::optional<uint64_t> maxSize, maxClosureSize; + std::experimental::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; + }; + + /* Compute the closure and closure size of some output. This + is slightly tricky because some of its references (namely + other outputs) may not be valid yet. */ + auto getClosure = [&](const Path & path) + { + uint64_t closureSize = 0; + PathSet pathsDone; + std::queue<Path> pathsLeft; + pathsLeft.push(path); + + while (!pathsLeft.empty()) { + auto path = pathsLeft.front(); + pathsLeft.pop(); + if (!pathsDone.insert(path).second) continue; + + auto i = outputsByPath.find(path); + if (i != outputsByPath.end()) { + closureSize += i->second.narSize; + for (auto & ref : i->second.references) + pathsLeft.push(ref); + } else { + auto info = worker.store.queryPathInfo(path); + closureSize += info->narSize; + for (auto & ref : info->references) + pathsLeft.push(ref); + } + } + + return std::make_pair(pathsDone, closureSize); + }; + + auto checkRefs = [&](const std::experimental::optional<Strings> & value, bool allowed, bool recursive) + { + if (!value) return; + + PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value); + + PathSet used = recursive ? getClosure(info.path).first : info.references; + + PathSet badPaths; + + for (auto & i : used) + if (allowed) { + if (spec.find(i) == spec.end()) + badPaths.insert(i); + } else { + if (spec.find(i) != spec.end()) + badPaths.insert(i); + } + + if (!badPaths.empty()) { + string badPathsStr; + for (auto & i : badPaths) { + badPathsStr += "\n "; + badPathsStr += i; + } + throw BuildError("output '%s' is not allowed to refer to the following paths:%s", info.path, badPathsStr); + } + }; + + auto applyChecks = [&](const Checks & checks) + { + if (checks.maxSize && info.narSize > *checks.maxSize) + throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", + info.path, info.narSize, *checks.maxSize); + + if (checks.maxClosureSize) { + uint64_t closureSize = getClosure(info.path).second; + if (closureSize > *checks.maxClosureSize) + throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", + info.path, closureSize, *checks.maxClosureSize); + } + + checkRefs(checks.allowedReferences, true, false); + checkRefs(checks.allowedRequisites, true, true); + checkRefs(checks.disallowedReferences, false, false); + checkRefs(checks.disallowedRequisites, false, true); + }; + + if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + auto outputChecks = structuredAttrs->find("outputChecks"); + if (outputChecks != structuredAttrs->end()) { + auto output = outputChecks->find(outputName); + + if (output != outputChecks->end()) { + Checks checks; + + auto maxSize = output->find("maxSize"); + if (maxSize != output->end()) + checks.maxSize = maxSize->get<uint64_t>(); + + auto maxClosureSize = output->find("maxClosureSize"); + if (maxClosureSize != output->end()) + checks.maxClosureSize = maxClosureSize->get<uint64_t>(); + + auto get = [&](const std::string & name) -> std::experimental::optional<Strings> { + auto i = output->find(name); + if (i != output->end()) { + Strings res; + for (auto j = i->begin(); j != i->end(); ++j) { + if (!j->is_string()) + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); + res.push_back(j->get<std::string>()); + } + checks.disallowedRequisites = res; + return res; + } + return {}; + }; + + checks.allowedReferences = get("allowedReferences"); + checks.allowedRequisites = get("allowedRequisites"); + checks.disallowedReferences = get("disallowedReferences"); + checks.disallowedRequisites = get("disallowedRequisites"); + + applyChecks(checks); + } + } + } else { + // legacy non-structured-attributes case + Checks checks; + checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); + checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); + checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences"); + checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites"); + applyChecks(checks); + } + } +} + + Path DerivationGoal::openLogFile() { logSize = 0; diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc new file mode 100644 index 000000000000..670fbe227a4c --- /dev/null +++ b/src/nix-store/graphml.cc @@ -0,0 +1,90 @@ +#include "graphml.hh" +#include "util.hh" +#include "store-api.hh" +#include "derivations.hh" + +#include <iostream> + + +using std::cout; + +namespace nix { + + +static inline const string & xmlQuote(const string & s) +{ + // Luckily, store paths shouldn't contain any character that needs to be + // quoted. + return s; +} + + +static string symbolicName(const string & path) +{ + string p = baseNameOf(path); + return string(p, p.find('-') + 1); +} + + +static string makeEdge(const string & src, const string & dst) +{ + return fmt(" <edge source=\"%1%\" target=\"%2%\"/>\n", + xmlQuote(src), xmlQuote(dst)); +} + + +static string makeNode(const ValidPathInfo & info) +{ + return fmt( + " <node id=\"%1%\">\n" + " <data key=\"narSize\">%2%</data>\n" + " <data key=\"name\">%3%</data>\n" + " <data key=\"type\">%4%</data>\n" + " </node>\n", + info.path, + info.narSize, + symbolicName(info.path), + (isDerivation(info.path) ? "derivation" : "output-path")); +} + + +void printGraphML(ref<Store> store, const PathSet & roots) +{ + PathSet workList(roots); + PathSet doneSet; + std::pair<PathSet::iterator,bool> ret; + + cout << "<?xml version='1.0' encoding='utf-8'?>\n" + << "<graphml xmlns='http://graphml.graphdrawing.org/xmlns'\n" + << " xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'\n" + << " xsi:schemaLocation='http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd'>\n" + << "<key id='narSize' for='node' attr.name='narSize' attr.type='int'/>" + << "<key id='name' for='node' attr.name='name' attr.type='string'/>" + << "<key id='type' for='node' attr.name='type' attr.type='string'/>" + << "<graph id='G' edgedefault='directed'>\n"; + + while (!workList.empty()) { + Path path = *(workList.begin()); + workList.erase(path); + + ret = doneSet.insert(path); + if (ret.second == false) continue; + + ValidPathInfo info = *(store->queryPathInfo(path)); + cout << makeNode(info); + + for (auto & p : store->queryPathInfo(path)->references) { + if (p != path) { + workList.insert(p); + cout << makeEdge(path, p); + } + } + + } + + cout << "</graph>\n"; + cout << "</graphml>\n"; +} + + +} diff --git a/src/nix-store/xmlgraph.hh b/src/nix-store/graphml.hh index a6e7d4e2805a..b78df1e49a67 100644 --- a/src/nix-store/xmlgraph.hh +++ b/src/nix-store/graphml.hh @@ -6,6 +6,6 @@ namespace nix { class Store; -void printXmlGraph(ref<Store> store, const PathSet & roots); +void printGraphML(ref<Store> store, const PathSet & roots); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index fe68f681ae28..e245bd643d4b 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -8,7 +8,7 @@ #include "shared.hh" #include "util.hh" #include "worker-protocol.hh" -#include "xmlgraph.hh" +#include "graphml.hh" #include <iostream> #include <algorithm> @@ -273,7 +273,7 @@ static void opQuery(Strings opFlags, Strings opArgs) enum QueryType { qDefault, qOutputs, qRequisites, qReferences, qReferrers , qReferrersClosure, qDeriver, qBinding, qHash, qSize - , qTree, qGraph, qXml, qResolve, qRoots }; + , qTree, qGraph, qGraphML, qResolve, qRoots }; QueryType query = qDefault; bool useOutput = false; bool includeOutputs = false; @@ -299,7 +299,7 @@ static void opQuery(Strings opFlags, Strings opArgs) else if (i == "--size") query = qSize; else if (i == "--tree") query = qTree; else if (i == "--graph") query = qGraph; - else if (i == "--xml") query = qXml; + else if (i == "--graphml") query = qGraphML; else if (i == "--resolve") query = qResolve; else if (i == "--roots") query = qRoots; else if (i == "--use-output" || i == "-u") useOutput = true; @@ -403,13 +403,13 @@ static void opQuery(Strings opFlags, Strings opArgs) break; } - case qXml: { + case qGraphML: { PathSet roots; for (auto & i : opArgs) { PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); roots.insert(paths.begin(), paths.end()); } - printXmlGraph(ref<Store>(store), roots); + printGraphML(ref<Store>(store), roots); break; } diff --git a/src/nix-store/xmlgraph.cc b/src/nix-store/xmlgraph.cc deleted file mode 100644 index 0f7be7f7a02d..000000000000 --- a/src/nix-store/xmlgraph.cc +++ /dev/null @@ -1,66 +0,0 @@ -#include "xmlgraph.hh" -#include "util.hh" -#include "store-api.hh" - -#include <iostream> - - -using std::cout; - -namespace nix { - - -static inline const string & xmlQuote(const string & s) -{ - // Luckily, store paths shouldn't contain any character that needs to be - // quoted. - return s; -} - - -static string makeEdge(const string & src, const string & dst) -{ - format f = format(" <edge src=\"%1%\" dst=\"%2%\"/>\n") - % xmlQuote(src) % xmlQuote(dst); - return f.str(); -} - - -static string makeNode(const string & id) -{ - format f = format(" <node name=\"%1%\"/>\n") % xmlQuote(id); - return f.str(); -} - - -void printXmlGraph(ref<Store> store, const PathSet & roots) -{ - PathSet workList(roots); - PathSet doneSet; - - cout << "<?xml version='1.0' encoding='utf-8'?>\n" - << "<nix>\n"; - - while (!workList.empty()) { - Path path = *(workList.begin()); - workList.erase(path); - - if (doneSet.find(path) != doneSet.end()) continue; - doneSet.insert(path); - - cout << makeNode(path); - - for (auto & p : store->queryPathInfo(path)->references) { - if (p != path) { - workList.insert(p); - cout << makeEdge(p, path); - } - } - - } - - cout << "</nix>\n"; -} - - -} diff --git a/tests/check-reqs.nix b/tests/check-reqs.nix index 41436cb48e08..47b5b3d9c723 100644 --- a/tests/check-reqs.nix +++ b/tests/check-reqs.nix @@ -33,7 +33,7 @@ rec { }; # When specifying all the requisites, the build succeeds. - test1 = makeTest 1 [ dep1 dep2 deps ]; + test1 = makeTest 1 [ "out" dep1 dep2 deps ]; # But missing anything it fails. test2 = makeTest 2 [ dep2 deps ]; |