about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/libexpr/get-drvs.hh7
-rw-r--r--src/libexpr/primops.cc6
-rw-r--r--src/libstore/build.cc46
-rw-r--r--src/libstore/globals.cc1
-rw-r--r--src/libstore/local-store.cc83
-rw-r--r--src/libstore/local-store.hh16
-rw-r--r--src/libstore/misc.cc106
-rw-r--r--src/libstore/optimise-store.cc2
-rw-r--r--src/libstore/remote-store.cc98
-rw-r--r--src/libstore/remote-store.hh10
-rw-r--r--src/libstore/store-api.hh32
-rw-r--r--src/libstore/worker-protocol.hh7
-rw-r--r--src/nix-env/nix-env.cc28
-rw-r--r--src/nix-store/nix-store.cc2
-rw-r--r--src/nix-worker/nix-worker.cc62
15 files changed, 368 insertions, 138 deletions
diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh
index 2d260c57beee..1e5d0a817cca 100644
--- a/src/libexpr/get-drvs.hh
+++ b/src/libexpr/get-drvs.hh
@@ -32,6 +32,8 @@ private:
 
     bool metaInfoRead;
     MetaInfo meta;
+
+    bool failed; // set if we get an AssertionError
     
 public:
     string name;
@@ -41,7 +43,7 @@ public:
     /* !!! make this private */
     Bindings * attrs;
 
-    DrvInfo() : metaInfoRead(false), attrs(0) { };
+    DrvInfo() : metaInfoRead(false), failed(false), attrs(0) { };
 
     string queryDrvPath(EvalState & state) const;
     string queryOutPath(EvalState & state) const;
@@ -59,6 +61,9 @@ public:
     }
 
     void setMetaInfo(const MetaInfo & meta);
+
+    void setFailed() { failed = true; };
+    bool hasFailed() { return failed; };
 };
 
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 5d5f0bfb3b5b..f20c2f2879ab 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -51,6 +51,12 @@ static void prim_import(EvalState & state, Value * * args, Value & v)
                 % path % ctx);
         if (isDerivation(ctx))
             try {
+                /* For performance, prefetch all substitute info. */
+                PathSet willBuild, willSubstitute, unknown;
+                unsigned long long downloadSize, narSize;
+                queryMissing(*store, singleton<PathSet>(ctx),
+                    willBuild, willSubstitute, unknown, downloadSize, narSize);
+                  
                 /* !!! If using a substitute, we only need to fetch
                    the selected output of this derivation. */
                 store->buildPaths(singleton<PathSet>(ctx));
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 26268f6ddb7d..4bef6f02dc00 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -94,7 +94,7 @@ typedef map<Path, WeakGoalPtr> WeakGoalMap;
 class Goal : public boost::enable_shared_from_this<Goal>
 {
 public:
-    typedef enum {ecBusy, ecSuccess, ecFailed} ExitCode;
+    typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters} ExitCode;
     
 protected:
     
@@ -111,6 +111,10 @@ protected:
     /* Number of goals we are/were waiting for that have failed. */
     unsigned int nrFailed;
 
+    /* Number of substitution goals we are/were waiting for that
+       failed because there are no substituters. */
+    unsigned int nrNoSubstituters;
+
     /* Name of this goal for debugging purposes. */
     string name;
 
@@ -119,7 +123,7 @@ protected:
 
     Goal(Worker & worker) : worker(worker)
     {
-        nrFailed = 0;
+        nrFailed = nrNoSubstituters = 0;
         exitCode = ecBusy;
     }
 
@@ -306,7 +310,9 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
     trace(format("waitee `%1%' done; %2% left") %
         waitee->name % waitees.size());
     
-    if (result == ecFailed) ++nrFailed;
+    if (result == ecFailed || result == ecNoSubstituters) ++nrFailed;
+
+    if (result == ecNoSubstituters) ++nrNoSubstituters;
     
     if (waitees.empty() || (result == ecFailed && !keepGoing)) {
 
@@ -330,7 +336,7 @@ void Goal::amDone(ExitCode result)
 {
     trace("done");
     assert(exitCode == ecBusy);
-    assert(result == ecSuccess || result == ecFailed);
+    assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters);
     exitCode = result;
     foreach (WeakGoals::iterator, i, waiters) {
         GoalPtr goal = i->lock();
@@ -736,6 +742,8 @@ HookInstance::~HookInstance()
 
 typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
 
+class SubstitutionGoal;
+
 class DerivationGoal : public Goal
 {
 private:
@@ -986,10 +994,8 @@ void DerivationGoal::haveDerivation()
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
-    foreach (PathSet::iterator, i, invalidOutputs)
-        /* Don't bother creating a substitution goal if there are no
-           substitutes. */
-        if (queryBoolSetting("build-use-substitutes", true) && worker.store.hasSubstitutes(*i))
+    if (queryBoolSetting("build-use-substitutes", true))
+        foreach (PathSet::iterator, i, invalidOutputs)
             addWaitee(worker.makeSubstitutionGoal(*i));
     
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
@@ -1003,10 +1009,10 @@ void DerivationGoal::outputsSubstituted()
 {
     trace("all outputs substituted (maybe)");
 
-    if (nrFailed > 0 && !tryFallback)
+    if (nrFailed > 0 && nrFailed > nrNoSubstituters && !tryFallback)
         throw Error(format("some substitutes for the outputs of derivation `%1%' failed; try `--fallback'") % drvPath);
 
-    nrFailed = 0;
+    nrFailed = nrNoSubstituters = 0;
 
     if (checkPathValidity(false).size() == 0) {
         amDone(ecSuccess);
@@ -2258,6 +2264,9 @@ private:
     /* The current substituter. */
     Path sub;
 
+    /* Whether any substituter can realise this path */
+    bool hasSubstitute;
+
     /* Path info returned by the substituter's query info operation. */
     SubstitutablePathInfo info;
 
@@ -2299,6 +2308,7 @@ public:
 
 SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker)
     : Goal(worker)
+    , hasSubstitute(false)
 {
     this->storePath = storePath;
     state = &SubstitutionGoal::init;
@@ -2362,17 +2372,23 @@ void SubstitutionGoal::tryNext()
         /* None left.  Terminate this goal and let someone else deal
            with it. */
         debug(format("path `%1%' is required, but there is no substituter that can build it") % storePath);
-        amDone(ecFailed);
+        /* Hack: don't indicate failure if there were no substituters.
+           In that case the calling derivation should just do a
+           build. */
+        amDone(hasSubstitute ? ecFailed : ecNoSubstituters);
         return;
     }
 
     sub = subs.front();
     subs.pop_front();
 
-    if (!worker.store.querySubstitutablePathInfo(sub, storePath, info)) {
-        tryNext();
-        return;
-    }
+    SubstitutablePathInfos infos;
+    PathSet dummy(singleton<PathSet>(storePath));
+    worker.store.querySubstitutablePathInfos(sub, dummy, infos);
+    SubstitutablePathInfos::iterator k = infos.find(storePath);
+    if (k == infos.end()) { tryNext(); return; }
+    info = k->second;
+    hasSubstitute = true;
 
     /* To maintain the closure invariant, we first have to realise the
        paths referenced by this one. */
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 5c22f1406649..9636bf49d987 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -157,6 +157,7 @@ void setDefaultsFromEnvironment()
     if (subs == "default") {
         substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
         substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
+        substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl");
     } else
         substituters = tokenizeString(subs, ":");
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 30398a244607..aa21774df4a7 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -748,7 +748,16 @@ bool LocalStore::isValidPath(const Path & path)
 }
 
 
-PathSet LocalStore::queryValidPaths()
+PathSet LocalStore::queryValidPaths(const PathSet & paths)
+{
+    PathSet res;
+    foreach (PathSet::const_iterator, i, paths)
+        if (isValidPath(*i)) res.insert(*i);
+    return res;
+}
+
+
+PathSet LocalStore::queryAllValidPaths()
 {
     SQLiteStmt stmt;
     stmt.create(db, "select path from ValidPaths");
@@ -947,50 +956,66 @@ template<class T> T getIntLine(int fd)
 }
 
 
-bool LocalStore::hasSubstitutes(const Path & path)
+PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
 {
+    PathSet res;
     foreach (Paths::iterator, i, substituters) {
+        if (res.size() == paths.size()) break;
         RunningSubstituter & run(runningSubstituters[*i]);
         startSubstituter(*i, run);
-        writeLine(run.to, "have\n" + path);
-        if (getIntLine<int>(run.from)) return true;
+        string s = "have ";
+        foreach (PathSet::const_iterator, i, paths)
+            if (res.find(*i) == res.end()) { s += *i; s += " "; }
+        writeLine(run.to, s);
+        while (true) {
+            Path path = readLine(run.from);
+            if (path == "") break;
+            res.insert(path);
+        }
     }
-
-    return false;
+    return res;
 }
 
 
-bool LocalStore::querySubstitutablePathInfo(const Path & substituter,
-    const Path & path, SubstitutablePathInfo & info)
+void LocalStore::querySubstitutablePathInfos(const Path & substituter,
+    PathSet & paths, SubstitutablePathInfos & infos)
 {
     RunningSubstituter & run(runningSubstituters[substituter]);
     startSubstituter(substituter, run);
 
-    writeLine(run.to, "info\n" + path);
+    string s = "info ";
+    foreach (PathSet::const_iterator, i, paths)
+        if (infos.find(*i) == infos.end()) { s += *i; s += " "; }
+    writeLine(run.to, s);
 
-    if (!getIntLine<int>(run.from)) return false;
-    
-    info.deriver = readLine(run.from);
-    if (info.deriver != "") assertStorePath(info.deriver);
-    int nrRefs = getIntLine<int>(run.from);
-    while (nrRefs--) {
-        Path p = readLine(run.from);
-        assertStorePath(p);
-        info.references.insert(p);
+    while (true) {
+        Path path = readLine(run.from);
+        if (path == "") break;
+        assert(paths.find(path) != paths.end());
+        paths.erase(path);
+        SubstitutablePathInfo & info(infos[path]);
+        info.deriver = readLine(run.from);
+        if (info.deriver != "") assertStorePath(info.deriver);
+        int nrRefs = getIntLine<int>(run.from);
+        while (nrRefs--) {
+            Path p = readLine(run.from);
+            assertStorePath(p);
+            info.references.insert(p);
+        }
+        info.downloadSize = getIntLine<long long>(run.from);
+        info.narSize = getIntLine<long long>(run.from);
     }
-    info.downloadSize = getIntLine<long long>(run.from);
-    info.narSize = getIntLine<long long>(run.from);
-    
-    return true;
 }
 
 
-bool LocalStore::querySubstitutablePathInfo(const Path & path,
-    SubstitutablePathInfo & info)
+void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
+    SubstitutablePathInfos & infos)
 {
-    foreach (Paths::iterator, i, substituters)
-        if (querySubstitutablePathInfo(*i, path, info)) return true;
-    return false;
+    PathSet todo = paths;
+    foreach (Paths::iterator, i, substituters) {
+        if (todo.empty()) break;
+        querySubstitutablePathInfos(*i, todo, infos);
+    }
 }
 
 
@@ -1134,7 +1159,7 @@ Path LocalStore::addToStore(const Path & _srcPath,
        method for very large paths, but `copyPath' is mainly used for
        small files. */
     StringSink sink;
-    if (recursive) 
+    if (recursive)
         dumpPath(srcPath, sink, filter);
     else
         sink.s = readFile(srcPath);
@@ -1465,7 +1490,7 @@ void LocalStore::verifyStore(bool checkContents)
     /* Check whether all valid paths actually exist. */
     printMsg(lvlInfo, "checking path existence...");
 
-    PathSet validPaths2 = queryValidPaths(), validPaths, done;
+    PathSet validPaths2 = queryAllValidPaths(), validPaths, done;
 
     foreach (PathSet::iterator, i, validPaths2)
         verifyPath(*i, store, done, validPaths);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 65ee029c261e..07d8198ecaa5 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -99,7 +99,9 @@ public:
     
     bool isValidPath(const Path & path);
 
-    PathSet queryValidPaths();
+    PathSet queryValidPaths(const PathSet & paths);
+    
+    PathSet queryAllValidPaths();
     
     ValidPathInfo queryPathInfo(const Path & path);
 
@@ -123,15 +125,13 @@ public:
     
     Path queryPathFromHashPart(const string & hashPart);
     
-    PathSet querySubstitutablePaths();
-    
-    bool hasSubstitutes(const Path & path);
+    PathSet querySubstitutablePaths(const PathSet & paths);
 
-    bool querySubstitutablePathInfo(const Path & path,
-        SubstitutablePathInfo & info);
+    void querySubstitutablePathInfos(const Path & substituter,
+        PathSet & paths, SubstitutablePathInfos & infos);
     
-    bool querySubstitutablePathInfo(const Path & substituter,
-        const Path & path, SubstitutablePathInfo & info);
+    void querySubstitutablePathInfos(const PathSet & paths,
+        SubstitutablePathInfos & infos);
     
     Path addToStore(const Path & srcPath,
         bool recursive = true, HashType hashAlgo = htSHA256,
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 093499936349..aa5f6ff727c9 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -55,45 +55,97 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
     
     PathSet todo(targets.begin(), targets.end()), done;
 
+    bool useSubstitutes = queryBoolSetting("build-use-substitutes", true);
+
+    /* Getting substitute info has high latency when using the binary
+       cache substituter.  Thus it's essential to do substitute
+       queries in parallel as much as possible.  To accomplish this
+       we do the following:
+
+       - For all paths still to be processed (‘todo’), we add all
+         paths for which we need info to the set ‘query’.  For an
+         unbuilt derivation this is the output paths; otherwise, it's
+         the path itself.
+
+       - We get info about all paths in ‘query’ in parallel.
+
+       - We process the results and add new items to ‘todo’ if
+         necessary.  E.g. if a path is substitutable, then we need to
+         get info on its references.
+
+       - Repeat until ‘todo’ is empty.
+    */
+
     while (!todo.empty()) {
-        Path p = *(todo.begin());
-        todo.erase(p);
-        if (done.find(p) != done.end()) continue;
-        done.insert(p);
-
-        if (isDerivation(p)) {
-            if (!store.isValidPath(p)) {
-                unknown.insert(p);
-                continue;
+              
+        PathSet query, todoDrv, todoNonDrv;
+
+        foreach (PathSet::iterator, i, todo) {
+            if (done.find(*i) != done.end()) continue;
+            done.insert(*i);
+
+            if (isDerivation(*i)) {
+                if (!store.isValidPath(*i)) {
+                    // FIXME: we could try to substitute p.
+                    unknown.insert(*i);
+                    continue;
+                }
+                Derivation drv = derivationFromPath(store, *i);
+
+                PathSet invalid;
+                foreach (DerivationOutputs::iterator, j, drv.outputs)
+                    if (!store.isValidPath(j->second.path)) invalid.insert(j->second.path);
+                if (invalid.empty()) continue;
+                
+                todoDrv.insert(*i);
+                if (useSubstitutes) query.insert(invalid.begin(), invalid.end());
+            }
+
+            else {
+                if (store.isValidPath(*i)) continue;
+                query.insert(*i);
+                todoNonDrv.insert(*i);
             }
-            Derivation drv = derivationFromPath(store, p);
+        }
+
+        todo.clear();
+        
+        SubstitutablePathInfos infos;
+        store.querySubstitutablePathInfos(query, infos);
+
+        foreach (PathSet::iterator, i, todoDrv) {
+            // FIXME: cache this
+            Derivation drv = derivationFromPath(store, *i);
 
             bool mustBuild = false;
-            foreach (DerivationOutputs::iterator, i, drv.outputs)
-                if (!store.isValidPath(i->second.path) &&
-                    !(queryBoolSetting("build-use-substitutes", true) && store.hasSubstitutes(i->second.path)))
-                    mustBuild = true;
+            if (useSubstitutes) {
+                foreach (DerivationOutputs::iterator, j, drv.outputs)
+                    if (!store.isValidPath(j->second.path) &&
+                        infos.find(j->second.path) == infos.end())
+                        mustBuild = true;
+            } else
+                mustBuild = true;
 
             if (mustBuild) {
-                willBuild.insert(p);
+                willBuild.insert(*i);
                 todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
                 foreach (DerivationInputs::iterator, i, drv.inputDrvs)
                     todo.insert(i->first);
-            } else 
+            } else
                 foreach (DerivationOutputs::iterator, i, drv.outputs)
-                    todo.insert(i->second.path);
+                    todoNonDrv.insert(i->second.path);
         }
-
-        else {
-            if (store.isValidPath(p)) continue;
-            SubstitutablePathInfo info;
-            if (store.querySubstitutablePathInfo(p, info)) {
-                willSubstitute.insert(p);
-                downloadSize += info.downloadSize;
-                narSize += info.narSize;
-                todo.insert(info.references.begin(), info.references.end());
+        
+        foreach (PathSet::iterator, i, todoNonDrv) {
+            done.insert(*i);
+            SubstitutablePathInfos::iterator info = infos.find(*i);
+            if (info != infos.end()) {
+                willSubstitute.insert(*i);
+                downloadSize += info->second.downloadSize;
+                narSize += info->second.narSize;
+                todo.insert(info->second.references.begin(), info->second.references.end());
             } else
-                unknown.insert(p);
+                unknown.insert(*i);
         }
     }
 }
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 2ca98f46ddf4..a486e66ef59e 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -178,7 +178,7 @@ void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats)
 {
     HashToPath hashToPath;
 
-    PathSet paths = queryValidPaths();
+    PathSet paths = queryAllValidPaths();
 
     foreach (PathSet::iterator, i, paths) {
         addTempRoot(*i);
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index cbb70b2fd726..35530acab1af 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -217,42 +217,96 @@ bool RemoteStore::isValidPath(const Path & path)
 }
 
 
-PathSet RemoteStore::queryValidPaths()
+PathSet RemoteStore::queryValidPaths(const PathSet & paths)
 {
     openConnection();
-    writeInt(wopQueryValidPaths, to);
+    if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
+        PathSet res;
+        foreach (PathSet::const_iterator, i, paths)
+            if (isValidPath(*i)) res.insert(*i);
+        return res;
+    } else {
+        writeInt(wopQueryValidPaths, to);
+        writeStrings(paths, to);
+        processStderr();
+        return readStorePaths<PathSet>(from);
+    }
+}
+
+
+PathSet RemoteStore::queryAllValidPaths()
+{
+    openConnection();
+    writeInt(wopQueryAllValidPaths, to);
     processStderr();
     return readStorePaths<PathSet>(from);
 }
 
 
-bool RemoteStore::hasSubstitutes(const Path & path)
+PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths)
 {
     openConnection();
-    writeInt(wopHasSubstitutes, to);
-    writeString(path, to);
-    processStderr();
-    unsigned int reply = readInt(from);
-    return reply != 0;
+    if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
+        PathSet res;
+        foreach (PathSet::const_iterator, i, paths) {
+            writeInt(wopHasSubstitutes, to);
+            writeString(*i, to);
+            processStderr();
+            if (readInt(from)) res.insert(*i);
+        }
+        return res;
+    } else {
+        writeInt(wopQuerySubstitutablePaths, to);
+        writeStrings(paths, to);
+        processStderr();
+        return readStorePaths<PathSet>(from);
+    }
 }
 
 
-bool RemoteStore::querySubstitutablePathInfo(const Path & path,
-    SubstitutablePathInfo & info)
+void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
+    SubstitutablePathInfos & infos)
 {
+    if (paths.empty()) return;
+
     openConnection();
-    if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return false;
-    writeInt(wopQuerySubstitutablePathInfo, to);
-    writeString(path, to);
-    processStderr();
-    unsigned int reply = readInt(from);
-    if (reply == 0) return false;
-    info.deriver = readString(from);
-    if (info.deriver != "") assertStorePath(info.deriver);
-    info.references = readStorePaths<PathSet>(from);
-    info.downloadSize = readLongLong(from);
-    info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0;
-    return true;
+    
+    if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return;
+    
+    if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
+        
+        foreach (PathSet::const_iterator, i, paths) {
+            SubstitutablePathInfo info;
+            writeInt(wopQuerySubstitutablePathInfo, to);
+            writeString(*i, to);
+            processStderr();
+            unsigned int reply = readInt(from);
+            if (reply == 0) continue;
+            info.deriver = readString(from);
+            if (info.deriver != "") assertStorePath(info.deriver);
+            info.references = readStorePaths<PathSet>(from);
+            info.downloadSize = readLongLong(from);
+            info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0;
+            infos[*i] = info;
+        }
+        
+    } else {
+        
+        writeInt(wopQuerySubstitutablePathInfos, to);
+        writeStrings(paths, to);
+        processStderr();
+        unsigned int count = readInt(from);
+        for (unsigned int n = 0; n < count; n++) {
+            Path path = readStorePath(from);
+            SubstitutablePathInfo & info(infos[path]);
+            info.deriver = readString(from);
+            if (info.deriver != "") assertStorePath(info.deriver);
+            info.references = readStorePaths<PathSet>(from);
+            info.downloadSize = readLongLong(from);
+            info.narSize = readLongLong(from);
+        }
+        
+    }
 }
 
 
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index f0e5dbf7695a..68db0640ae70 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -27,7 +27,9 @@ public:
     
     bool isValidPath(const Path & path);
 
-    PathSet queryValidPaths();
+    PathSet queryValidPaths(const PathSet & paths);
+    
+    PathSet queryAllValidPaths();
     
     ValidPathInfo queryPathInfo(const Path & path);
 
@@ -45,10 +47,10 @@ public:
 
     Path queryPathFromHashPart(const string & hashPart);
     
-    bool hasSubstitutes(const Path & path);
+    PathSet querySubstitutablePaths(const PathSet & paths);
     
-    bool querySubstitutablePathInfo(const Path & path,
-        SubstitutablePathInfo & info);
+    void querySubstitutablePathInfos(const PathSet & paths,
+        SubstitutablePathInfos & infos);
     
     Path addToStore(const Path & srcPath,
         bool recursive = true, HashType hashAlgo = htSHA256,
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 0ab15c38063e..5d8c09f5a526 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -86,6 +86,8 @@ struct SubstitutablePathInfo
     unsigned long long narSize; /* 0 = unknown */
 };
 
+typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos;
+
 
 struct ValidPathInfo 
 {
@@ -108,20 +110,23 @@ public:
 
     virtual ~StoreAPI() { }
 
-    /* Checks whether a path is valid. */ 
+    /* Check whether a path is valid. */ 
     virtual bool isValidPath(const Path & path) = 0;
 
-    /* Query the set of valid paths. */
-    virtual PathSet queryValidPaths() = 0;
+    /* Query which of the given paths is valid. */
+    virtual PathSet queryValidPaths(const PathSet & paths) = 0;
+
+    /* Query the set of all valid paths. */
+    virtual PathSet queryAllValidPaths() = 0;
 
     /* Query information about a valid path. */
     virtual ValidPathInfo queryPathInfo(const Path & path) = 0;
 
-    /* Queries the hash of a valid path. */ 
+    /* Query the hash of a valid path. */ 
     virtual Hash queryPathHash(const Path & path) = 0;
 
-    /* Queries the set of outgoing FS references for a store path.
-       The result is not cleared. */
+    /* Query the set of outgoing FS references for a store path.  The
+       result is not cleared. */
     virtual void queryReferences(const Path & path,
         PathSet & references) = 0;
 
@@ -144,13 +149,14 @@ public:
        path, or "" if the path doesn't exist. */
     virtual Path queryPathFromHashPart(const string & hashPart) = 0;
     
-    /* Query whether a path has substitutes. */
-    virtual bool hasSubstitutes(const Path & path) = 0;
-
-    /* Query the references, deriver and download size of a
-       substitutable path. */
-    virtual bool querySubstitutablePathInfo(const Path & path,
-        SubstitutablePathInfo & info) = 0;
+    /* Query which of the given paths have substitutes. */
+    virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0;
+
+    /* Query substitute info (i.e. references, derivers and download
+       sizes) of a set of paths.  If a path does not have substitute
+       info, it's omitted from the resulting ‘infos’ map. */
+    virtual void querySubstitutablePathInfos(const PathSet & paths,
+        SubstitutablePathInfos & infos) = 0;
     
     /* Copy the contents of a path to the store and register the
        validity the resulting path.  The resulting path is returned.
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index b08410fa1c42..d4ac0ea16fed 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -8,7 +8,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x10b
+#define PROTOCOL_VERSION 0x10c
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
@@ -34,13 +34,16 @@ typedef enum {
     wopCollectGarbage = 20,
     wopQuerySubstitutablePathInfo = 21,
     wopQueryDerivationOutputs = 22,
-    wopQueryValidPaths = 23,
+    wopQueryAllValidPaths = 23,
     wopQueryFailedPaths = 24,
     wopClearFailedPaths = 25,
     wopQueryPathInfo = 26,
     wopImportPaths = 27,
     wopQueryDerivationOutputNames = 28,
     wopQueryPathFromHashPart = 29,
+    wopQuerySubstitutablePathInfos = 30,
+    wopQueryValidPaths = 31,
+    wopQuerySubstitutablePaths = 32,
 } WorkerOp;
 
 
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 7aa6276e3a0a..f06f23dad523 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -211,9 +211,12 @@ static int comparePriorities(EvalState & state,
 
 static bool isPrebuilt(EvalState & state, const DrvInfo & elem)
 {
+    assert(false);
+#if 0
     return
         store->isValidPath(elem.queryOutPath(state)) ||
         store->hasSubstitutes(elem.queryOutPath(state));
+#endif
 }
 
 
@@ -929,6 +932,22 @@ static void opQuery(Globals & globals,
             installed.insert(i->queryOutPath(globals.state));
     }
 
+
+    /* Query which paths have substitutes. */
+    PathSet validPaths, substitutablePaths;
+    if (printStatus) {
+        PathSet paths;
+        foreach (vector<DrvInfo>::iterator, i, elems2)
+            try {
+                paths.insert(i->queryOutPath(globals.state));
+            } catch (AssertionError & e) {
+                printMsg(lvlTalkative, format("skipping derivation named `%1%' which gives an assertion failure") % i->name);
+                i->setFailed();
+            }
+        validPaths = store->queryValidPaths(paths);
+        substitutablePaths = store->querySubstitutablePaths(paths);
+    }
+
     
     /* Print the desired columns, or XML output. */
     Table table;
@@ -938,6 +957,8 @@ static void opQuery(Globals & globals,
     
     foreach (vector<DrvInfo>::iterator, i, elems2) {
         try {
+            if (i->hasFailed()) continue;
+            
             startNest(nest, lvlDebug, format("outputting query result `%1%'") % i->attrPath);
 
             if (globals.prebuiltOnly && !isPrebuilt(globals.state, *i)) continue;
@@ -949,9 +970,10 @@ static void opQuery(Globals & globals,
             XMLAttrs attrs;
 
             if (printStatus) {
-                bool hasSubs = store->hasSubstitutes(i->queryOutPath(globals.state));
-                bool isInstalled = installed.find(i->queryOutPath(globals.state)) != installed.end();
-                bool isValid = store->isValidPath(i->queryOutPath(globals.state));
+                Path outPath = i->queryOutPath(globals.state);
+                bool hasSubs = substitutablePaths.find(outPath) != substitutablePaths.end();
+                bool isInstalled = installed.find(outPath) != installed.end();
+                bool isValid = validPaths.find(outPath) != validPaths.end();
                 if (xmlOutput) {
                     attrs["installed"] = isInstalled ? "1" : "0";
                     attrs["valid"] = isValid ? "1" : "0";
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 23863525fe8a..47c76693f1dc 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -465,7 +465,7 @@ static void opDumpDB(Strings opFlags, Strings opArgs)
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (!opArgs.empty())
         throw UsageError("no arguments expected");
-    PathSet validPaths = store->queryValidPaths();
+    PathSet validPaths = store->queryAllValidPaths();
     foreach (PathSet::iterator, i, validPaths)
         cout << store->makeValidityRegistration(singleton<PathSet>(*i), true, true);
 }
diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc
index 74a619c71d0a..f2ca0a89233e 100644
--- a/src/nix-worker/nix-worker.cc
+++ b/src/nix-worker/nix-worker.cc
@@ -297,15 +297,33 @@ static void performOp(unsigned int clientVersion,
         break;
     }
 
+    case wopQueryValidPaths: {
+        PathSet paths = readStorePaths<PathSet>(from);
+        startWork();
+        PathSet res = store->queryValidPaths(paths);
+        stopWork();
+        writeStrings(res, to);
+        break;
+    }
+
     case wopHasSubstitutes: {
         Path path = readStorePath(from);
         startWork();
-        bool result = store->hasSubstitutes(path);
+        PathSet res = store->querySubstitutablePaths(singleton<PathSet>(path));
         stopWork();
-        writeInt(result, to);
+        writeInt(res.find(path) != res.end(), to);
         break;
     }
 
+    case wopQuerySubstitutablePaths: {
+        PathSet paths = readStorePaths<PathSet>(from);
+        startWork();
+        PathSet res = store->querySubstitutablePaths(paths);
+        stopWork();
+        writeStrings(res, to);
+        break;
+    }
+        
     case wopQueryPathHash: {
         Path path = readStorePath(from);
         startWork();
@@ -538,23 +556,43 @@ static void performOp(unsigned int clientVersion,
     case wopQuerySubstitutablePathInfo: {
         Path path = absPath(readString(from));
         startWork();
-        SubstitutablePathInfo info;
-        bool res = store->querySubstitutablePathInfo(path, info);
+        SubstitutablePathInfos infos;
+        store->querySubstitutablePathInfos(singleton<PathSet>(path), infos);
         stopWork();
-        writeInt(res ? 1 : 0, to);
-        if (res) {
-            writeString(info.deriver, to);
-            writeStrings(info.references, to);
-            writeLongLong(info.downloadSize, to);
+        SubstitutablePathInfos::iterator i = infos.find(path);
+        if (i == infos.end())
+            writeInt(0, to);
+        else {
+            writeInt(1, to);
+            writeString(i->second.deriver, to);
+            writeStrings(i->second.references, to);
+            writeLongLong(i->second.downloadSize, to);
             if (GET_PROTOCOL_MINOR(clientVersion) >= 7)
-                writeLongLong(info.narSize, to);
+                writeLongLong(i->second.narSize, to);
         }
         break;
     }
             
-    case wopQueryValidPaths: {
+    case wopQuerySubstitutablePathInfos: {
+        PathSet paths = readStorePaths<PathSet>(from);
+        startWork();
+        SubstitutablePathInfos infos;
+        store->querySubstitutablePathInfos(paths, infos);
+        stopWork();
+        writeInt(infos.size(), to);
+        foreach (SubstitutablePathInfos::iterator, i, infos) {
+            writeString(i->first, to);
+            writeString(i->second.deriver, to);
+            writeStrings(i->second.references, to);
+            writeLongLong(i->second.downloadSize, to);
+            writeLongLong(i->second.narSize, to);
+        }
+        break;
+    }
+            
+    case wopQueryAllValidPaths: {
         startWork();
-        PathSet paths = store->queryValidPaths();
+        PathSet paths = store->queryAllValidPaths();
         stopWork();
         writeStrings(paths, to);
         break;
generated by cgit-pink 1.4.1 (git 2.44.2) at 2024-12-04T07·14+0000