about summary refs log tree commit diff
path: root/src/libstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/libstore')
-rw-r--r--src/libstore/build.cc10
-rw-r--r--src/libstore/globals.cc5
-rw-r--r--src/libstore/local-store.cc60
-rw-r--r--src/libstore/local-store.hh7
-rw-r--r--src/libstore/misc.cc106
-rw-r--r--src/libstore/remote-store.cc13
-rw-r--r--src/libstore/remote-store.hh3
-rw-r--r--src/libstore/store-api.hh5
8 files changed, 154 insertions, 55 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index d5bbd540b34d..1c84e5b9f92d 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2352,10 +2352,12 @@ void SubstitutionGoal::tryNext()
     sub = subs.front();
     subs.pop_front();
 
-    if (!worker.store.querySubstitutablePathInfo(sub, storePath, info)) {
-        tryNext();
-        return;
-    }
+    SubstitutablePathInfos infos;
+    PathSet dummy(singleton<PathSet>(storePath));
+    worker.store.querySubstitutablePathInfos(sub, dummy, infos);
+    SubstitutablePathInfos::iterator k = infos.find(storePath);
+    if (k == infos.end()) { tryNext(); return; }
+    info = k->second;
 
     /* To maintain the closure invariant, we first have to realise the
        paths referenced by this one. */
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 5c22f1406649..f660ed68df0d 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -155,8 +155,9 @@ void setDefaultsFromEnvironment()
 
     string subs = getEnv("NIX_SUBSTITUTERS", "default");
     if (subs == "default") {
-        substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
-        substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
+        //substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
+        //substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
+        substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl");
     } else
         substituters = tokenizeString(subs, ":");
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 1ce62aeafcef..b4ea4b748178 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -936,37 +936,57 @@ bool LocalStore::hasSubstitutes(const Path & path)
 }
 
 
-bool LocalStore::querySubstitutablePathInfo(const Path & substituter,
-    const Path & path, SubstitutablePathInfo & info)
+void LocalStore::querySubstitutablePathInfos(const Path & substituter,
+    PathSet & paths, SubstitutablePathInfos & infos)
 {
     RunningSubstituter & run(runningSubstituters[substituter]);
     startSubstituter(substituter, run);
 
-    writeLine(run.to, "info\n" + path);
+    string s = "info ";
+    foreach (PathSet::const_iterator, i, paths)
+        if (infos.find(*i) == infos.end()) { s += *i; s += " "; }
+    writeLine(run.to, s);
 
-    if (!getIntLine<int>(run.from)) return false;
-    
-    info.deriver = readLine(run.from);
-    if (info.deriver != "") assertStorePath(info.deriver);
-    int nrRefs = getIntLine<int>(run.from);
-    while (nrRefs--) {
-        Path p = readLine(run.from);
-        assertStorePath(p);
-        info.references.insert(p);
+    while (true) {
+        Path path = readLine(run.from);
+        if (path == "") break;
+        assert(paths.find(path) != paths.end());
+        paths.erase(path);
+        SubstitutablePathInfo & info(infos[path]);
+        info.deriver = readLine(run.from);
+        if (info.deriver != "") assertStorePath(info.deriver);
+        int nrRefs = getIntLine<int>(run.from);
+        while (nrRefs--) {
+            Path p = readLine(run.from);
+            assertStorePath(p);
+            info.references.insert(p);
+        }
+        info.downloadSize = getIntLine<long long>(run.from);
+        info.narSize = getIntLine<long long>(run.from);
     }
-    info.downloadSize = getIntLine<long long>(run.from);
-    info.narSize = getIntLine<long long>(run.from);
-    
-    return true;
 }
 
 
 bool LocalStore::querySubstitutablePathInfo(const Path & path,
     SubstitutablePathInfo & info)
 {
-    foreach (Paths::iterator, i, substituters)
-        if (querySubstitutablePathInfo(*i, path, info)) return true;
-    return false;
+    SubstitutablePathInfos infos;
+    querySubstitutablePathInfos(singleton<PathSet>(path), infos);
+    SubstitutablePathInfos::iterator i = infos.find(path);
+    if (i == infos.end()) return false;
+    info = i->second;
+    return true;
+}
+
+
+void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
+    SubstitutablePathInfos & infos)
+{
+    PathSet todo = paths;
+    foreach (Paths::iterator, i, substituters) {
+        if (todo.empty()) break;
+        querySubstitutablePathInfos(*i, todo, infos);
+    }
 }
 
 
@@ -1110,7 +1130,7 @@ Path LocalStore::addToStore(const Path & _srcPath,
        method for very large paths, but `copyPath' is mainly used for
        small files. */
     StringSink sink;
-    if (recursive) 
+    if (recursive)
         dumpPath(srcPath, sink, filter);
     else
         sink.s = readFile(srcPath);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index aa8e8582fb0d..c4d8be692a36 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -128,8 +128,11 @@ public:
     bool querySubstitutablePathInfo(const Path & path,
         SubstitutablePathInfo & info);
     
-    bool querySubstitutablePathInfo(const Path & substituter,
-        const Path & path, SubstitutablePathInfo & info);
+    void querySubstitutablePathInfos(const Path & substituter,
+        PathSet & paths, SubstitutablePathInfos & infos);
+    
+    void querySubstitutablePathInfos(const PathSet & paths,
+        SubstitutablePathInfos & infos);
     
     Path addToStore(const Path & srcPath,
         bool recursive = true, HashType hashAlgo = htSHA256,
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 093499936349..aa5f6ff727c9 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -55,45 +55,97 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
     
     PathSet todo(targets.begin(), targets.end()), done;
 
+    bool useSubstitutes = queryBoolSetting("build-use-substitutes", true);
+
+    /* Getting substitute info has high latency when using the binary
+       cache substituter.  Thus it's essential to do substitute
+       queries in parallel as much as possible.  To accomplish this
+       we do the following:
+
+       - For all paths still to be processed (‘todo’), we add all
+         paths for which we need info to the set ‘query’.  For an
+         unbuilt derivation this is the output paths; otherwise, it's
+         the path itself.
+
+       - We get info about all paths in ‘query’ in parallel.
+
+       - We process the results and add new items to ‘todo’ if
+         necessary.  E.g. if a path is substitutable, then we need to
+         get info on its references.
+
+       - Repeat until ‘todo’ is empty.
+    */
+
     while (!todo.empty()) {
-        Path p = *(todo.begin());
-        todo.erase(p);
-        if (done.find(p) != done.end()) continue;
-        done.insert(p);
-
-        if (isDerivation(p)) {
-            if (!store.isValidPath(p)) {
-                unknown.insert(p);
-                continue;
+              
+        PathSet query, todoDrv, todoNonDrv;
+
+        foreach (PathSet::iterator, i, todo) {
+            if (done.find(*i) != done.end()) continue;
+            done.insert(*i);
+
+            if (isDerivation(*i)) {
+                if (!store.isValidPath(*i)) {
+                    // FIXME: we could try to substitute p.
+                    unknown.insert(*i);
+                    continue;
+                }
+                Derivation drv = derivationFromPath(store, *i);
+
+                PathSet invalid;
+                foreach (DerivationOutputs::iterator, j, drv.outputs)
+                    if (!store.isValidPath(j->second.path)) invalid.insert(j->second.path);
+                if (invalid.empty()) continue;
+                
+                todoDrv.insert(*i);
+                if (useSubstitutes) query.insert(invalid.begin(), invalid.end());
+            }
+
+            else {
+                if (store.isValidPath(*i)) continue;
+                query.insert(*i);
+                todoNonDrv.insert(*i);
             }
-            Derivation drv = derivationFromPath(store, p);
+        }
+
+        todo.clear();
+        
+        SubstitutablePathInfos infos;
+        store.querySubstitutablePathInfos(query, infos);
+
+        foreach (PathSet::iterator, i, todoDrv) {
+            // FIXME: cache this
+            Derivation drv = derivationFromPath(store, *i);
 
             bool mustBuild = false;
-            foreach (DerivationOutputs::iterator, i, drv.outputs)
-                if (!store.isValidPath(i->second.path) &&
-                    !(queryBoolSetting("build-use-substitutes", true) && store.hasSubstitutes(i->second.path)))
-                    mustBuild = true;
+            if (useSubstitutes) {
+                foreach (DerivationOutputs::iterator, j, drv.outputs)
+                    if (!store.isValidPath(j->second.path) &&
+                        infos.find(j->second.path) == infos.end())
+                        mustBuild = true;
+            } else
+                mustBuild = true;
 
             if (mustBuild) {
-                willBuild.insert(p);
+                willBuild.insert(*i);
                 todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
                 foreach (DerivationInputs::iterator, i, drv.inputDrvs)
                     todo.insert(i->first);
-            } else 
+            } else
                 foreach (DerivationOutputs::iterator, i, drv.outputs)
-                    todo.insert(i->second.path);
+                    todoNonDrv.insert(i->second.path);
         }
-
-        else {
-            if (store.isValidPath(p)) continue;
-            SubstitutablePathInfo info;
-            if (store.querySubstitutablePathInfo(p, info)) {
-                willSubstitute.insert(p);
-                downloadSize += info.downloadSize;
-                narSize += info.narSize;
-                todo.insert(info.references.begin(), info.references.end());
+        
+        foreach (PathSet::iterator, i, todoNonDrv) {
+            done.insert(*i);
+            SubstitutablePathInfos::iterator info = infos.find(*i);
+            if (info != infos.end()) {
+                willSubstitute.insert(*i);
+                downloadSize += info->second.downloadSize;
+                narSize += info->second.narSize;
+                todo.insert(info->second.references.begin(), info->second.references.end());
             } else
-                unknown.insert(p);
+                unknown.insert(*i);
         }
     }
 }
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 5e5561a6aecf..1cf67d3731d1 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -256,6 +256,19 @@ bool RemoteStore::querySubstitutablePathInfo(const Path & path,
 }
 
 
+void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
+    SubstitutablePathInfos & infos)
+{
+    if (paths.empty()) return;
+    printMsg(lvlError, format("QUERYING %1% (REMOTE)") % showPaths(paths));
+    foreach (PathSet::const_iterator, i, paths) {
+        SubstitutablePathInfo info;
+        if (querySubstitutablePathInfo(*i, info))
+            infos[*i] = info;
+    }
+}
+
+
 ValidPathInfo RemoteStore::queryPathInfo(const Path & path)
 {
     openConnection();
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index e9f40da6dbf4..1056a6115849 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -48,6 +48,9 @@ public:
     bool querySubstitutablePathInfo(const Path & path,
         SubstitutablePathInfo & info);
     
+    void querySubstitutablePathInfos(const PathSet & paths,
+        SubstitutablePathInfos & infos);
+    
     Path addToStore(const Path & srcPath,
         bool recursive = true, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter);
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index bf3269f57818..92b2ddb1e730 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -86,6 +86,8 @@ struct SubstitutablePathInfo
     unsigned long long narSize; /* 0 = unknown */
 };
 
+typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos;
+
 
 struct ValidPathInfo 
 {
@@ -147,6 +149,9 @@ public:
        substitutable path. */
     virtual bool querySubstitutablePathInfo(const Path & path,
         SubstitutablePathInfo & info) = 0;
+
+    virtual void querySubstitutablePathInfos(const PathSet & paths,
+        SubstitutablePathInfos & infos) = 0;
     
     /* Copy the contents of a path to the store and register the
        validity the resulting path.  The resulting path is returned.
generated by cgit-pink 1.4.1 (git 2.44.2) at 2024-11-01T14·09+0000