From 75989bdca773eedb8b8d1cc8a7675900358acd25 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 16 Sep 2016 18:54:14 +0200
Subject: Make computeFSClosure() single-threaded again

The fact that queryPathInfo() is synchronous meant that we needed a
thread for every concurrent binary cache lookup, even though they end
up being handled by the same download thread. Requiring hundreds of
threads is not a good idea. So now there is an asynchronous version of
queryPathInfo() that takes a callback function to process the
result. Similarly, enqueueDownload() now takes a callback rather than
returning a future.

Thus, a command like

  nix path-info --store https://cache.nixos.org/ -r /nix/store/slljrzwmpygy1daay14kjszsr9xix063-nixos-16.09beta231.dccf8c5

that returns 4941 paths now takes 1.87s using only 2 threads (the main
thread and the downloader thread). (This is with a prewarmed
CloudFront.)
---
 src/libstore/remote-store.cc | 64 +++++++++++++++++++++++---------------------
 1 file changed, 34 insertions(+), 30 deletions(-)

(limited to 'src/libstore/remote-store.cc')

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 94075f3b9b39..7b73557a5cae 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -246,36 +246,40 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
 }
 
 
-std::shared_ptr<ValidPathInfo> RemoteStore::queryPathInfoUncached(const Path & path)
-{
-    auto conn(connections->get());
-    conn->to << wopQueryPathInfo << path;
-    try {
-        conn->processStderr();
-    } catch (Error & e) {
-        // Ugly backwards compatibility hack.
-        if (e.msg().find("is not valid") != std::string::npos)
-            throw InvalidPath(e.what());
-        throw;
-    }
-    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
-        bool valid = readInt(conn->from) != 0;
-        if (!valid) throw InvalidPath(format("path ‘%s’ is not valid") % path);
-    }
-    auto info = std::make_shared<ValidPathInfo>();
-    info->path = path;
-    info->deriver = readString(conn->from);
-    if (info->deriver != "") assertStorePath(info->deriver);
-    info->narHash = parseHash(htSHA256, readString(conn->from));
-    info->references = readStorePaths<PathSet>(*this, conn->from);
-    info->registrationTime = readInt(conn->from);
-    info->narSize = readLongLong(conn->from);
-    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
-        info->ultimate = readInt(conn->from) != 0;
-        info->sigs = readStrings<StringSet>(conn->from);
-        info->ca = readString(conn->from);
-    }
-    return info;
+void RemoteStore::queryPathInfoUncached(const Path & path,
+    std::function<void(std::shared_ptr<ValidPathInfo>)> success,
+    std::function<void(std::exception_ptr exc)> failure)
+{
+    sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() {
+        auto conn(connections->get());
+        conn->to << wopQueryPathInfo << path;
+        try {
+            conn->processStderr();
+        } catch (Error & e) {
+            // Ugly backwards compatibility hack.
+            if (e.msg().find("is not valid") != std::string::npos)
+                throw InvalidPath(e.what());
+            throw;
+        }
+        if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
+            bool valid = readInt(conn->from) != 0;
+            if (!valid) throw InvalidPath(format("path ‘%s’ is not valid") % path);
+        }
+        auto info = std::make_shared<ValidPathInfo>();
+        info->path = path;
+        info->deriver = readString(conn->from);
+        if (info->deriver != "") assertStorePath(info->deriver);
+        info->narHash = parseHash(htSHA256, readString(conn->from));
+        info->references = readStorePaths<PathSet>(*this, conn->from);
+        info->registrationTime = readInt(conn->from);
+        info->narSize = readLongLong(conn->from);
+        if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
+            info->ultimate = readInt(conn->from) != 0;
+            info->sigs = readStrings<StringSet>(conn->from);
+            info->ca = readString(conn->from);
+        }
+        return info;
+    });
 }
 
 
-- 
cgit 1.4.1