Implement RemoteStore::queryMissing()

This provides a significant speedup, e.g. 64 s -> 12 s for

  nix-build --dry-run -I nixpkgs=channel:nixos-16.03 '<nixpkgs/nixos/tests/misc.nix>' -A test

on a cold local and CloudFront cache.

The alternative is to use lots of concurrent daemon connections but
that seems wasteful.

5 files changed, 44 insertions, 3 deletions

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index a1f2db5b0ec8..c9c590787450 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -588,6 +588,31 @@ void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs)
 }
 
 
+void RemoteStore::queryMissing(const PathSet & targets,
+    PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
+    unsigned long long & downloadSize, unsigned long long & narSize)
+{
+    {
+        auto conn(connections->get());
+        if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19)
+            // Don't hold the connection handle in the fallback case
+            // to prevent a deadlock.
+            goto fallback;
+        conn->to << wopQueryMissing << targets;
+        conn->processStderr();
+        willBuild = readStorePaths<PathSet>(*this, conn->from);
+        willSubstitute = readStorePaths<PathSet>(*this, conn->from);
+        unknown = readStorePaths<PathSet>(*this, conn->from);
+        conn->from >> downloadSize >> narSize;
+        return;
+    }
+
+ fallback:
+    return Store::queryMissing(targets, willBuild, willSubstitute,
+        unknown, downloadSize, narSize);
+}
+
+
 RemoteStore::Connection::~Connection()
 {
     try {
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index a08bd305639d..db8da7eaa8c5 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -85,6 +85,10 @@ public:
 
     void addSignatures(const Path & storePath, const StringSet & sigs) override;
 
+    void queryMissing(const PathSet & targets,
+        PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
+        unsigned long long & downloadSize, unsigned long long & narSize) override;
+
 protected:
 
     struct Connection
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index e07dec495470..68c59a9f2922 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -524,7 +524,7 @@ public:
     /* Given a set of paths that are to be built, return the set of
        derivations that will be built, and the set of output paths
        that will be substituted. */
-    void queryMissing(const PathSet & targets,
+    virtual void queryMissing(const PathSet & targets,
         PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
         unsigned long long & downloadSize, unsigned long long & narSize);
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 6a4ed47cc9fa..6c6766b36124 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -6,7 +6,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x112
+#define PROTOCOL_VERSION 0x113
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
@@ -47,7 +47,8 @@ typedef enum {
     wopBuildDerivation = 36,
     wopAddSignatures = 37,
     wopNarFromPath = 38,
-    wopAddToStoreNar = 39
+    wopAddToStoreNar = 39,
+    wopQueryMissing = 40,
 } WorkerOp;
 
 
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index b6a46642c7c1..8786e2561b9c 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -592,6 +592,17 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         break;
     }
 
+    case wopQueryMissing: {
+        PathSet targets = readStorePaths<PathSet>(*store, from);
+        startWork();
+        PathSet willBuild, willSubstitute, unknown;
+        unsigned long long downloadSize, narSize;
+        store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
+        stopWork();
+        to << willBuild << willSubstitute << unknown << downloadSize << narSize;
+        break;
+    }
+
     default:
         throw Error(format("invalid operation %1%") % op);
     }