Move path info caching from BinaryCacheStore to Store

Caching path info is generally useful. For instance, it speeds up "nix
path-info -rS /run/current-system" (i.e. showing the closure sizes of
all paths in the closure of the current system) from 5.6s to 0.15s.

This also eliminates some APIs like Store::queryDeriver() and
Store::queryReferences().

12 files changed, 232 insertions, 272 deletions

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 7016eedfa701..81800d4cb24e 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -40,11 +40,6 @@ void BinaryCacheStore::notImpl()
     throw Error("operation not implemented for binary cache stores");
 }
 
-const BinaryCacheStore::Stats & BinaryCacheStore::getStats()
-{
-    return stats;
-}
-
 Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
 {
     assertStorePath(storePath);
@@ -100,67 +95,15 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info,
 
     {
         auto state_(state.lock());
-        state_->narInfoCache.upsert(narInfo->path, narInfo);
-        stats.narInfoCacheSize = state_->narInfoCache.size();
+        state_->pathInfoCache.upsert(narInfo->path, std::shared_ptr<NarInfo>(narInfo));
+        stats.pathInfoCacheSize = state_->pathInfoCache.size();
     }
 
     stats.narInfoWrite++;
 }
 
-NarInfo BinaryCacheStore::readNarInfo(const Path & storePath)
+bool BinaryCacheStore::isValidPathUncached(const Path & storePath)
 {
-    {
-        auto state_(state.lock());
-        auto res = state_->narInfoCache.get(storePath);
-        if (res) {
-            stats.narInfoReadAverted++;
-            if (!*res)
-                throw InvalidPath(format("path ‘%s’ is not valid") % storePath);
-            return **res;
-        }
-    }
-
-    auto narInfoFile = narInfoFileFor(storePath);
-    auto data = getFile(narInfoFile);
-    if (!data) {
-        stats.narInfoMissing++;
-        auto state_(state.lock());
-        state_->narInfoCache.upsert(storePath, 0);
-        stats.narInfoCacheSize = state_->narInfoCache.size();
-        throw InvalidPath(format("path ‘%s’ is not valid") % storePath);
-    }
-
-    auto narInfo = make_ref<NarInfo>(*data, narInfoFile);
-    if (narInfo->path != storePath)
-        throw Error(format("NAR info file for store path ‘%1%’ does not match ‘%2%’") % narInfo->path % storePath);
-
-    stats.narInfoRead++;
-
-    if (publicKeys) {
-        if (!narInfo->checkSignatures(*publicKeys))
-            throw Error(format("no good signature on NAR info file ‘%1%’") % narInfoFile);
-    }
-
-    {
-        auto state_(state.lock());
-        state_->narInfoCache.upsert(storePath, narInfo);
-        stats.narInfoCacheSize = state_->narInfoCache.size();
-    }
-
-    return *narInfo;
-}
-
-bool BinaryCacheStore::isValidPath(const Path & storePath)
-{
-    {
-        auto state_(state.lock());
-        auto res = state_->narInfoCache.get(storePath);
-        if (res) {
-            stats.narInfoReadAverted++;
-            return *res != 0;
-        }
-    }
-
     // FIXME: this only checks whether a .narinfo with a matching hash
     // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even
     // though they shouldn't. Not easily fixed.
@@ -169,20 +112,20 @@ bool BinaryCacheStore::isValidPath(const Path & storePath)
 
 void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
 {
-    auto res = readNarInfo(storePath);
+    auto info = queryPathInfo(storePath).cast<const NarInfo>();
 
-    auto nar = getFile(res.url);
+    auto nar = getFile(info->url);
 
-    if (!nar) throw Error(format("file ‘%s’ missing from binary cache") % res.url);
+    if (!nar) throw Error(format("file ‘%s’ missing from binary cache") % info->url);
 
     stats.narRead++;
     stats.narReadCompressedBytes += nar->size();
 
     /* Decompress the NAR. FIXME: would be nice to have the remote
        side do this. */
-    if (res.compression == "none")
+    if (info->compression == "none")
         ;
-    else if (res.compression == "xz")
+    else if (info->compression == "xz")
         nar = decompressXZ(*nar);
     else
         throw Error(format("unknown NAR compression type ‘%1%’") % nar);
@@ -200,13 +143,13 @@ void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink
 {
     assert(!sign);
 
-    auto res = readNarInfo(storePath);
+    auto res = queryPathInfo(storePath);
 
     narFromPath(storePath, sink);
 
     // FIXME: check integrity of NAR.
 
-    sink << exportMagic << storePath << res.references << res.deriver << 0;
+    sink << exportMagic << storePath << res->references << res->deriver << 0;
 }
 
 Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source,
@@ -244,9 +187,24 @@ struct NopSink : ParseSink
 {
 };
 
-ValidPathInfo BinaryCacheStore::queryPathInfo(const Path & storePath)
+std::shared_ptr<ValidPathInfo> BinaryCacheStore::queryPathInfoUncached(const Path & storePath)
 {
-    return ValidPathInfo(readNarInfo(storePath));
+    auto narInfoFile = narInfoFileFor(storePath);
+    auto data = getFile(narInfoFile);
+    if (!data) return 0;
+
+    auto narInfo = make_ref<NarInfo>(*data, narInfoFile);
+    if (narInfo->path != storePath)
+        throw Error(format("NAR info file for store path ‘%1%’ does not match ‘%2%’") % narInfo->path % storePath);
+
+    stats.narInfoRead++;
+
+    if (publicKeys) {
+        if (!narInfo->checkSignatures(*publicKeys))
+            throw Error(format("no good signature on NAR info file ‘%1%’") % narInfoFile);
+    }
+
+    return std::shared_ptr<NarInfo>(narInfo);
 }
 
 void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths,
@@ -257,16 +215,16 @@ void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths,
     if (!localStore) return;
 
     for (auto & storePath : paths) {
-        if (!localStore->isValidPath(storePath)) {
+        try {
+            auto info = localStore->queryPathInfo(storePath);
+            SubstitutablePathInfo sub;
+            sub.references = info->references;
+            sub.downloadSize = 0;
+            sub.narSize = info->narSize;
+            infos.emplace(storePath, sub);
+        } catch (InvalidPath &) {
             left.insert(storePath);
-            continue;
         }
-        ValidPathInfo info = localStore->queryPathInfo(storePath);
-        SubstitutablePathInfo sub;
-        sub.references = info.references;
-        sub.downloadSize = 0;
-        sub.narSize = info.narSize;
-        infos.emplace(storePath, sub);
     }
 
     if (settings.useSubstitutes)
@@ -332,16 +290,16 @@ void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode)
         if (!localStore->isValidPath(storePath))
             localStore->ensurePath(storePath);
 
-        ValidPathInfo info = localStore->queryPathInfo(storePath);
+        auto info = localStore->queryPathInfo(storePath);
 
-        for (auto & ref : info.references)
+        for (auto & ref : info->references)
             if (ref != storePath)
                 ensurePath(ref);
 
         StringSink sink;
         dumpPath(storePath, sink);
 
-        addToCache(info, *sink.s);
+        addToCache(*info, *sink.s);
     }
 }
 
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 95e5d68b70ae..4e4346a43438 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -3,8 +3,6 @@
 #include "crypto.hh"
 #include "store-api.hh"
 
-#include "lru-cache.hh"
-#include "sync.hh"
 #include "pool.hh"
 
 #include <atomic>
@@ -22,13 +20,6 @@ private:
 
     std::shared_ptr<Store> localStore;
 
-    struct State
-    {
-        LRUCache<Path, std::shared_ptr<NarInfo>> narInfoCache{64 * 1024};
-    };
-
-    Sync<State> state;
-
 protected:
 
     BinaryCacheStore(std::shared_ptr<Store> localStore, const Path & secretKeyFile);
@@ -47,42 +38,17 @@ public:
 
     virtual void init();
 
-    struct Stats
-    {
-        std::atomic<uint64_t> narInfoRead{0};
-        std::atomic<uint64_t> narInfoReadAverted{0};
-        std::atomic<uint64_t> narInfoMissing{0};
-        std::atomic<uint64_t> narInfoWrite{0};
-        std::atomic<uint64_t> narInfoCacheSize{0};
-        std::atomic<uint64_t> narRead{0};
-        std::atomic<uint64_t> narReadBytes{0};
-        std::atomic<uint64_t> narReadCompressedBytes{0};
-        std::atomic<uint64_t> narWrite{0};
-        std::atomic<uint64_t> narWriteAverted{0};
-        std::atomic<uint64_t> narWriteBytes{0};
-        std::atomic<uint64_t> narWriteCompressedBytes{0};
-        std::atomic<uint64_t> narWriteCompressionTimeMs{0};
-    };
-
-    const Stats & getStats();
-
 private:
 
-    Stats stats;
-
     std::string narMagic;
 
     std::string narInfoFileFor(const Path & storePath);
 
     void addToCache(const ValidPathInfo & info, const string & nar);
 
-protected:
-
-    NarInfo readNarInfo(const Path & storePath);
-
 public:
 
-    bool isValidPath(const Path & path) override;
+    bool isValidPathUncached(const Path & path) override;
 
     PathSet queryValidPaths(const PathSet & paths) override
     { notImpl(); }
@@ -90,18 +56,12 @@ public:
     PathSet queryAllValidPaths() override
     { notImpl(); }
 
-    ValidPathInfo queryPathInfo(const Path & path) override;
-
-    Hash queryPathHash(const Path & path) override
-    { notImpl(); }
+    std::shared_ptr<ValidPathInfo> queryPathInfoUncached(const Path & path) override;
 
     void queryReferrers(const Path & path,
         PathSet & referrers) override
     { notImpl(); }
 
-    Path queryDeriver(const Path & path) override
-    { return ""; }
-
     PathSet queryValidDerivers(const Path & path) override
     { return {}; }
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 4b1c177fe27b..ae8078069d07 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2724,7 +2724,7 @@ void DerivationGoal::registerOutputs()
 
         if (buildMode == bmCheck) {
             if (!worker.store.isValidPath(path)) continue;
-            ValidPathInfo info = worker.store.queryPathInfo(path);
+            auto info = *worker.store.queryPathInfo(path);
             if (hash.first != info.narHash) {
                 if (settings.keepFailed) {
                     Path dst = path + checkSuffix;
@@ -3778,14 +3778,14 @@ bool Worker::pathContentsGood(const Path & path)
     std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path);
     if (i != pathContentsGoodCache.end()) return i->second;
     printMsg(lvlInfo, format("checking path ‘%1%’...") % path);
-    ValidPathInfo info = store.queryPathInfo(path);
+    auto info = store.queryPathInfo(path);
     bool res;
     if (!pathExists(path))
         res = false;
     else {
-        HashResult current = hashPath(info.narHash.type, path);
+        HashResult current = hashPath(info->narHash.type, path);
         Hash nullHash(htSHA256);
-        res = info.narHash == nullHash || info.narHash == current.first;
+        res = info->narHash == nullHash || info->narHash == current.first;
     }
     pathContentsGoodCache[path] = res;
     if (!res) printMsg(lvlError, format("path ‘%1%’ is corrupted or missing!") % path);
@@ -3881,7 +3881,7 @@ void LocalStore::repairPath(const Path & path)
     if (goal->getExitCode() != Goal::ecSuccess) {
         /* Since substituting the path didn't work, if we have a valid
            deriver, then rebuild the deriver. */
-        Path deriver = queryDeriver(path);
+        auto deriver = queryPathInfo(path)->deriver;
         if (deriver != "" && isValidPath(deriver)) {
             goals.clear();
             goals.insert(worker.makeDerivationGoal(deriver, StringSet(), bmRepair));
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 52afa1b14e03..13123838f3bf 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -407,7 +407,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
         queryReferrers(path, referrers);
         for (auto & i : referrers)
             if (i != path) deletePathRecursive(state, i);
-        size = queryPathInfo(path).narSize;
+        size = queryPathInfo(path)->narSize;
         invalidatePathChecked(path);
     }
 
@@ -485,7 +485,7 @@ bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & p
     if (state.gcKeepDerivations && isDerivation(path)) {
         PathSet outputs = queryDerivationOutputs(path);
         for (auto & i : outputs)
-            if (isValidPath(i) && queryDeriver(i) == path)
+            if (isValidPath(i) && queryPathInfo(i)->deriver == path)
                 incoming.insert(i);
     }
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index d6e1e4d7e0d4..cef2eb3f0fc7 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -577,6 +577,12 @@ uint64_t LocalStore::addValidPath(State & state,
         }
     }
 
+    {
+        auto state_(Store::state.lock());
+        state_->pathInfoCache.upsert(info.path, std::make_shared<ValidPathInfo>(info));
+        stats.pathInfoCacheSize = state_->pathInfoCache.size();
+    }
+
     return id;
 }
 
@@ -595,44 +601,44 @@ Hash parseHashField(const Path & path, const string & s)
 }
 
 
-ValidPathInfo LocalStore::queryPathInfo(const Path & path)
+std::shared_ptr<ValidPathInfo> LocalStore::queryPathInfoUncached(const Path & path)
 {
-    ValidPathInfo info;
-    info.path = path;
+    auto info = std::make_shared<ValidPathInfo>();
+    info->path = path;
 
     assertStorePath(path);
 
-    return retrySQLite<ValidPathInfo>([&]() {
+    return retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() {
         auto state(_state.lock());
 
         /* Get the path info. */
         auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path));
 
         if (!useQueryPathInfo.next())
-            throw Error(format("path ‘%1%’ is not valid") % path);
+            return std::shared_ptr<ValidPathInfo>();
 
-        info.id = useQueryPathInfo.getInt(0);
+        info->id = useQueryPathInfo.getInt(0);
 
-        info.narHash = parseHashField(path, useQueryPathInfo.getStr(1));
+        info->narHash = parseHashField(path, useQueryPathInfo.getStr(1));
 
-        info.registrationTime = useQueryPathInfo.getInt(2);
+        info->registrationTime = useQueryPathInfo.getInt(2);
 
         auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3);
-        if (s) info.deriver = s;
+        if (s) info->deriver = s;
 
         /* Note that narSize = NULL yields 0. */
-        info.narSize = useQueryPathInfo.getInt(4);
+        info->narSize = useQueryPathInfo.getInt(4);
 
-        info.ultimate = useQueryPathInfo.getInt(5) == 1;
+        info->ultimate = useQueryPathInfo.getInt(5) == 1;
 
         s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6);
-        if (s) info.sigs = tokenizeString<StringSet>(s, " ");
+        if (s) info->sigs = tokenizeString<StringSet>(s, " ");
 
         /* Get the references. */
-        auto useQueryReferences(state->stmtQueryReferences.use()(info.id));
+        auto useQueryReferences(state->stmtQueryReferences.use()(info->id));
 
         while (useQueryReferences.next())
-            info.references.insert(useQueryReferences.getStr(0));
+            info->references.insert(useQueryReferences.getStr(0));
 
         return info;
     });
@@ -661,17 +667,17 @@ uint64_t LocalStore::queryValidPathId(State & state, const Path & path)
 }
 
 
-bool LocalStore::isValidPath(State & state, const Path & path)
+bool LocalStore::isValidPath_(State & state, const Path & path)
 {
     return state.stmtQueryPathInfo.use()(path).next();
 }
 
 
-bool LocalStore::isValidPath(const Path & path)
+bool LocalStore::isValidPathUncached(const Path & path)
 {
     return retrySQLite<bool>([&]() {
         auto state(_state.lock());
-        return isValidPath(*state, path);
+        return isValidPath_(*state, path);
     });
 }
 
@@ -716,12 +722,6 @@ void LocalStore::queryReferrers(const Path & path, PathSet & referrers)
 }
 
 
-Path LocalStore::queryDeriver(const Path & path)
-{
-    return queryPathInfo(path).deriver;
-}
-
-
 PathSet LocalStore::queryValidDerivers(const Path & path)
 {
     assertStorePath(path);
@@ -996,12 +996,6 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
 }
 
 
-Hash LocalStore::queryPathHash(const Path & path)
-{
-    return queryPathInfo(path).narHash;
-}
-
-
 void LocalStore::registerValidPath(const ValidPathInfo & info)
 {
     ValidPathInfos infos;
@@ -1026,7 +1020,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
 
         for (auto & i : infos) {
             assert(i.narHash.type == htSHA256);
-            if (isValidPath(*state, i.path))
+            if (isValidPath_(*state, i.path))
                 updatePathInfo(*state, i);
             else
                 addValidPath(*state, i, false);
@@ -1071,6 +1065,12 @@ void LocalStore::invalidatePath(State & state, const Path & path)
 
     /* Note that the foreign key constraints on the Refs table take
        care of deleting the references entries for `path'. */
+
+    {
+        auto state_(Store::state.lock());
+        state_->pathInfoCache.erase(path);
+        stats.pathInfoCacheSize = state_->pathInfoCache.size();
+    }
 }
 
 
@@ -1225,8 +1225,7 @@ void LocalStore::exportPath(const Path & path, bool sign,
 
     printMsg(lvlTalkative, format("exporting path ‘%1%’") % path);
 
-    if (!isValidPath(path))
-        throw Error(format("path ‘%1%’ is not valid") % path);
+    auto info = queryPathInfo(path);
 
     HashAndWriteSink hashAndWriteSink(sink);
 
@@ -1236,15 +1235,11 @@ void LocalStore::exportPath(const Path & path, bool sign,
        filesystem corruption from spreading to other machines.
        Don't complain if the stored hash is zero (unknown). */
     Hash hash = hashAndWriteSink.currentHash();
-    Hash storedHash = queryPathHash(path);
-    if (hash != storedHash && storedHash != Hash(storedHash.type))
+    if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
         throw Error(format("hash of path ‘%1%’ has changed from ‘%2%’ to ‘%3%’!") % path
-            % printHash(storedHash) % printHash(hash));
-
-    PathSet references;
-    queryReferences(path, references);
+            % printHash(info->narHash) % printHash(hash));
 
-    hashAndWriteSink << exportMagic << path << references << queryDeriver(path);
+    hashAndWriteSink << exportMagic << path << info->references << info->deriver;
 
     if (sign) {
         Hash hash = hashAndWriteSink.currentHash();
@@ -1440,7 +1435,7 @@ void LocalStore::invalidatePathChecked(const Path & path)
 
         SQLiteTxn txn(state->db);
 
-        if (isValidPath(*state, path)) {
+        if (isValidPath_(*state, path)) {
             PathSet referrers; queryReferrers(*state, path, referrers);
             referrers.erase(path); /* ignore self-references */
             if (!referrers.empty())
@@ -1486,38 +1481,38 @@ bool LocalStore::verifyStore(bool checkContents, bool repair)
 
         for (auto & i : validPaths) {
             try {
-                ValidPathInfo info = queryPathInfo(i);
+                auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(i)));
 
                 /* Check the content hash (optionally - slow). */
                 printMsg(lvlTalkative, format("checking contents of ‘%1%’") % i);
-                HashResult current = hashPath(info.narHash.type, i);
+                HashResult current = hashPath(info->narHash.type, i);
 
-                if (info.narHash != nullHash && info.narHash != current.first) {
+                if (info->narHash != nullHash && info->narHash != current.first) {
                     printMsg(lvlError, format("path ‘%1%’ was modified! "
                             "expected hash ‘%2%’, got ‘%3%’")
-                        % i % printHash(info.narHash) % printHash(current.first));
+                        % i % printHash(info->narHash) % printHash(current.first));
                     if (repair) repairPath(i); else errors = true;
                 } else {
 
                     bool update = false;
 
                     /* Fill in missing hashes. */
-                    if (info.narHash == nullHash) {
+                    if (info->narHash == nullHash) {
                         printMsg(lvlError, format("fixing missing hash on ‘%1%’") % i);
-                        info.narHash = current.first;
+                        info->narHash = current.first;
                         update = true;
                     }
 
                     /* Fill in missing narSize fields (from old stores). */
-                    if (info.narSize == 0) {
+                    if (info->narSize == 0) {
                         printMsg(lvlError, format("updating size field on ‘%1%’ to %2%") % i % current.second);
-                        info.narSize = current.second;
+                        info->narSize = current.second;
                         update = true;
                     }
 
                     if (update) {
                         auto state(_state.lock());
-                        updatePathInfo(*state, info);
+                        updatePathInfo(*state, *info);
                     }
 
                 }
@@ -1656,11 +1651,11 @@ void LocalStore::addSignatures(const Path & storePath, const StringSet & sigs)
 
         SQLiteTxn txn(state->db);
 
-        auto info = queryPathInfo(storePath);
+        auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(storePath)));
 
-        info.sigs.insert(sigs.begin(), sigs.end());
+        info->sigs.insert(sigs.begin(), sigs.end());
 
-        updatePathInfo(*state, info);
+        updatePathInfo(*state, *info);
 
         txn.commit();
     });
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 14ff92c35cc5..daf394c92805 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -102,20 +102,16 @@ public:
 
     /* Implementations of abstract store API methods. */
 
-    bool isValidPath(const Path & path) override;
+    bool isValidPathUncached(const Path & path) override;
 
     PathSet queryValidPaths(const PathSet & paths) override;
 
     PathSet queryAllValidPaths() override;
 
-    ValidPathInfo queryPathInfo(const Path & path) override;
-
-    Hash queryPathHash(const Path & path) override;
+    std::shared_ptr<ValidPathInfo> queryPathInfoUncached(const Path & path) override;
 
     void queryReferrers(const Path & path, PathSet & referrers) override;
 
-    Path queryDeriver(const Path & path) override;
-
     PathSet queryValidDerivers(const Path & path) override;
 
     PathSet queryDerivationOutputs(const Path & path) override;
@@ -270,7 +266,7 @@ private:
     void optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash);
 
     // Internal versions that are not wrapped in retry_sqlite.
-    bool isValidPath(State & state, const Path & path);
+    bool isValidPath_(State & state, const Path & path);
     void queryReferrers(State & state, const Path & path, PathSet & referrers);
 
     /* Add signatures to a ValidPathInfo using the secret keys
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 12472f017ce4..5c284d1b9ab2 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -35,12 +35,13 @@ void Store::computeFSClosure(const Path & path,
         if (includeDerivers && isDerivation(path)) {
             PathSet outputs = queryDerivationOutputs(path);
             for (auto & i : outputs)
-                if (isValidPath(i) && queryDeriver(i) == path)
+                if (isValidPath(i) && queryPathInfo(i)->deriver == path)
                     edges.insert(i);
         }
 
     } else {
-        queryReferences(path, edges);
+        auto info = queryPathInfo(path);
+        edges = info->references;
 
         if (includeOutputs && isDerivation(path)) {
             PathSet outputs = queryDerivationOutputs(path);
@@ -48,10 +49,8 @@ void Store::computeFSClosure(const Path & path,
                 if (isValidPath(i)) edges.insert(i);
         }
 
-        if (includeDerivers) {
-            Path deriver = queryDeriver(path);
-            if (isValidPath(deriver)) edges.insert(deriver);
-        }
+        if (includeDerivers && isValidPath(info->deriver))
+            edges.insert(info->deriver);
     }
 
     for (auto & i : edges)
@@ -189,8 +188,10 @@ Paths Store::topoSortPaths(const PathSet & paths)
         parents.insert(path);
 
         PathSet references;
-        if (isValidPath(path))
-            queryReferences(path, references);
+        try {
+            references = queryPathInfo(path)->references;
+        } catch (InvalidPath &) {
+        }
 
         for (auto & i : references)
             /* Don't traverse into paths that don't exist.  That can
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 761e835481a8..55196397667a 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -141,7 +141,7 @@ void RemoteStore::setOptions(ref<Connection> conn)
 }
 
 
-bool RemoteStore::isValidPath(const Path & path)
+bool RemoteStore::isValidPathUncached(const Path & path)
 {
     auto conn(connections->get());
     conn->to << wopIsValidPath << path;
@@ -239,48 +239,27 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
 }
 
 
-ValidPathInfo RemoteStore::queryPathInfo(const Path & path)
+std::shared_ptr<ValidPathInfo> RemoteStore::queryPathInfoUncached(const Path & path)
 {
     auto conn(connections->get());
     conn->to << wopQueryPathInfo << path;
     conn->processStderr();
-    ValidPathInfo info;
-    info.path = path;
-    info.deriver = readString(conn->from);
-    if (info.deriver != "") assertStorePath(info.deriver);
-    info.narHash = parseHash(htSHA256, readString(conn->from));
-    info.references = readStorePaths<PathSet>(conn->from);
-    info.registrationTime = readInt(conn->from);
-    info.narSize = readLongLong(conn->from);
+    auto info = std::make_shared<ValidPathInfo>();
+    info->path = path;
+    info->deriver = readString(conn->from);
+    if (info->deriver != "") assertStorePath(info->deriver);
+    info->narHash = parseHash(htSHA256, readString(conn->from));
+    info->references = readStorePaths<PathSet>(conn->from);
+    info->registrationTime = readInt(conn->from);
+    info->narSize = readLongLong(conn->from);
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
-        info.ultimate = readInt(conn->from) != 0;
-        info.sigs = readStrings<StringSet>(conn->from);
+        info->ultimate = readInt(conn->from) != 0;
+        info->sigs = readStrings<StringSet>(conn->from);
     }
     return info;
 }
 
 
-Hash RemoteStore::queryPathHash(const Path & path)
-{
-    auto conn(connections->get());
-    conn->to << wopQueryPathHash << path;
-    conn->processStderr();
-    string hash = readString(conn->from);
-    return parseHash(htSHA256, hash);
-}
-
-
-void RemoteStore::queryReferences(const Path & path,
-    PathSet & references)
-{
-    auto conn(connections->get());
-    conn->to << wopQueryReferences << path;
-    conn->processStderr();
-    PathSet references2 = readStorePaths<PathSet>(conn->from);
-    references.insert(references2.begin(), references2.end());
-}
-
-
 void RemoteStore::queryReferrers(const Path & path,
     PathSet & referrers)
 {
@@ -292,17 +271,6 @@ void RemoteStore::queryReferrers(const Path & path,
 }
 
 
-Path RemoteStore::queryDeriver(const Path & path)
-{
-    auto conn(connections->get());
-    conn->to << wopQueryDeriver << path;
-    conn->processStderr();
-    Path drvPath = readString(conn->from);
-    if (drvPath != "") assertStorePath(drvPath);
-    return drvPath;
-}
-
-
 PathSet RemoteStore::queryValidDerivers(const Path & path)
 {
     auto conn(connections->get());
@@ -517,6 +485,12 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
     results.paths = readStrings<PathSet>(conn->from);
     results.bytesFreed = readLongLong(conn->from);
     readLongLong(conn->from); // obsolete
+
+    {
+        auto state_(Store::state.lock());
+        state_->pathInfoCache.clear();
+        stats.pathInfoCacheSize = 0;
+    }
 }
 
 
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 45bc41804ccf..4ea8e8a5be4d 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -26,22 +26,16 @@ public:
 
     /* Implementations of abstract store API methods. */
 
-    bool isValidPath(const Path & path) override;
+    bool isValidPathUncached(const Path & path) override;
 
     PathSet queryValidPaths(const PathSet & paths) override;
 
     PathSet queryAllValidPaths() override;
 
-    ValidPathInfo queryPathInfo(const Path & path) override;
-
-    Hash queryPathHash(const Path & path) override;
-
-    void queryReferences(const Path & path, PathSet & references) override;
+    std::shared_ptr<ValidPathInfo> queryPathInfoUncached(const Path & path) override;
 
     void queryReferrers(const Path & path, PathSet & referrers) override;
 
-    Path queryDeriver(const Path & path) override;
-
     PathSet queryValidDerivers(const Path & path) override;
 
     PathSet queryDerivationOutputs(const Path & path) override;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index cc91ed287768..6543ed1f6d19 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -225,10 +225,48 @@ Path computeStorePathForText(const string & name, const string & s,
 }
 
 
-void Store::queryReferences(const Path & path, PathSet & references)
+bool Store::isValidPath(const Path & storePath)
 {
-    ValidPathInfo info = queryPathInfo(path);
-    references.insert(info.references.begin(), info.references.end());
+    {
+        auto state_(state.lock());
+        auto res = state_->pathInfoCache.get(storePath);
+        if (res) {
+            stats.narInfoReadAverted++;
+            return *res != 0;
+        }
+    }
+
+    return isValidPathUncached(storePath);
+}
+
+
+ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
+{
+    {
+        auto state_(state.lock());
+        auto res = state_->pathInfoCache.get(storePath);
+        if (res) {
+            stats.narInfoReadAverted++;
+            if (!*res)
+                throw InvalidPath(format("path ‘%s’ is not valid") % storePath);
+            return ref<ValidPathInfo>(*res);
+        }
+    }
+
+    auto info = queryPathInfoUncached(storePath);
+
+    {
+        auto state_(state.lock());
+        state_->pathInfoCache.upsert(storePath, info);
+        stats.pathInfoCacheSize = state_->pathInfoCache.size();
+    }
+
+    if (!info) {
+        stats.narInfoMissing++;
+        throw InvalidPath(format("path ‘%s’ is not valid") % storePath);
+    }
+
+    return ref<ValidPathInfo>(info);
 }
 
 
@@ -243,19 +281,19 @@ string Store::makeValidityRegistration(const PathSet & paths,
     for (auto & i : paths) {
         s += i + "\n";
 
-        ValidPathInfo info = queryPathInfo(i);
+        auto info = queryPathInfo(i);
 
         if (showHash) {
-            s += printHash(info.narHash) + "\n";
-            s += (format("%1%\n") % info.narSize).str();
+            s += printHash(info->narHash) + "\n";
+            s += (format("%1%\n") % info->narSize).str();
         }
 
-        Path deriver = showDerivers ? info.deriver : "";
+        Path deriver = showDerivers ? info->deriver : "";
         s += deriver + "\n";
 
-        s += (format("%1%\n") % info.references.size()).str();
+        s += (format("%1%\n") % info->references.size()).str();
 
-        for (auto & j : info.references)
+        for (auto & j : info->references)
             s += j + "\n";
     }
 
@@ -263,6 +301,12 @@ string Store::makeValidityRegistration(const PathSet & paths,
 }
 
 
+const Store::Stats & Store::getStats()
+{
+    return stats;
+}
+
+
 ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
 {
     ValidPathInfo info;
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 8827654fafdd..d45e401c391a 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -3,11 +3,14 @@
 #include "hash.hh"
 #include "serialise.hh"
 #include "crypto.hh"
+#include "lru-cache.hh"
+#include "sync.hh"
 
-#include <string>
+#include <atomic>
 #include <limits>
 #include <map>
 #include <memory>
+#include <string>
 
 
 namespace nix {
@@ -130,6 +133,8 @@ struct ValidPathInfo
 
     /* Verify a single signature. */
     bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
+
+    virtual ~ValidPathInfo() { }
 };
 
 typedef list<ValidPathInfo> ValidPathInfos;
@@ -169,12 +174,27 @@ class FSAccessor;
 
 class Store : public std::enable_shared_from_this<Store>
 {
+protected:
+
+    struct State
+    {
+        LRUCache<Path, std::shared_ptr<ValidPathInfo>> pathInfoCache{64 * 1024};
+    };
+
+    Sync<State> state;
+
 public:
 
     virtual ~Store() { }
 
     /* Check whether a path is valid. */
-    virtual bool isValidPath(const Path & path) = 0;
+    bool isValidPath(const Path & path);
+
+protected:
+
+    virtual bool isValidPathUncached(const Path & path) = 0;
+
+public:
 
     /* Query which of the given paths is valid. */
     virtual PathSet queryValidPaths(const PathSet & paths) = 0;
@@ -183,24 +203,19 @@ public:
     virtual PathSet queryAllValidPaths() = 0;
 
     /* Query information about a valid path. */
-    virtual ValidPathInfo queryPathInfo(const Path & path) = 0;
+    ref<const ValidPathInfo> queryPathInfo(const Path & path);
+
+protected:
 
-    /* Query the hash of a valid path. */
-    virtual Hash queryPathHash(const Path & path) = 0;
+    virtual std::shared_ptr<ValidPathInfo> queryPathInfoUncached(const Path & path) = 0;
 
-    /* Query the set of outgoing FS references for a store path. The
-       result is not cleared. */
-    virtual void queryReferences(const Path & path, PathSet & references);
+public:
 
     /* Queries the set of incoming FS references for a store path.
        The result is not cleared. */
     virtual void queryReferrers(const Path & path,
         PathSet & referrers) = 0;
 
-    /* Query the deriver of a store path.  Return the empty string if
-       no deriver has been set. */
-    virtual Path queryDeriver(const Path & path) = 0;
-
     /* Return all currently valid derivations that have `path' as an
        output.  (Note that the result of `queryDeriver()' is the
        derivation that was actually used to produce `path', which may
@@ -373,6 +388,29 @@ public:
        relation.  If p refers to q, then p preceeds q in this list. */
     Paths topoSortPaths(const PathSet & paths);
 
+    struct Stats
+    {
+        std::atomic<uint64_t> narInfoRead{0};
+        std::atomic<uint64_t> narInfoReadAverted{0};
+        std::atomic<uint64_t> narInfoMissing{0};
+        std::atomic<uint64_t> narInfoWrite{0};
+        std::atomic<uint64_t> pathInfoCacheSize{0};
+        std::atomic<uint64_t> narRead{0};
+        std::atomic<uint64_t> narReadBytes{0};
+        std::atomic<uint64_t> narReadCompressedBytes{0};
+        std::atomic<uint64_t> narWrite{0};
+        std::atomic<uint64_t> narWriteAverted{0};
+        std::atomic<uint64_t> narWriteBytes{0};
+        std::atomic<uint64_t> narWriteCompressedBytes{0};
+        std::atomic<uint64_t> narWriteCompressionTimeMs{0};
+    };
+
+    const Stats & getStats();
+
+protected:
+
+    Stats stats;
+
 };
 
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index c10598d5d301..d62244d18c12 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -14,8 +14,8 @@ namespace nix {
 typedef enum {
     wopIsValidPath = 1,
     wopHasSubstitutes = 3,
-    wopQueryPathHash = 4,
-    wopQueryReferences = 5,
+    wopQueryPathHash = 4, // obsolete
+    wopQueryReferences = 5, // obsolete
     wopQueryReferrers = 6,
     wopAddToStore = 7,
     wopAddTextToStore = 8,
@@ -26,7 +26,7 @@ typedef enum {
     wopSyncWithGC = 13,
     wopFindRoots = 14,
     wopExportPath = 16,
-    wopQueryDeriver = 18,
+    wopQueryDeriver = 18, // obsolete
     wopSetOptions = 19,
     wopCollectGarbage = 20,
     wopQuerySubstitutablePathInfo = 21,