Cache path info lookups in SQLite

This re-implements the binary cache database in C++, allowing it to be
used by other Store backends, in particular the S3 backend.

14 files changed, 328 insertions, 29 deletions

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 81800d4cb24e..3857ed93e212 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -59,7 +59,7 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info,
     narInfo->narSize = nar.size();
     narInfo->narHash = hashString(htSHA256, nar);
 
-    if (info.narHash.type != htUnknown && info.narHash != narInfo->narHash)
+    if (info.narHash && info.narHash != narInfo->narHash)
         throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path);
 
     /* Compress the NAR. */
@@ -96,7 +96,6 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info,
     {
         auto state_(state.lock());
         state_->pathInfoCache.upsert(narInfo->path, std::shared_ptr<NarInfo>(narInfo));
-        stats.pathInfoCacheSize = state_->pathInfoCache.size();
     }
 
     stats.narInfoWrite++;
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index d9b009d40322..becf8524546c 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -290,7 +290,7 @@ Hash hashDerivationModulo(Store & store, Derivation drv)
     DerivationInputs inputs2;
     for (auto & i : drv.inputDrvs) {
         Hash h = drvHashes[i.first];
-        if (h.type == htUnknown) {
+        if (!h) {
             assert(store.isValidPath(i.first));
             Derivation drv2 = readDerivation(i.first);
             h = hashDerivationModulo(store, drv2);
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 8cd3ad741e10..eed630517d11 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -225,7 +225,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
 {
     auto url = resolveUri(url_);
 
-    Path cacheDir = getEnv("XDG_CACHE_HOME", getEnv("HOME", "") + "/.cache") + "/nix/tarballs";
+    Path cacheDir = getCacheDir() + "/nix/tarballs";
     createDirs(cacheDir);
 
     string urlHash = printHash32(hashString(htSHA256, url));
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 6dcea1cbf090..771eb42ee561 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -1,6 +1,7 @@
 #include "binary-cache-store.hh"
 #include "download.hh"
 #include "globals.hh"
+#include "nar-info-disk-cache.hh"
 
 namespace nix {
 
@@ -24,13 +25,23 @@ public:
     {
         if (cacheUri.back() == '/')
             cacheUri.pop_back();
+
+        diskCache = getNarInfoDiskCache();
+    }
+
+    std::string getUri() override
+    {
+        return cacheUri;
     }
 
     void init() override
     {
         // FIXME: do this lazily?
-        if (!fileExists("nix-cache-info"))
-            throw Error(format("‘%s’ does not appear to be a binary cache") % cacheUri);
+        if (!diskCache->cacheExists(cacheUri)) {
+            if (!fileExists("nix-cache-info"))
+                throw Error(format("‘%s’ does not appear to be a binary cache") % cacheUri);
+            diskCache->createCache(cacheUri);
+        }
     }
 
 protected:
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index cef2eb3f0fc7..9bc164e19f93 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -580,7 +580,6 @@ uint64_t LocalStore::addValidPath(State & state,
     {
         auto state_(Store::state.lock());
         state_->pathInfoCache.upsert(info.path, std::make_shared<ValidPathInfo>(info));
-        stats.pathInfoCacheSize = state_->pathInfoCache.size();
     }
 
     return id;
@@ -1069,7 +1068,6 @@ void LocalStore::invalidatePath(State & state, const Path & path)
     {
         auto state_(Store::state.lock());
         state_->pathInfoCache.erase(path);
-        stats.pathInfoCacheSize = state_->pathInfoCache.size();
     }
 }
 
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
new file mode 100644
index 000000000000..30ef7b36c9dd
--- /dev/null
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -0,0 +1,217 @@
+#include "nar-info-disk-cache.hh"
+#include "sync.hh"
+#include "sqlite.hh"
+#include "globals.hh"
+
+#include <sqlite3.h>
+
+namespace nix {
+
+static const char * schema = R"sql(
+
+create table if not exists BinaryCaches (
+    id        integer primary key autoincrement not null,
+    url       text unique not null,
+    timestamp integer not null,
+    storeDir  text not null,
+    wantMassQuery integer not null,
+    priority  integer not null
+);
+
+create table if not exists NARs (
+    cache            integer not null,
+    storePath        text not null,
+    url              text,
+    compression      text,
+    fileHash         text,
+    fileSize         integer,
+    narHash          text,
+    narSize          integer,
+    refs             text,
+    deriver          text,
+    sigs             text,
+    timestamp        integer not null,
+    primary key (cache, storePath),
+    foreign key (cache) references BinaryCaches(id) on delete cascade
+);
+
+create table if not exists NARExistence (
+    cache            integer not null,
+    storePath        text not null,
+    exist            integer not null,
+    timestamp        integer not null,
+    primary key (cache, storePath),
+    foreign key (cache) references BinaryCaches(id) on delete cascade
+);
+
+)sql";
+
+class NarInfoDiskCacheImpl : public NarInfoDiskCache
+{
+public:
+
+    /* How long negative lookups are valid. */
+    const int ttlNegative = 3600;
+
+    struct State
+    {
+        SQLite db;
+        SQLiteStmt insertCache, queryCache, insertNAR, queryNAR, insertNARExistence, queryNARExistence;
+        std::map<std::string, int> caches;
+    };
+
+    Sync<State> _state;
+
+    NarInfoDiskCacheImpl()
+    {
+        auto state(_state.lock());
+
+        Path dbPath = getCacheDir() + "/nix/binary-cache-v3.sqlite";
+        createDirs(dirOf(dbPath));
+
+        if (sqlite3_open_v2(dbPath.c_str(), &state->db.db,
+                SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK)
+            throw Error(format("cannot open store cache ‘%s’") % dbPath);
+
+        if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK)
+            throwSQLiteError(state->db, "setting timeout");
+
+        // We can always reproduce the cache.
+        if (sqlite3_exec(state->db, "pragma synchronous = off", 0, 0, 0) != SQLITE_OK)
+            throwSQLiteError(state->db, "making database asynchronous");
+        if (sqlite3_exec(state->db, "pragma main.journal_mode = truncate", 0, 0, 0) != SQLITE_OK)
+            throwSQLiteError(state->db, "setting journal mode");
+
+        if (sqlite3_exec(state->db, schema, 0, 0, 0) != SQLITE_OK)
+            throwSQLiteError(state->db, "initialising database schema");
+
+        state->insertCache.create(state->db,
+            "insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)");
+
+        state->queryCache.create(state->db,
+            "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ?");
+
+        state->insertNAR.create(state->db,
+            "insert or replace into NARs(cache, storePath, url, compression, fileHash, fileSize, narHash, "
+            "narSize, refs, deriver, sigs, timestamp) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
+
+        state->queryNAR.create(state->db,
+            "select * from NARs where cache = ? and storePath = ?");
+
+        state->insertNARExistence.create(state->db,
+            "insert or replace into NARExistence(cache, storePath, exist, timestamp) values (?, ?, ?, ?)");
+
+        state->queryNARExistence.create(state->db,
+            "select exist, timestamp from NARExistence where cache = ? and storePath = ?");
+    }
+
+    int uriToInt(State & state, const std::string & uri)
+    {
+        auto i = state.caches.find(uri);
+        if (i == state.caches.end()) abort();
+        return i->second;
+    }
+
+    void createCache(const std::string & uri) override
+    {
+        auto state(_state.lock());
+
+        // FIXME: race
+
+        state->insertCache.use()(uri)(time(0))(settings.nixStore)(1)(0).exec();
+        assert(sqlite3_changes(state->db) == 1);
+        state->caches[uri] = sqlite3_last_insert_rowid(state->db);
+    }
+
+    bool cacheExists(const std::string & uri) override
+    {
+        auto state(_state.lock());
+
+        auto i = state->caches.find(uri);
+        if (i != state->caches.end()) return true;
+
+        auto queryCache(state->queryCache.use()(uri));
+
+        if (queryCache.next()) {
+            state->caches[uri] = queryCache.getInt(0);
+            return true;
+        }
+
+        return false;
+    }
+
+    std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
+        const std::string & uri, const Path & storePath) override
+    {
+        auto state(_state.lock());
+
+        auto queryNAR(state->queryNAR.use()
+            (uriToInt(*state, uri))
+            (baseNameOf(storePath)));
+
+        if (!queryNAR.next())
+            // FIXME: check NARExistence
+            return {oUnknown, 0};
+
+        auto narInfo = make_ref<NarInfo>();
+
+        // FIXME: implement TTL.
+
+        narInfo->path = storePath;
+        narInfo->url = queryNAR.getStr(2);
+        narInfo->compression = queryNAR.getStr(3);
+        if (!queryNAR.isNull(4))
+            narInfo->fileHash = parseHash(queryNAR.getStr(4));
+        narInfo->fileSize = queryNAR.getInt(5);
+        narInfo->narHash = parseHash(queryNAR.getStr(6));
+        narInfo->narSize = queryNAR.getInt(7);
+        for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
+            narInfo->references.insert(settings.nixStore + "/" + r);
+        if (!queryNAR.isNull(9))
+            narInfo->deriver = settings.nixStore + "/" + queryNAR.getStr(9);
+        for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
+            narInfo->sigs.insert(sig);
+
+        return {oValid, narInfo};
+    }
+
+    void upsertNarInfo(
+        const std::string & uri, std::shared_ptr<ValidPathInfo> info) override
+    {
+        auto state(_state.lock());
+
+        if (info) {
+
+            auto narInfo = std::dynamic_pointer_cast<NarInfo>(info);
+
+            state->insertNAR.use()
+                (uriToInt(*state, uri))
+                (baseNameOf(info->path))
+                (narInfo ? narInfo->url : "", narInfo != 0)
+                (narInfo ? narInfo->compression : "", narInfo != 0)
+                (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash)
+                (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
+                (info->narHash.to_string())
+                (info->narSize)
+                (concatStringsSep(" ", info->shortRefs()))
+                (info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "")
+                (concatStringsSep(" ", info->sigs))
+                (time(0)).exec();
+
+        } else {
+            // not implemented
+            abort();
+        }
+    }
+};
+
+ref<NarInfoDiskCache> getNarInfoDiskCache()
+{
+    static Sync<std::shared_ptr<NarInfoDiskCache>> cache;
+
+    auto cache_(cache.lock());
+    if (!*cache_) *cache_ = std::make_shared<NarInfoDiskCacheImpl>();
+    return ref<NarInfoDiskCache>(*cache_);
+}
+
+}
diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh
new file mode 100644
index 000000000000..ee1aafc63a5d
--- /dev/null
+++ b/src/libstore/nar-info-disk-cache.hh
@@ -0,0 +1,28 @@
+#pragma once
+
+#include "ref.hh"
+#include "nar-info.hh"
+
+namespace nix {
+
+class NarInfoDiskCache
+{
+public:
+    typedef enum { oValid, oInvalid, oUnknown } Outcome;
+
+    virtual void createCache(const std::string & uri) = 0;
+
+    virtual bool cacheExists(const std::string & uri) = 0;
+
+    virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
+        const std::string & uri, const Path & storePath) = 0;
+
+    virtual void upsertNarInfo(
+        const std::string & uri, std::shared_ptr<ValidPathInfo> narInfo) = 0;
+};
+
+/* Return a singleton cache object that can be used concurrently by
+   multiple threads. */
+ref<NarInfoDiskCache> getNarInfoDiskCache();
+
+}
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index 680facdcfeb8..c0c5cecd1730 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -5,16 +5,16 @@ namespace nix {
 
 NarInfo::NarInfo(const std::string & s, const std::string & whence)
 {
-    auto corrupt = [&]() {
+    auto corrupt = [&]() [[noreturn]] {
         throw Error("NAR info file ‘%1%’ is corrupt");
     };
 
     auto parseHashField = [&](const string & s) {
-        string::size_type colon = s.find(':');
-        if (colon == string::npos) corrupt();
-        HashType ht = parseHashType(string(s, 0, colon));
-        if (ht == htUnknown) corrupt();
-        return parseHash16or32(ht, string(s, colon + 1));
+        try {
+            return parseHash(s);
+        } catch (BadHash &) {
+            corrupt();
+        }
     };
 
     size_t pos = 0;
@@ -103,12 +103,4 @@ std::string NarInfo::to_string() const
     return res;
 }
 
-Strings NarInfo::shortRefs() const
-{
-    Strings refs;
-    for (auto & r : references)
-        refs.push_back(baseNameOf(r));
-    return refs;
-}
-
 }
diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh
index 3c783cf83fef..6bc2f03b139b 100644
--- a/src/libstore/nar-info.hh
+++ b/src/libstore/nar-info.hh
@@ -19,10 +19,6 @@ struct NarInfo : ValidPathInfo
     NarInfo(const std::string & s, const std::string & whence);
 
     std::string to_string() const;
-
-private:
-
-    Strings shortRefs() const;
 };
 
 }
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 55196397667a..430d0ecf1143 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -489,7 +489,6 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
     {
         auto state_(Store::state.lock());
         state_->pathInfoCache.clear();
-        stats.pathInfoCacheSize = 0;
     }
 }
 
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index f93fa0857588..816f9984d6eb 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -139,6 +139,11 @@ int64_t SQLiteStmt::Use::getInt(int col)
     return sqlite3_column_int64(stmt, col);
 }
 
+bool SQLiteStmt::Use::isNull(int col)
+{
+    return sqlite3_column_type(stmt, col) == SQLITE_NULL;
+}
+
 SQLiteTxn::SQLiteTxn(sqlite3 * db)
 {
     this->db = db;
diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh
index 326e4a4855b7..d6b4a8d9117b 100644
--- a/src/libstore/sqlite.hh
+++ b/src/libstore/sqlite.hh
@@ -58,6 +58,7 @@ struct SQLiteStmt
 
         std::string getStr(int col);
         int64_t getInt(int col);
+        bool isNull(int col);
     };
 
     Use use()
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 6543ed1f6d19..cac137a99d7c 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -2,6 +2,7 @@
 #include "globals.hh"
 #include "store-api.hh"
 #include "util.hh"
+#include "nar-info-disk-cache.hh"
 
 
 namespace nix {
@@ -225,6 +226,12 @@ Path computeStorePathForText(const string & name, const string & s,
 }
 
 
+std::string Store::getUri()
+{
+    return "";
+}
+
+
 bool Store::isValidPath(const Path & storePath)
 {
     {
@@ -236,7 +243,19 @@ bool Store::isValidPath(const Path & storePath)
         }
     }
 
+    if (diskCache) {
+        auto res = diskCache->lookupNarInfo(getUri(), storePath);
+        if (res.first != NarInfoDiskCache::oUnknown) {
+            auto state_(state.lock());
+            state_->pathInfoCache.upsert(storePath,
+                res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
+            return res.first == NarInfoDiskCache::oValid;
+        }
+    }
+
     return isValidPathUncached(storePath);
+
+    // FIXME: insert result into NARExistence table of diskCache.
 }
 
 
@@ -253,12 +272,26 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
         }
     }
 
+    if (diskCache) {
+        auto res = diskCache->lookupNarInfo(getUri(), storePath);
+        if (res.first != NarInfoDiskCache::oUnknown) {
+            auto state_(state.lock());
+            state_->pathInfoCache.upsert(storePath,
+                res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
+            if (res.first == NarInfoDiskCache::oInvalid)
+                throw InvalidPath(format("path ‘%s’ is not valid") % storePath);
+            return ref<ValidPathInfo>(res.second);
+        }
+    }
+
     auto info = queryPathInfoUncached(storePath);
 
+    if (diskCache && info)
+        diskCache->upsertNarInfo(getUri(), info);
+
     {
         auto state_(state.lock());
         state_->pathInfoCache.upsert(storePath, info);
-        stats.pathInfoCacheSize = state_->pathInfoCache.size();
     }
 
     if (!info) {
@@ -303,6 +336,10 @@ string Store::makeValidityRegistration(const PathSet & paths,
 
 const Store::Stats & Store::getStats()
 {
+    {
+        auto state_(state.lock());
+        stats.pathInfoCacheSize = state_->pathInfoCache.size();
+    }
     return stats;
 }
 
@@ -356,7 +393,7 @@ void Store::exportPaths(const Paths & paths,
 
 std::string ValidPathInfo::fingerprint() const
 {
-    if (narSize == 0 || narHash.type == htUnknown)
+    if (narSize == 0 || !narHash)
         throw Error(format("cannot calculate fingerprint of path ‘%s’ because its size/hash is not known")
             % path);
     return
@@ -389,6 +426,15 @@ bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::str
 }
 
 
+Strings ValidPathInfo::shortRefs() const
+{
+    Strings refs;
+    for (auto & r : references)
+        refs.push_back(baseNameOf(r));
+    return refs;
+}
+
+
 }
 
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index d45e401c391a..eab6da91ee89 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -134,6 +134,8 @@ struct ValidPathInfo
     /* Verify a single signature. */
     bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
 
+    Strings shortRefs() const;
+
     virtual ~ValidPathInfo() { }
 };
 
@@ -170,6 +172,7 @@ struct BuildResult
 struct BasicDerivation;
 struct Derivation;
 class FSAccessor;
+class NarInfoDiskCache;
 
 
 class Store : public std::enable_shared_from_this<Store>
@@ -183,10 +186,14 @@ protected:
 
     Sync<State> state;
 
+    std::shared_ptr<NarInfoDiskCache> diskCache;
+
 public:
 
     virtual ~Store() { }
 
+    virtual std::string getUri();
+
     /* Check whether a path is valid. */
     bool isValidPath(const Path & path);