From 451ebf24ce532f8d59f929efd486104fcebf1aa6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Apr 2016 14:12:38 +0200 Subject: Cache path info lookups in SQLite This re-implements the binary cache database in C++, allowing it to be used by other Store backends, in particular the S3 backend. --- src/libstore/nar-info-disk-cache.cc | 217 ++++++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 src/libstore/nar-info-disk-cache.cc (limited to 'src/libstore/nar-info-disk-cache.cc') diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc new file mode 100644 index 000000000000..30ef7b36c9dd --- /dev/null +++ b/src/libstore/nar-info-disk-cache.cc @@ -0,0 +1,217 @@ +#include "nar-info-disk-cache.hh" +#include "sync.hh" +#include "sqlite.hh" +#include "globals.hh" + +#include + +namespace nix { + +static const char * schema = R"sql( + +create table if not exists BinaryCaches ( + id integer primary key autoincrement not null, + url text unique not null, + timestamp integer not null, + storeDir text not null, + wantMassQuery integer not null, + priority integer not null +); + +create table if not exists NARs ( + cache integer not null, + storePath text not null, + url text, + compression text, + fileHash text, + fileSize integer, + narHash text, + narSize integer, + refs text, + deriver text, + sigs text, + timestamp integer not null, + primary key (cache, storePath), + foreign key (cache) references BinaryCaches(id) on delete cascade +); + +create table if not exists NARExistence ( + cache integer not null, + storePath text not null, + exist integer not null, + timestamp integer not null, + primary key (cache, storePath), + foreign key (cache) references BinaryCaches(id) on delete cascade +); + +)sql"; + +class NarInfoDiskCacheImpl : public NarInfoDiskCache +{ +public: + + /* How long negative lookups are valid. */ + const int ttlNegative = 3600; + + struct State + { + SQLite db; + SQLiteStmt insertCache, queryCache, insertNAR, queryNAR, insertNARExistence, queryNARExistence; + std::map caches; + }; + + Sync _state; + + NarInfoDiskCacheImpl() + { + auto state(_state.lock()); + + Path dbPath = getCacheDir() + "/nix/binary-cache-v3.sqlite"; + createDirs(dirOf(dbPath)); + + if (sqlite3_open_v2(dbPath.c_str(), &state->db.db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) + throw Error(format("cannot open store cache ‘%s’") % dbPath); + + if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK) + throwSQLiteError(state->db, "setting timeout"); + + // We can always reproduce the cache. + if (sqlite3_exec(state->db, "pragma synchronous = off", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "making database asynchronous"); + if (sqlite3_exec(state->db, "pragma main.journal_mode = truncate", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "setting journal mode"); + + if (sqlite3_exec(state->db, schema, 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "initialising database schema"); + + state->insertCache.create(state->db, + "insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)"); + + state->queryCache.create(state->db, + "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ?"); + + state->insertNAR.create(state->db, + "insert or replace into NARs(cache, storePath, url, compression, fileHash, fileSize, narHash, " + "narSize, refs, deriver, sigs, timestamp) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); + + state->queryNAR.create(state->db, + "select * from NARs where cache = ? and storePath = ?"); + + state->insertNARExistence.create(state->db, + "insert or replace into NARExistence(cache, storePath, exist, timestamp) values (?, ?, ?, ?)"); + + state->queryNARExistence.create(state->db, + "select exist, timestamp from NARExistence where cache = ? and storePath = ?"); + } + + int uriToInt(State & state, const std::string & uri) + { + auto i = state.caches.find(uri); + if (i == state.caches.end()) abort(); + return i->second; + } + + void createCache(const std::string & uri) override + { + auto state(_state.lock()); + + // FIXME: race + + state->insertCache.use()(uri)(time(0))(settings.nixStore)(1)(0).exec(); + assert(sqlite3_changes(state->db) == 1); + state->caches[uri] = sqlite3_last_insert_rowid(state->db); + } + + bool cacheExists(const std::string & uri) override + { + auto state(_state.lock()); + + auto i = state->caches.find(uri); + if (i != state->caches.end()) return true; + + auto queryCache(state->queryCache.use()(uri)); + + if (queryCache.next()) { + state->caches[uri] = queryCache.getInt(0); + return true; + } + + return false; + } + + std::pair> lookupNarInfo( + const std::string & uri, const Path & storePath) override + { + auto state(_state.lock()); + + auto queryNAR(state->queryNAR.use() + (uriToInt(*state, uri)) + (baseNameOf(storePath))); + + if (!queryNAR.next()) + // FIXME: check NARExistence + return {oUnknown, 0}; + + auto narInfo = make_ref(); + + // FIXME: implement TTL. + + narInfo->path = storePath; + narInfo->url = queryNAR.getStr(2); + narInfo->compression = queryNAR.getStr(3); + if (!queryNAR.isNull(4)) + narInfo->fileHash = parseHash(queryNAR.getStr(4)); + narInfo->fileSize = queryNAR.getInt(5); + narInfo->narHash = parseHash(queryNAR.getStr(6)); + narInfo->narSize = queryNAR.getInt(7); + for (auto & r : tokenizeString(queryNAR.getStr(8), " ")) + narInfo->references.insert(settings.nixStore + "/" + r); + if (!queryNAR.isNull(9)) + narInfo->deriver = settings.nixStore + "/" + queryNAR.getStr(9); + for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) + narInfo->sigs.insert(sig); + + return {oValid, narInfo}; + } + + void upsertNarInfo( + const std::string & uri, std::shared_ptr info) override + { + auto state(_state.lock()); + + if (info) { + + auto narInfo = std::dynamic_pointer_cast(info); + + state->insertNAR.use() + (uriToInt(*state, uri)) + (baseNameOf(info->path)) + (narInfo ? narInfo->url : "", narInfo != 0) + (narInfo ? narInfo->compression : "", narInfo != 0) + (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash) + (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) + (info->narHash.to_string()) + (info->narSize) + (concatStringsSep(" ", info->shortRefs())) + (info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "") + (concatStringsSep(" ", info->sigs)) + (time(0)).exec(); + + } else { + // not implemented + abort(); + } + } +}; + +ref getNarInfoDiskCache() +{ + static Sync> cache; + + auto cache_(cache.lock()); + if (!*cache_) *cache_ = std::make_shared(); + return ref(*cache_); +} + +} -- cgit 1.4.1