diff options
Diffstat (limited to 'src/libstore/local-store.cc')
| -rw-r--r-- | src/libstore/local-store.cc | 731 |
1 files changed, 234 insertions, 497 deletions
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 532d511ef8bf..d6e1e4d7e0d4 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -10,6 +10,7 @@ #include <iostream> #include <algorithm> #include <cstring> +#include <atomic> #include <sys/types.h> #include <sys/stat.h> @@ -36,168 +37,6 @@ namespace nix { -MakeError(SQLiteError, Error); -MakeError(SQLiteBusy, SQLiteError); - - -[[noreturn]] static void throwSQLiteError(sqlite3 * db, const format & f) -{ - int err = sqlite3_errcode(db); - if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { - if (err == SQLITE_PROTOCOL) - printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)"); - else { - static bool warned = false; - if (!warned) { - printMsg(lvlError, "warning: SQLite database is busy"); - warned = true; - } - } - /* Sleep for a while since retrying the transaction right away - is likely to fail again. */ -#if HAVE_NANOSLEEP - struct timespec t; - t.tv_sec = 0; - t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ - nanosleep(&t, 0); -#else - sleep(1); -#endif - throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); - } - else - throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); -} - - -/* Convenience macros for retrying a SQLite transaction. */ -#define retry_sqlite while (1) { try { -#define end_retry_sqlite break; } catch (SQLiteBusy & e) { } } - - -SQLite::~SQLite() -{ - try { - if (db && sqlite3_close(db) != SQLITE_OK) - throwSQLiteError(db, "closing database"); - } catch (...) { - ignoreException(); - } -} - - -void SQLiteStmt::create(sqlite3 * db, const string & s) -{ - checkInterrupt(); - assert(!stmt); - if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) - throwSQLiteError(db, "creating statement"); - this->db = db; -} - - -void SQLiteStmt::reset() -{ - assert(stmt); - /* Note: sqlite3_reset() returns the error code for the most - recent call to sqlite3_step(). So ignore it. */ - sqlite3_reset(stmt); - curArg = 1; -} - - -SQLiteStmt::~SQLiteStmt() -{ - try { - if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) - throwSQLiteError(db, "finalizing statement"); - } catch (...) { - ignoreException(); - } -} - - -void SQLiteStmt::bind(const string & value) -{ - if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind(int value) -{ - if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind64(long long value) -{ - if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind() -{ - if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -/* Helper class to ensure that prepared statements are reset when - leaving the scope that uses them. Unfinished prepared statements - prevent transactions from being aborted, and can cause locks to be - kept when they should be released. */ -struct SQLiteStmtUse -{ - SQLiteStmt & stmt; - SQLiteStmtUse(SQLiteStmt & stmt) : stmt(stmt) - { - stmt.reset(); - } - ~SQLiteStmtUse() - { - try { - stmt.reset(); - } catch (...) { - ignoreException(); - } - } -}; - - -struct SQLiteTxn -{ - bool active; - sqlite3 * db; - - SQLiteTxn(sqlite3 * db) : active(false) { - this->db = db; - if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "starting transaction"); - active = true; - } - - void commit() - { - if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "committing transaction"); - active = false; - } - - ~SQLiteTxn() - { - try { - if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "aborting transaction"); - } catch (...) { - ignoreException(); - } - } -}; - - void checkStoreNotSymlink() { if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; @@ -217,20 +56,21 @@ void checkStoreNotSymlink() LocalStore::LocalStore() - : reservedPath(settings.nixDBPath + "/reserved") - , didSetSubstituterEnv(false) + : linksDir(settings.nixStore + "/.links") + , reservedPath(settings.nixDBPath + "/reserved") + , schemaPath(settings.nixDBPath + "/schema") { - schemaPath = settings.nixDBPath + "/schema"; + auto state(_state.lock()); if (settings.readOnlyMode) { - openDB(false); + openDB(*state, false); return; } /* Create missing state directories if they don't already exist. */ createDirs(settings.nixStore); makeStoreWritable(); - createDirs(linksDir = settings.nixStore + "/.links"); + createDirs(linksDir); Path profilesDir = settings.nixStateDir + "/profiles"; createDirs(profilesDir); createDirs(settings.nixStateDir + "/temproots"); @@ -302,7 +142,7 @@ LocalStore::LocalStore() } catch (SysError & e) { if (e.errNo != EACCES) throw; settings.readOnlyMode = true; - openDB(false); + openDB(*state, false); return; } @@ -320,7 +160,7 @@ LocalStore::LocalStore() else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; - openDB(true); + openDB(*state, true); writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); } @@ -346,21 +186,41 @@ LocalStore::LocalStore() have performed the upgrade already. */ curSchema = getSchema(); - if (curSchema < 7) { upgradeStore7(); openDB(true); } + if (curSchema < 7) { upgradeStore7(); } + + openDB(*state, false); + + if (curSchema < 8) { + SQLiteTxn txn(state->db); + if (sqlite3_exec(state->db, "alter table ValidPaths add column ultimate integer", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "upgrading database schema"); + if (sqlite3_exec(state->db, "alter table ValidPaths add column sigs text", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "upgrading database schema"); + txn.commit(); + } + + if (curSchema < 9) { + SQLiteTxn txn(state->db); + if (sqlite3_exec(state->db, "drop table FailedPaths", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "upgrading database schema"); + txn.commit(); + } writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); lockFile(globalLock, ltRead, true); } - else openDB(false); + else openDB(*state, false); } LocalStore::~LocalStore() { + auto state(_state.lock()); + try { - for (auto & i : runningSubstituters) { + for (auto & i : state->runningSubstituters) { if (i.second.disabled) continue; i.second.to.close(); i.second.from.close(); @@ -373,9 +233,9 @@ LocalStore::~LocalStore() } try { - if (fdTempRoots != -1) { - fdTempRoots.close(); - unlink(fnTempRoots.c_str()); + if (state->fdTempRoots != -1) { + state->fdTempRoots.close(); + unlink(state->fnTempRoots.c_str()); } } catch (...) { ignoreException(); @@ -401,13 +261,14 @@ bool LocalStore::haveWriteAccess() } -void LocalStore::openDB(bool create) +void LocalStore::openDB(State & state, bool create) { if (!haveWriteAccess()) throw SysError(format("Nix database directory ‘%1%’ is not writable") % settings.nixDBPath); /* Open the Nix database. */ string dbPath = settings.nixDBPath + "/db.sqlite"; + auto & db(state.db); if (sqlite3_open_v2(dbPath.c_str(), &db.db, SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) throw Error(format("cannot open Nix database ‘%1%’") % dbPath); @@ -460,40 +321,31 @@ void LocalStore::openDB(bool create) } /* Prepare SQL statements. */ - stmtRegisterValidPath.create(db, - "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);"); - stmtUpdatePathInfo.create(db, - "update ValidPaths set narSize = ?, hash = ? where path = ?;"); - stmtAddReference.create(db, + state.stmtRegisterValidPath.create(db, + "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs) values (?, ?, ?, ?, ?, ?, ?);"); + state.stmtUpdatePathInfo.create(db, + "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ? where path = ?;"); + state.stmtAddReference.create(db, "insert or replace into Refs (referrer, reference) values (?, ?);"); - stmtQueryPathInfo.create(db, - "select id, hash, registrationTime, deriver, narSize from ValidPaths where path = ?;"); - stmtQueryReferences.create(db, + state.stmtQueryPathInfo.create(db, + "select id, hash, registrationTime, deriver, narSize, ultimate, sigs from ValidPaths where path = ?;"); + state.stmtQueryReferences.create(db, "select path from Refs join ValidPaths on reference = id where referrer = ?;"); - stmtQueryReferrers.create(db, + state.stmtQueryReferrers.create(db, "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); - stmtInvalidatePath.create(db, + state.stmtInvalidatePath.create(db, "delete from ValidPaths where path = ?;"); - stmtRegisterFailedPath.create(db, - "insert or ignore into FailedPaths (path, time) values (?, ?);"); - stmtHasPathFailed.create(db, - "select time from FailedPaths where path = ?;"); - stmtQueryFailedPaths.create(db, - "select path from FailedPaths;"); - // If the path is a derivation, then clear its outputs. - stmtClearFailedPath.create(db, - "delete from FailedPaths where ?1 = '*' or path = ?1 " - "or path in (select d.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where v.path = ?1);"); - stmtAddDerivationOutput.create(db, + state.stmtAddDerivationOutput.create(db, "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); - stmtQueryValidDerivers.create(db, + state.stmtQueryValidDerivers.create(db, "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); - stmtQueryDerivationOutputs.create(db, + state.stmtQueryDerivationOutputs.create(db, "select id, path from DerivationOutputs where drv = ?;"); // Use "path >= ?" with limit 1 rather than "path like '?%'" to // ensure efficient lookup. - stmtQueryPathFromHashPart.create(db, + state.stmtQueryPathFromHashPart.create(db, "select path from ValidPaths where path >= ? limit 1;"); + state.stmtQueryValidPaths.create(db, "select path from ValidPaths"); } @@ -688,23 +540,19 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & } -unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs) +uint64_t LocalStore::addValidPath(State & state, + const ValidPathInfo & info, bool checkOutputs) { - SQLiteStmtUse use(stmtRegisterValidPath); - stmtRegisterValidPath.bind(info.path); - stmtRegisterValidPath.bind("sha256:" + printHash(info.narHash)); - stmtRegisterValidPath.bind(info.registrationTime == 0 ? time(0) : info.registrationTime); - if (info.deriver != "") - stmtRegisterValidPath.bind(info.deriver); - else - stmtRegisterValidPath.bind(); // null - if (info.narSize != 0) - stmtRegisterValidPath.bind64(info.narSize); - else - stmtRegisterValidPath.bind(); // null - if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering valid path ‘%1%’ in database") % info.path); - unsigned long long id = sqlite3_last_insert_rowid(db); + state.stmtRegisterValidPath.use() + (info.path) + ("sha256:" + printHash(info.narHash)) + (info.registrationTime == 0 ? time(0) : info.registrationTime) + (info.deriver, info.deriver != "") + (info.narSize, info.narSize != 0) + (info.ultimate ? 1 : 0, info.ultimate) + (concatStringsSep(" ", info.sigs), !info.sigs.empty()) + .exec(); + uint64_t id = sqlite3_last_insert_rowid(state.db); /* If this is a derivation, then store the derivation outputs in the database. This is useful for the garbage collector: it can @@ -721,12 +569,11 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che if (checkOutputs) checkDerivationOutputs(info.path, drv); for (auto & i : drv.outputs) { - SQLiteStmtUse use(stmtAddDerivationOutput); - stmtAddDerivationOutput.bind(id); - stmtAddDerivationOutput.bind(i.first); - stmtAddDerivationOutput.bind(i.second.path); - if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE) - throwSQLiteError(db, format("adding derivation output for ‘%1%’ in database") % info.path); + state.stmtAddDerivationOutput.use() + (id) + (i.first) + (i.second.path) + .exec(); } } @@ -734,79 +581,6 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che } -void LocalStore::addReference(unsigned long long referrer, unsigned long long reference) -{ - SQLiteStmtUse use(stmtAddReference); - stmtAddReference.bind(referrer); - stmtAddReference.bind(reference); - if (sqlite3_step(stmtAddReference) != SQLITE_DONE) - throwSQLiteError(db, "adding reference to database"); -} - - -void LocalStore::registerFailedPath(const Path & path) -{ - retry_sqlite { - SQLiteStmtUse use(stmtRegisterFailedPath); - stmtRegisterFailedPath.bind(path); - stmtRegisterFailedPath.bind(time(0)); - if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering failed path ‘%1%’") % path); - } end_retry_sqlite; -} - - -bool LocalStore::hasPathFailed(const Path & path) -{ - retry_sqlite { - SQLiteStmtUse use(stmtHasPathFailed); - stmtHasPathFailed.bind(path); - int res = sqlite3_step(stmtHasPathFailed); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying whether path failed"); - return res == SQLITE_ROW; - } end_retry_sqlite; -} - - -PathSet LocalStore::queryFailedPaths() -{ - retry_sqlite { - SQLiteStmtUse use(stmtQueryFailedPaths); - - PathSet res; - int r; - while ((r = sqlite3_step(stmtQueryFailedPaths)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryFailedPaths, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error querying failed paths"); - - return res; - } end_retry_sqlite; -} - - -void LocalStore::clearFailedPaths(const PathSet & paths) -{ - retry_sqlite { - SQLiteTxn txn(db); - - for (auto & i : paths) { - SQLiteStmtUse use(stmtClearFailedPath); - stmtClearFailedPath.bind(i); - if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("clearing failed path ‘%1%’ in database") % i); - } - - txn.commit(); - } end_retry_sqlite; -} - - Hash parseHashField(const Path & path, const string & s) { string::size_type colon = s.find(':'); @@ -828,153 +602,117 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) assertStorePath(path); - retry_sqlite { + return retrySQLite<ValidPathInfo>([&]() { + auto state(_state.lock()); /* Get the path info. */ - SQLiteStmtUse use1(stmtQueryPathInfo); + auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path)); - stmtQueryPathInfo.bind(path); + if (!useQueryPathInfo.next()) + throw Error(format("path ‘%1%’ is not valid") % path); - int r = sqlite3_step(stmtQueryPathInfo); - if (r == SQLITE_DONE) throw Error(format("path ‘%1%’ is not valid") % path); - if (r != SQLITE_ROW) throwSQLiteError(db, "querying path in database"); + info.id = useQueryPathInfo.getInt(0); - info.id = sqlite3_column_int(stmtQueryPathInfo, 0); + info.narHash = parseHashField(path, useQueryPathInfo.getStr(1)); - const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1); - assert(s); - info.narHash = parseHashField(path, s); + info.registrationTime = useQueryPathInfo.getInt(2); - info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); - - s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); + auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3); if (s) info.deriver = s; /* Note that narSize = NULL yields 0. */ - info.narSize = sqlite3_column_int64(stmtQueryPathInfo, 4); + info.narSize = useQueryPathInfo.getInt(4); - /* Get the references. */ - SQLiteStmtUse use2(stmtQueryReferences); + info.ultimate = useQueryPathInfo.getInt(5) == 1; - stmtQueryReferences.bind(info.id); + s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6); + if (s) info.sigs = tokenizeString<StringSet>(s, " "); - while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) { - s = (const char *) sqlite3_column_text(stmtQueryReferences, 0); - assert(s); - info.references.insert(s); - } + /* Get the references. */ + auto useQueryReferences(state->stmtQueryReferences.use()(info.id)); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of ‘%1%’") % path); + while (useQueryReferences.next()) + info.references.insert(useQueryReferences.getStr(0)); return info; - } end_retry_sqlite; + }); } -/* Update path info in the database. Currently only updates the - narSize field. */ -void LocalStore::updatePathInfo(const ValidPathInfo & info) +/* Update path info in the database. */ +void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { - SQLiteStmtUse use(stmtUpdatePathInfo); - if (info.narSize != 0) - stmtUpdatePathInfo.bind64(info.narSize); - else - stmtUpdatePathInfo.bind(); // null - stmtUpdatePathInfo.bind("sha256:" + printHash(info.narHash)); - stmtUpdatePathInfo.bind(info.path); - if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE) - throwSQLiteError(db, format("updating info of path ‘%1%’ in database") % info.path); + state.stmtUpdatePathInfo.use() + (info.narSize, info.narSize != 0) + ("sha256:" + printHash(info.narHash)) + (info.ultimate ? 1 : 0, info.ultimate) + (concatStringsSep(" ", info.sigs), !info.sigs.empty()) + (info.path) + .exec(); } -unsigned long long LocalStore::queryValidPathId(const Path & path) +uint64_t LocalStore::queryValidPathId(State & state, const Path & path) { - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0); - if (res == SQLITE_DONE) throw Error(format("path ‘%1%’ is not valid") % path); - throwSQLiteError(db, "querying path in database"); + auto use(state.stmtQueryPathInfo.use()(path)); + if (!use.next()) + throw Error(format("path ‘%1%’ is not valid") % path); + return use.getInt(0); } -bool LocalStore::isValidPath_(const Path & path) +bool LocalStore::isValidPath(State & state, const Path & path) { - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying path in database"); - return res == SQLITE_ROW; + return state.stmtQueryPathInfo.use()(path).next(); } bool LocalStore::isValidPath(const Path & path) { - retry_sqlite { - return isValidPath_(path); - } end_retry_sqlite; + return retrySQLite<bool>([&]() { + auto state(_state.lock()); + return isValidPath(*state, path); + }); } PathSet LocalStore::queryValidPaths(const PathSet & paths) { - retry_sqlite { - PathSet res; - for (auto & i : paths) - if (isValidPath_(i)) res.insert(i); - return res; - } end_retry_sqlite; + PathSet res; + for (auto & i : paths) + if (isValidPath(i)) res.insert(i); + return res; } PathSet LocalStore::queryAllValidPaths() { - retry_sqlite { - SQLiteStmt stmt; - stmt.create(db, "select path from ValidPaths"); - + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); + auto use(state->stmtQueryValidPaths.use()); PathSet res; - int r; - while ((r = sqlite3_step(stmt)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmt, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error getting valid paths"); - + while (use.next()) res.insert(use.getStr(0)); return res; - } end_retry_sqlite; + }); } -void LocalStore::queryReferrers_(const Path & path, PathSet & referrers) +void LocalStore::queryReferrers(State & state, const Path & path, PathSet & referrers) { - SQLiteStmtUse use(stmtQueryReferrers); - - stmtQueryReferrers.bind(path); - - int r; - while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryReferrers, 0); - assert(s); - referrers.insert(s); - } + auto useQueryReferrers(state.stmtQueryReferrers.use()(path)); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of ‘%1%’") % path); + while (useQueryReferrers.next()) + referrers.insert(useQueryReferrers.getStr(0)); } void LocalStore::queryReferrers(const Path & path, PathSet & referrers) { assertStorePath(path); - retry_sqlite { - queryReferrers_(path, referrers); - } end_retry_sqlite; + return retrySQLite<void>([&]() { + auto state(_state.lock()); + queryReferrers(*state, path, referrers); + }); } @@ -988,67 +726,51 @@ PathSet LocalStore::queryValidDerivers(const Path & path) { assertStorePath(path); - retry_sqlite { - SQLiteStmtUse use(stmtQueryValidDerivers); - stmtQueryValidDerivers.bind(path); + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); - PathSet derivers; - int r; - while ((r = sqlite3_step(stmtQueryValidDerivers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryValidDerivers, 1); - assert(s); - derivers.insert(s); - } + auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(path)); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting valid derivers of ‘%1%’") % path); + PathSet derivers; + while (useQueryValidDerivers.next()) + derivers.insert(useQueryValidDerivers.getStr(1)); return derivers; - } end_retry_sqlite; + }); } PathSet LocalStore::queryDerivationOutputs(const Path & path) { - retry_sqlite { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); - PathSet outputs; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 1); - assert(s); - outputs.insert(s); - } + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() + (queryValidPathId(*state, path))); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting outputs of ‘%1%’") % path); + PathSet outputs; + while (useQueryDerivationOutputs.next()) + outputs.insert(useQueryDerivationOutputs.getStr(1)); return outputs; - } end_retry_sqlite; + }); } StringSet LocalStore::queryDerivationOutputNames(const Path & path) { - retry_sqlite { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + return retrySQLite<StringSet>([&]() { + auto state(_state.lock()); - StringSet outputNames; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 0); - assert(s); - outputNames.insert(s); - } + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() + (queryValidPathId(*state, path))); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting output names of ‘%1%’") % path); + StringSet outputNames; + while (useQueryDerivationOutputs.next()) + outputNames.insert(useQueryDerivationOutputs.getStr(0)); return outputNames; - } end_retry_sqlite; + }); } @@ -1058,29 +780,28 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart) Path prefix = settings.nixStore + "/" + hashPart; - retry_sqlite { - SQLiteStmtUse use(stmtQueryPathFromHashPart); - stmtQueryPathFromHashPart.bind(prefix); + return retrySQLite<Path>([&]() { + auto state(_state.lock()); + + auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix)); - int res = sqlite3_step(stmtQueryPathFromHashPart); - if (res == SQLITE_DONE) return ""; - if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database"); + if (!useQueryPathFromHashPart.next()) return ""; - const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0); + const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0); return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; - } end_retry_sqlite; + }); } void LocalStore::setSubstituterEnv() { - if (didSetSubstituterEnv) return; + static std::atomic_flag done; + + if (done.test_and_set()) return; /* Pass configuration options (including those overridden with --option) to substituters. */ setenv("_NIX_OPTIONS", settings.pack().c_str(), 1); - - didSetSubstituterEnv = true; } @@ -1202,10 +923,12 @@ template<class T> T LocalStore::getIntLineFromSubstituter(RunningSubstituter & r PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) { + auto state(_state.lock()); + PathSet res; for (auto & i : settings.substituters) { if (res.size() == paths.size()) break; - RunningSubstituter & run(runningSubstituters[i]); + RunningSubstituter & run(state->runningSubstituters[i]); startSubstituter(i, run); if (run.disabled) continue; string s = "have "; @@ -1222,6 +945,7 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) res.insert(path); } } + return res; } @@ -1229,7 +953,9 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) void LocalStore::querySubstitutablePathInfos(const Path & substituter, PathSet & paths, SubstitutablePathInfos & infos) { - RunningSubstituter & run(runningSubstituters[substituter]); + auto state(_state.lock()); + + RunningSubstituter & run(state->runningSubstituters[substituter]); startSubstituter(substituter, run); if (run.disabled) return; @@ -1286,28 +1012,31 @@ void LocalStore::registerValidPath(const ValidPathInfo & info) void LocalStore::registerValidPaths(const ValidPathInfos & infos) { - /* SQLite will fsync by default, but the new valid paths may not be fsync-ed. - * So some may want to fsync them before registering the validity, at the - * expense of some speed of the path registering operation. */ + /* SQLite will fsync by default, but the new valid paths may not + be fsync-ed. So some may want to fsync them before registering + the validity, at the expense of some speed of the path + registering operation. */ if (settings.syncBeforeRegistering) sync(); - retry_sqlite { - SQLiteTxn txn(db); + return retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); PathSet paths; for (auto & i : infos) { assert(i.narHash.type == htSHA256); - if (isValidPath_(i.path)) - updatePathInfo(i); + if (isValidPath(*state, i.path)) + updatePathInfo(*state, i); else - addValidPath(i, false); + addValidPath(*state, i, false); paths.insert(i.path); } for (auto & i : infos) { - unsigned long long referrer = queryValidPathId(i.path); + auto referrer = queryValidPathId(*state, i.path); for (auto & j : i.references) - addReference(referrer, queryValidPathId(j)); + state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } /* Check that the derivation outputs are correct. We can't do @@ -1328,24 +1057,17 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) topoSortPaths(paths); txn.commit(); - } end_retry_sqlite; + }); } /* Invalidate a path. The caller is responsible for checking that there are no referrers. */ -void LocalStore::invalidatePath(const Path & path) +void LocalStore::invalidatePath(State & state, const Path & path) { debug(format("invalidating path ‘%1%’") % path); - drvHashes.erase(path); - - SQLiteStmtUse use(stmtInvalidatePath); - - stmtInvalidatePath.bind(path); - - if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE) - throwSQLiteError(db, format("invalidating path ‘%1%’ in database") % path); + state.stmtInvalidatePath.use()(path).exec(); /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ @@ -1397,6 +1119,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, info.path = dstPath; info.narHash = hash.first; info.narSize = hash.second; + info.ultimate = true; registerValidPath(info); } @@ -1411,14 +1134,13 @@ Path LocalStore::addToStore(const string & name, const Path & _srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) { Path srcPath(absPath(_srcPath)); - debug(format("adding ‘%1%’ to the store") % srcPath); /* Read the whole path into memory. This is not a very scalable method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; if (recursive) - nix::dumpPath(srcPath, sink, filter); + dumpPath(srcPath, sink, filter); else sink.s = make_ref<std::string>(readFile(srcPath)); @@ -1456,6 +1178,7 @@ Path LocalStore::addTextToStore(const string & name, const string & s, info.narHash = hash; info.narSize = sink.s->size(); info.references = references; + info.ultimate = true; registerValidPath(info); } @@ -1712,20 +1435,22 @@ void LocalStore::invalidatePathChecked(const Path & path) { assertStorePath(path); - retry_sqlite { - SQLiteTxn txn(db); + retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); - if (isValidPath_(path)) { - PathSet referrers; queryReferrers_(path, referrers); + if (isValidPath(*state, path)) { + PathSet referrers; queryReferrers(*state, path, referrers); referrers.erase(path); /* ignore self-references */ if (!referrers.empty()) throw PathInUse(format("cannot delete path ‘%1%’ because it is in use by %2%") % path % showPaths(referrers)); - invalidatePath(path); + invalidatePath(*state, path); } txn.commit(); - } end_retry_sqlite; + }); } @@ -1790,7 +1515,10 @@ bool LocalStore::verifyStore(bool checkContents, bool repair) update = true; } - if (update) updatePathInfo(info); + if (update) { + auto state(_state.lock()); + updatePathInfo(*state, info); + } } @@ -1820,7 +1548,8 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, if (!isStorePath(path)) { printMsg(lvlError, format("path ‘%1%’ is not in the Nix store") % path); - invalidatePath(path); + auto state(_state.lock()); + invalidatePath(*state, path); return; } @@ -1838,7 +1567,8 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, if (canInvalidate) { printMsg(lvlError, format("path ‘%1%’ disappeared, removing from database...") % path); - invalidatePath(path); + auto state(_state.lock()); + invalidatePath(*state, path); } else { printMsg(lvlError, format("path ‘%1%’ disappeared, but it still has valid referrers!") % path); if (repair) @@ -1858,32 +1588,6 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, } -bool LocalStore::pathContentsGood(const Path & path) -{ - std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); - if (i != pathContentsGoodCache.end()) return i->second; - printMsg(lvlInfo, format("checking path ‘%1%’...") % path); - ValidPathInfo info = queryPathInfo(path); - bool res; - if (!pathExists(path)) - res = false; - else { - HashResult current = hashPath(info.narHash.type, path); - Hash nullHash(htSHA256); - res = info.narHash == nullHash || info.narHash == current.first; - } - pathContentsGoodCache[path] = res; - if (!res) printMsg(lvlError, format("path ‘%1%’ is corrupted or missing!") % path); - return res; -} - - -void LocalStore::markContentsGood(const Path & path) -{ - pathContentsGoodCache[path] = true; -} - - #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) static void makeMutable(const Path & path) @@ -1938,8 +1642,41 @@ void LocalStore::upgradeStore7() void LocalStore::vacuumDB() { - if (sqlite3_exec(db, "vacuum;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "vacuuming SQLite database"); + auto state(_state.lock()); + + if (sqlite3_exec(state->db, "vacuum;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "vacuuming SQLite database"); +} + + +void LocalStore::addSignatures(const Path & storePath, const StringSet & sigs) +{ + retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); + + auto info = queryPathInfo(storePath); + + info.sigs.insert(sigs.begin(), sigs.end()); + + updatePathInfo(*state, info); + + txn.commit(); + }); +} + + +void LocalStore::signPathInfo(ValidPathInfo & info) +{ + // FIXME: keep secret keys in memory. + + auto secretKeyFiles = settings.get("secret-key-files", Strings()); + + for (auto & secretKeyFile : secretKeyFiles) { + SecretKey secretKey(readFile(secretKeyFile)); + info.sign(secretKey); + } } |