diff options
Diffstat (limited to 'src/libstore/local-store.cc')
| -rw-r--r-- | src/libstore/local-store.cc | 1494 |
1 files changed, 390 insertions, 1104 deletions
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 074c3394ffb1..b44384957ca6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -5,7 +5,7 @@ #include "pathlocks.hh" #include "worker-protocol.hh" #include "derivations.hh" -#include "affinity.hh" +#include "nar-info.hh" #include <iostream> #include <algorithm> @@ -23,16 +23,11 @@ #include <time.h> #include <grp.h> -#if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H +#if __linux__ #include <sched.h> #include <sys/statvfs.h> #include <sys/mount.h> -#endif - -#if HAVE_LINUX_FS_H -#include <linux/fs.h> #include <sys/ioctl.h> -#include <errno.h> #endif #include <sqlite3.h> @@ -41,171 +36,6 @@ namespace nix { -MakeError(SQLiteError, Error); -MakeError(SQLiteBusy, SQLiteError); - - -static void throwSQLiteError(sqlite3 * db, const format & f) - __attribute__ ((noreturn)); - -static void throwSQLiteError(sqlite3 * db, const format & f) -{ - int err = sqlite3_errcode(db); - if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { - if (err == SQLITE_PROTOCOL) - printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)"); - else { - static bool warned = false; - if (!warned) { - printMsg(lvlError, "warning: SQLite database is busy"); - warned = true; - } - } - /* Sleep for a while since retrying the transaction right away - is likely to fail again. */ -#if HAVE_NANOSLEEP - struct timespec t; - t.tv_sec = 0; - t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ - nanosleep(&t, 0); -#else - sleep(1); -#endif - throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); - } - else - throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); -} - - -/* Convenience macros for retrying a SQLite transaction. */ -#define retry_sqlite while (1) { try { -#define end_retry_sqlite break; } catch (SQLiteBusy & e) { } } - - -SQLite::~SQLite() -{ - try { - if (db && sqlite3_close(db) != SQLITE_OK) - throwSQLiteError(db, "closing database"); - } catch (...) { - ignoreException(); - } -} - - -void SQLiteStmt::create(sqlite3 * db, const string & s) -{ - checkInterrupt(); - assert(!stmt); - if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) - throwSQLiteError(db, "creating statement"); - this->db = db; -} - - -void SQLiteStmt::reset() -{ - assert(stmt); - /* Note: sqlite3_reset() returns the error code for the most - recent call to sqlite3_step(). So ignore it. */ - sqlite3_reset(stmt); - curArg = 1; -} - - -SQLiteStmt::~SQLiteStmt() -{ - try { - if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) - throwSQLiteError(db, "finalizing statement"); - } catch (...) { - ignoreException(); - } -} - - -void SQLiteStmt::bind(const string & value) -{ - if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind(int value) -{ - if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind64(long long value) -{ - if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind() -{ - if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -/* Helper class to ensure that prepared statements are reset when - leaving the scope that uses them. Unfinished prepared statements - prevent transactions from being aborted, and can cause locks to be - kept when they should be released. */ -struct SQLiteStmtUse -{ - SQLiteStmt & stmt; - SQLiteStmtUse(SQLiteStmt & stmt) : stmt(stmt) - { - stmt.reset(); - } - ~SQLiteStmtUse() - { - try { - stmt.reset(); - } catch (...) { - ignoreException(); - } - } -}; - - -struct SQLiteTxn -{ - bool active; - sqlite3 * db; - - SQLiteTxn(sqlite3 * db) : active(false) { - this->db = db; - if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "starting transaction"); - active = true; - } - - void commit() - { - if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "committing transaction"); - active = false; - } - - ~SQLiteTxn() - { - try { - if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "aborting transaction"); - } catch (...) { - ignoreException(); - } - } -}; - - void checkStoreNotSymlink() { if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; @@ -224,20 +54,24 @@ void checkStoreNotSymlink() } -LocalStore::LocalStore(bool reserveSpace) - : didSetSubstituterEnv(false) +LocalStore::LocalStore() + : linksDir(settings.nixStore + "/.links") + , reservedPath(settings.nixDBPath + "/reserved") + , schemaPath(settings.nixDBPath + "/schema") + , requireSigs(settings.get("signed-binary-caches", std::string("")) != "") // FIXME: rename option + , publicKeys(getDefaultPublicKeys()) { - schemaPath = settings.nixDBPath + "/schema"; + auto state(_state.lock()); if (settings.readOnlyMode) { - openDB(false); + openDB(*state, false); return; } /* Create missing state directories if they don't already exist. */ createDirs(settings.nixStore); makeStoreWritable(); - createDirs(linksDir = settings.nixStore + "/.links"); + createDirs(linksDir); Path profilesDir = settings.nixStateDir + "/profiles"; createDirs(profilesDir); createDirs(settings.nixStateDir + "/temproots"); @@ -284,25 +118,20 @@ LocalStore::LocalStore(bool reserveSpace) needed, we reserve some dummy space that we can free just before doing a garbage collection. */ try { - Path reservedPath = settings.nixDBPath + "/reserved"; - if (reserveSpace) { - struct stat st; - if (stat(reservedPath.c_str(), &st) == -1 || - st.st_size != settings.reservedSize) - { - AutoCloseFD fd = open(reservedPath.c_str(), O_WRONLY | O_CREAT, 0600); - int res = -1; + struct stat st; + if (stat(reservedPath.c_str(), &st) == -1 || + st.st_size != settings.reservedSize) + { + AutoCloseFD fd = open(reservedPath.c_str(), O_WRONLY | O_CREAT, 0600); + int res = -1; #if HAVE_POSIX_FALLOCATE - res = posix_fallocate(fd, 0, settings.reservedSize); + res = posix_fallocate(fd, 0, settings.reservedSize); #endif - if (res == -1) { - writeFull(fd, string(settings.reservedSize, 'X')); - ftruncate(fd, settings.reservedSize); - } + if (res == -1) { + writeFull(fd, string(settings.reservedSize, 'X')); + ftruncate(fd, settings.reservedSize); } } - else - deletePath(reservedPath); } catch (SysError & e) { /* don't care about errors */ } @@ -314,7 +143,7 @@ LocalStore::LocalStore(bool reserveSpace) } catch (SysError & e) { if (e.errNo != EACCES) throw; settings.readOnlyMode = true; - openDB(false); + openDB(*state, false); return; } @@ -332,7 +161,7 @@ LocalStore::LocalStore(bool reserveSpace) else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; - openDB(true); + openDB(*state, true); writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); } @@ -343,6 +172,12 @@ LocalStore::LocalStore(bool reserveSpace) "which is no longer supported. To convert to the new format,\n" "please upgrade Nix to version 0.12 first."); + if (curSchema < 6) + throw Error( + "Your Nix store has a database in flat file format,\n" + "which is no longer supported. To convert to the new format,\n" + "please upgrade Nix to version 1.11 first."); + if (!lockFile(globalLock, ltWrite, false)) { printMsg(lvlError, "waiting for exclusive access to the Nix store..."); lockFile(globalLock, ltWrite, true); @@ -352,37 +187,43 @@ LocalStore::LocalStore(bool reserveSpace) have performed the upgrade already. */ curSchema = getSchema(); - if (curSchema < 6) upgradeStore6(); - else if (curSchema < 7) { upgradeStore7(); openDB(true); } + if (curSchema < 7) { upgradeStore7(); } + + openDB(*state, false); + + if (curSchema < 8) { + SQLiteTxn txn(state->db); + if (sqlite3_exec(state->db, "alter table ValidPaths add column ultimate integer", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "upgrading database schema"); + if (sqlite3_exec(state->db, "alter table ValidPaths add column sigs text", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "upgrading database schema"); + txn.commit(); + } + + if (curSchema < 9) { + SQLiteTxn txn(state->db); + if (sqlite3_exec(state->db, "drop table FailedPaths", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "upgrading database schema"); + txn.commit(); + } writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); lockFile(globalLock, ltRead, true); } - else openDB(false); + else openDB(*state, false); } LocalStore::~LocalStore() { - try { - foreach (RunningSubstituters::iterator, i, runningSubstituters) { - if (i->second.disabled) continue; - i->second.to.close(); - i->second.from.close(); - i->second.error.close(); - if (i->second.pid != -1) - i->second.pid.wait(true); - } - } catch (...) { - ignoreException(); - } + auto state(_state.lock()); try { - if (fdTempRoots != -1) { - fdTempRoots.close(); - unlink(fnTempRoots.c_str()); + if (state->fdTempRoots != -1) { + state->fdTempRoots.close(); + unlink(state->fnTempRoots.c_str()); } } catch (...) { ignoreException(); @@ -390,6 +231,12 @@ LocalStore::~LocalStore() } +std::string LocalStore::getUri() +{ + return "local"; +} + + int LocalStore::getSchema() { int curSchema = 0; @@ -402,13 +249,20 @@ int LocalStore::getSchema() } -void LocalStore::openDB(bool create) +bool LocalStore::haveWriteAccess() +{ + return access(settings.nixDBPath.c_str(), R_OK | W_OK) == 0; +} + + +void LocalStore::openDB(State & state, bool create) { - if (access(settings.nixDBPath.c_str(), R_OK | W_OK)) + if (!haveWriteAccess()) throw SysError(format("Nix database directory ‘%1%’ is not writable") % settings.nixDBPath); /* Open the Nix database. */ string dbPath = settings.nixDBPath + "/db.sqlite"; + auto & db(state.db); if (sqlite3_open_v2(dbPath.c_str(), &db.db, SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) throw Error(format("cannot open Nix database ‘%1%’") % dbPath); @@ -461,40 +315,31 @@ void LocalStore::openDB(bool create) } /* Prepare SQL statements. */ - stmtRegisterValidPath.create(db, - "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);"); - stmtUpdatePathInfo.create(db, - "update ValidPaths set narSize = ?, hash = ? where path = ?;"); - stmtAddReference.create(db, + state.stmtRegisterValidPath.create(db, + "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs) values (?, ?, ?, ?, ?, ?, ?);"); + state.stmtUpdatePathInfo.create(db, + "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ? where path = ?;"); + state.stmtAddReference.create(db, "insert or replace into Refs (referrer, reference) values (?, ?);"); - stmtQueryPathInfo.create(db, - "select id, hash, registrationTime, deriver, narSize from ValidPaths where path = ?;"); - stmtQueryReferences.create(db, + state.stmtQueryPathInfo.create(db, + "select id, hash, registrationTime, deriver, narSize, ultimate, sigs from ValidPaths where path = ?;"); + state.stmtQueryReferences.create(db, "select path from Refs join ValidPaths on reference = id where referrer = ?;"); - stmtQueryReferrers.create(db, + state.stmtQueryReferrers.create(db, "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); - stmtInvalidatePath.create(db, + state.stmtInvalidatePath.create(db, "delete from ValidPaths where path = ?;"); - stmtRegisterFailedPath.create(db, - "insert or ignore into FailedPaths (path, time) values (?, ?);"); - stmtHasPathFailed.create(db, - "select time from FailedPaths where path = ?;"); - stmtQueryFailedPaths.create(db, - "select path from FailedPaths;"); - // If the path is a derivation, then clear its outputs. - stmtClearFailedPath.create(db, - "delete from FailedPaths where ?1 = '*' or path = ?1 " - "or path in (select d.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where v.path = ?1);"); - stmtAddDerivationOutput.create(db, + state.stmtAddDerivationOutput.create(db, "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); - stmtQueryValidDerivers.create(db, + state.stmtQueryValidDerivers.create(db, "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); - stmtQueryDerivationOutputs.create(db, + state.stmtQueryDerivationOutputs.create(db, "select id, path from DerivationOutputs where drv = ?;"); // Use "path >= ?" with limit 1 rather than "path like '?%'" to // ensure efficient lookup. - stmtQueryPathFromHashPart.create(db, + state.stmtQueryPathFromHashPart.create(db, "select path from ValidPaths where path >= ? limit 1;"); + state.stmtQueryValidPaths.create(db, "select path from ValidPaths"); } @@ -502,7 +347,7 @@ void LocalStore::openDB(bool create) bind mount. So make the Nix store writable for this process. */ void LocalStore::makeStoreWritable() { -#if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_REMOUNT) +#if __linux__ if (getuid() != 0) return; /* Check if /nix/store is on a read-only mount. */ struct statvfs stat; @@ -607,10 +452,10 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe users group); we check for this case below. */ if (st.st_uid != geteuid()) { #if HAVE_LCHOWN - if (lchown(path.c_str(), geteuid(), (gid_t) -1) == -1) + if (lchown(path.c_str(), geteuid(), getegid()) == -1) #else if (!S_ISLNK(st.st_mode) && - chown(path.c_str(), geteuid(), (gid_t) -1) == -1) + chown(path.c_str(), geteuid(), getegid()) == -1) #endif throw SysError(format("changing owner of ‘%1%’ to %2%") % path % geteuid()); @@ -654,7 +499,7 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & assert(isDerivation(drvName)); drvName = string(drvName, 0, drvName.size() - drvExtension.size()); - if (isFixedOutputDrv(drv)) { + if (drv.isFixedOutput()) { DerivationOutputs::const_iterator out = drv.outputs.find("out"); if (out == drv.outputs.end()) throw Error(format("derivation ‘%1%’ does not have an output named ‘out’") % drvPath); @@ -671,41 +516,37 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & else { Derivation drvCopy(drv); - foreach (DerivationOutputs::iterator, i, drvCopy.outputs) { - i->second.path = ""; - drvCopy.env[i->first] = ""; + for (auto & i : drvCopy.outputs) { + i.second.path = ""; + drvCopy.env[i.first] = ""; } Hash h = hashDerivationModulo(*this, drvCopy); - foreach (DerivationOutputs::const_iterator, i, drv.outputs) { - Path outPath = makeOutputPath(i->first, h, drvName); - StringPairs::const_iterator j = drv.env.find(i->first); - if (i->second.path != outPath || j == drv.env.end() || j->second != outPath) + for (auto & i : drv.outputs) { + Path outPath = makeOutputPath(i.first, h, drvName); + StringPairs::const_iterator j = drv.env.find(i.first); + if (i.second.path != outPath || j == drv.env.end() || j->second != outPath) throw Error(format("derivation ‘%1%’ has incorrect output ‘%2%’, should be ‘%3%’") - % drvPath % i->second.path % outPath); + % drvPath % i.second.path % outPath); } } } -unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs) +uint64_t LocalStore::addValidPath(State & state, + const ValidPathInfo & info, bool checkOutputs) { - SQLiteStmtUse use(stmtRegisterValidPath); - stmtRegisterValidPath.bind(info.path); - stmtRegisterValidPath.bind("sha256:" + printHash(info.hash)); - stmtRegisterValidPath.bind(info.registrationTime == 0 ? time(0) : info.registrationTime); - if (info.deriver != "") - stmtRegisterValidPath.bind(info.deriver); - else - stmtRegisterValidPath.bind(); // null - if (info.narSize != 0) - stmtRegisterValidPath.bind64(info.narSize); - else - stmtRegisterValidPath.bind(); // null - if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering valid path ‘%1%’ in database") % info.path); - unsigned long long id = sqlite3_last_insert_rowid(db); + state.stmtRegisterValidPath.use() + (info.path) + ("sha256:" + printHash(info.narHash)) + (info.registrationTime == 0 ? time(0) : info.registrationTime) + (info.deriver, info.deriver != "") + (info.narSize, info.narSize != 0) + (info.ultimate ? 1 : 0, info.ultimate) + (concatStringsSep(" ", info.sigs), !info.sigs.empty()) + .exec(); + uint64_t id = sqlite3_last_insert_rowid(state.db); /* If this is a derivation, then store the derivation outputs in the database. This is useful for the garbage collector: it can @@ -721,90 +562,21 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che registration above is undone. */ if (checkOutputs) checkDerivationOutputs(info.path, drv); - foreach (DerivationOutputs::iterator, i, drv.outputs) { - SQLiteStmtUse use(stmtAddDerivationOutput); - stmtAddDerivationOutput.bind(id); - stmtAddDerivationOutput.bind(i->first); - stmtAddDerivationOutput.bind(i->second.path); - if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE) - throwSQLiteError(db, format("adding derivation output for ‘%1%’ in database") % info.path); + for (auto & i : drv.outputs) { + state.stmtAddDerivationOutput.use() + (id) + (i.first) + (i.second.path) + .exec(); } } - return id; -} - - -void LocalStore::addReference(unsigned long long referrer, unsigned long long reference) -{ - SQLiteStmtUse use(stmtAddReference); - stmtAddReference.bind(referrer); - stmtAddReference.bind(reference); - if (sqlite3_step(stmtAddReference) != SQLITE_DONE) - throwSQLiteError(db, "adding reference to database"); -} - - -void LocalStore::registerFailedPath(const Path & path) -{ - retry_sqlite { - SQLiteStmtUse use(stmtRegisterFailedPath); - stmtRegisterFailedPath.bind(path); - stmtRegisterFailedPath.bind(time(0)); - if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering failed path ‘%1%’") % path); - } end_retry_sqlite; -} - - -bool LocalStore::hasPathFailed(const Path & path) -{ - retry_sqlite { - SQLiteStmtUse use(stmtHasPathFailed); - stmtHasPathFailed.bind(path); - int res = sqlite3_step(stmtHasPathFailed); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying whether path failed"); - return res == SQLITE_ROW; - } end_retry_sqlite; -} - - -PathSet LocalStore::queryFailedPaths() -{ - retry_sqlite { - SQLiteStmtUse use(stmtQueryFailedPaths); - - PathSet res; - int r; - while ((r = sqlite3_step(stmtQueryFailedPaths)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryFailedPaths, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error querying failed paths"); - - return res; - } end_retry_sqlite; -} - - -void LocalStore::clearFailedPaths(const PathSet & paths) -{ - retry_sqlite { - SQLiteTxn txn(db); - - foreach (PathSet::const_iterator, i, paths) { - SQLiteStmtUse use(stmtClearFailedPath); - stmtClearFailedPath.bind(*i); - if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("clearing failed path ‘%1%’ in database") % *i); - } + { + auto state_(Store::state.lock()); + state_->pathInfoCache.upsert(storePathToHash(info.path), std::make_shared<ValidPathInfo>(info)); + } - txn.commit(); - } end_retry_sqlite; + return id; } @@ -822,174 +594,124 @@ Hash parseHashField(const Path & path, const string & s) } -ValidPathInfo LocalStore::queryPathInfo(const Path & path) +std::shared_ptr<ValidPathInfo> LocalStore::queryPathInfoUncached(const Path & path) { - ValidPathInfo info; - info.path = path; + auto info = std::make_shared<ValidPathInfo>(); + info->path = path; assertStorePath(path); - retry_sqlite { + return retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { + auto state(_state.lock()); /* Get the path info. */ - SQLiteStmtUse use1(stmtQueryPathInfo); - - stmtQueryPathInfo.bind(path); + auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path)); - int r = sqlite3_step(stmtQueryPathInfo); - if (r == SQLITE_DONE) throw Error(format("path ‘%1%’ is not valid") % path); - if (r != SQLITE_ROW) throwSQLiteError(db, "querying path in database"); + if (!useQueryPathInfo.next()) + return std::shared_ptr<ValidPathInfo>(); - info.id = sqlite3_column_int(stmtQueryPathInfo, 0); + info->id = useQueryPathInfo.getInt(0); - const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1); - assert(s); - info.hash = parseHashField(path, s); + info->narHash = parseHashField(path, useQueryPathInfo.getStr(1)); - info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); + info->registrationTime = useQueryPathInfo.getInt(2); - s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); - if (s) info.deriver = s; + auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3); + if (s) info->deriver = s; /* Note that narSize = NULL yields 0. */ - info.narSize = sqlite3_column_int64(stmtQueryPathInfo, 4); + info->narSize = useQueryPathInfo.getInt(4); - /* Get the references. */ - SQLiteStmtUse use2(stmtQueryReferences); + info->ultimate = useQueryPathInfo.getInt(5) == 1; - stmtQueryReferences.bind(info.id); + s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6); + if (s) info->sigs = tokenizeString<StringSet>(s, " "); - while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) { - s = (const char *) sqlite3_column_text(stmtQueryReferences, 0); - assert(s); - info.references.insert(s); - } + /* Get the references. */ + auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of ‘%1%’") % path); + while (useQueryReferences.next()) + info->references.insert(useQueryReferences.getStr(0)); return info; - } end_retry_sqlite; + }); } -/* Update path info in the database. Currently only updates the - narSize field. */ -void LocalStore::updatePathInfo(const ValidPathInfo & info) +/* Update path info in the database. */ +void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { - SQLiteStmtUse use(stmtUpdatePathInfo); - if (info.narSize != 0) - stmtUpdatePathInfo.bind64(info.narSize); - else - stmtUpdatePathInfo.bind(); // null - stmtUpdatePathInfo.bind("sha256:" + printHash(info.hash)); - stmtUpdatePathInfo.bind(info.path); - if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE) - throwSQLiteError(db, format("updating info of path ‘%1%’ in database") % info.path); + state.stmtUpdatePathInfo.use() + (info.narSize, info.narSize != 0) + ("sha256:" + printHash(info.narHash)) + (info.ultimate ? 1 : 0, info.ultimate) + (concatStringsSep(" ", info.sigs), !info.sigs.empty()) + (info.path) + .exec(); } -unsigned long long LocalStore::queryValidPathId(const Path & path) +uint64_t LocalStore::queryValidPathId(State & state, const Path & path) { - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0); - if (res == SQLITE_DONE) throw Error(format("path ‘%1%’ is not valid") % path); - throwSQLiteError(db, "querying path in database"); + auto use(state.stmtQueryPathInfo.use()(path)); + if (!use.next()) + throw Error(format("path ‘%1%’ is not valid") % path); + return use.getInt(0); } -bool LocalStore::isValidPath_(const Path & path) +bool LocalStore::isValidPath_(State & state, const Path & path) { - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying path in database"); - return res == SQLITE_ROW; + return state.stmtQueryPathInfo.use()(path).next(); } -bool LocalStore::isValidPath(const Path & path) +bool LocalStore::isValidPathUncached(const Path & path) { - retry_sqlite { - return isValidPath_(path); - } end_retry_sqlite; + return retrySQLite<bool>([&]() { + auto state(_state.lock()); + return isValidPath_(*state, path); + }); } PathSet LocalStore::queryValidPaths(const PathSet & paths) { - retry_sqlite { - PathSet res; - foreach (PathSet::const_iterator, i, paths) - if (isValidPath_(*i)) res.insert(*i); - return res; - } end_retry_sqlite; + PathSet res; + for (auto & i : paths) + if (isValidPath(i)) res.insert(i); + return res; } PathSet LocalStore::queryAllValidPaths() { - retry_sqlite { - SQLiteStmt stmt; - stmt.create(db, "select path from ValidPaths"); - + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); + auto use(state->stmtQueryValidPaths.use()); PathSet res; - int r; - while ((r = sqlite3_step(stmt)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmt, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error getting valid paths"); - + while (use.next()) res.insert(use.getStr(0)); return res; - } end_retry_sqlite; -} - - -void LocalStore::queryReferences(const Path & path, - PathSet & references) -{ - ValidPathInfo info = queryPathInfo(path); - references.insert(info.references.begin(), info.references.end()); + }); } -void LocalStore::queryReferrers_(const Path & path, PathSet & referrers) +void LocalStore::queryReferrers(State & state, const Path & path, PathSet & referrers) { - SQLiteStmtUse use(stmtQueryReferrers); - - stmtQueryReferrers.bind(path); + auto useQueryReferrers(state.stmtQueryReferrers.use()(path)); - int r; - while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryReferrers, 0); - assert(s); - referrers.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of ‘%1%’") % path); + while (useQueryReferrers.next()) + referrers.insert(useQueryReferrers.getStr(0)); } void LocalStore::queryReferrers(const Path & path, PathSet & referrers) { assertStorePath(path); - retry_sqlite { - queryReferrers_(path, referrers); - } end_retry_sqlite; -} - - -Path LocalStore::queryDeriver(const Path & path) -{ - return queryPathInfo(path).deriver; + return retrySQLite<void>([&]() { + auto state(_state.lock()); + queryReferrers(*state, path, referrers); + }); } @@ -997,294 +719,114 @@ PathSet LocalStore::queryValidDerivers(const Path & path) { assertStorePath(path); - retry_sqlite { - SQLiteStmtUse use(stmtQueryValidDerivers); - stmtQueryValidDerivers.bind(path); + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); - PathSet derivers; - int r; - while ((r = sqlite3_step(stmtQueryValidDerivers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryValidDerivers, 1); - assert(s); - derivers.insert(s); - } + auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(path)); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting valid derivers of ‘%1%’") % path); + PathSet derivers; + while (useQueryValidDerivers.next()) + derivers.insert(useQueryValidDerivers.getStr(1)); return derivers; - } end_retry_sqlite; + }); } PathSet LocalStore::queryDerivationOutputs(const Path & path) { - retry_sqlite { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); - PathSet outputs; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 1); - assert(s); - outputs.insert(s); - } + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() + (queryValidPathId(*state, path))); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting outputs of ‘%1%’") % path); + PathSet outputs; + while (useQueryDerivationOutputs.next()) + outputs.insert(useQueryDerivationOutputs.getStr(1)); return outputs; - } end_retry_sqlite; + }); } StringSet LocalStore::queryDerivationOutputNames(const Path & path) { - retry_sqlite { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + return retrySQLite<StringSet>([&]() { + auto state(_state.lock()); - StringSet outputNames; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 0); - assert(s); - outputNames.insert(s); - } + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() + (queryValidPathId(*state, path))); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting output names of ‘%1%’") % path); + StringSet outputNames; + while (useQueryDerivationOutputs.next()) + outputNames.insert(useQueryDerivationOutputs.getStr(0)); return outputNames; - } end_retry_sqlite; + }); } Path LocalStore::queryPathFromHashPart(const string & hashPart) { - if (hashPart.size() != 32) throw Error("invalid hash part"); + if (hashPart.size() != storePathHashLen) throw Error("invalid hash part"); Path prefix = settings.nixStore + "/" + hashPart; - retry_sqlite { - SQLiteStmtUse use(stmtQueryPathFromHashPart); - stmtQueryPathFromHashPart.bind(prefix); - - int res = sqlite3_step(stmtQueryPathFromHashPart); - if (res == SQLITE_DONE) return ""; - if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database"); - - const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0); - return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; - } end_retry_sqlite; -} - - -void LocalStore::setSubstituterEnv() -{ - if (didSetSubstituterEnv) return; - - /* Pass configuration options (including those overridden with - --option) to substituters. */ - setenv("_NIX_OPTIONS", settings.pack().c_str(), 1); - - didSetSubstituterEnv = true; -} - + return retrySQLite<Path>([&]() { + auto state(_state.lock()); -void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run) -{ - if (run.disabled || run.pid != -1) return; - - debug(format("starting substituter program ‘%1%’") % substituter); - - Pipe toPipe, fromPipe, errorPipe; - - toPipe.create(); - fromPipe.create(); - errorPipe.create(); + auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix)); - setSubstituterEnv(); + if (!useQueryPathFromHashPart.next()) return ""; - run.pid = startProcess([&]() { - if (dup2(toPipe.readSide, STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1) - throw SysError("dupping stderr"); - execl(substituter.c_str(), substituter.c_str(), "--query", NULL); - throw SysError(format("executing ‘%1%’") % substituter); + const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0); + return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; }); - - run.program = baseNameOf(substituter); - run.to = toPipe.writeSide.borrow(); - run.from = run.fromBuf.fd = fromPipe.readSide.borrow(); - run.error = errorPipe.readSide.borrow(); - - toPipe.readSide.close(); - fromPipe.writeSide.close(); - errorPipe.writeSide.close(); - - /* The substituter may exit right away if it's disabled in any way - (e.g. copy-from-other-stores.pl will exit if no other stores - are configured). */ - try { - getLineFromSubstituter(run); - } catch (EndOfFile & e) { - run.to.close(); - run.from.close(); - run.error.close(); - run.disabled = true; - if (run.pid.wait(true) != 0) throw; - } -} - - -/* Read a line from the substituter's stdout, while also processing - its stderr. */ -string LocalStore::getLineFromSubstituter(RunningSubstituter & run) -{ - string res, err; - - /* We might have stdout data left over from the last time. */ - if (run.fromBuf.hasData()) goto haveData; - - while (1) { - checkInterrupt(); - - fd_set fds; - FD_ZERO(&fds); - FD_SET(run.from, &fds); - FD_SET(run.error, &fds); - - /* Wait for data to appear on the substituter's stdout or - stderr. */ - if (select(run.from > run.error ? run.from + 1 : run.error + 1, &fds, 0, 0, 0) == -1) { - if (errno == EINTR) continue; - throw SysError("waiting for input from the substituter"); - } - - /* Completely drain stderr before dealing with stdout. */ - if (FD_ISSET(run.error, &fds)) { - char buf[4096]; - ssize_t n = read(run.error, (unsigned char *) buf, sizeof(buf)); - if (n == -1) { - if (errno == EINTR) continue; - throw SysError("reading from substituter's stderr"); - } - if (n == 0) throw EndOfFile(format("substituter ‘%1%’ died unexpectedly") % run.program); - err.append(buf, n); - string::size_type p; - while ((p = err.find('\n')) != string::npos) { - printMsg(lvlError, run.program + ": " + string(err, 0, p)); - err = string(err, p + 1); - } - } - - /* Read from stdout until we get a newline or the buffer is empty. */ - else if (run.fromBuf.hasData() || FD_ISSET(run.from, &fds)) { - haveData: - do { - unsigned char c; - run.fromBuf(&c, 1); - if (c == '\n') { - if (!err.empty()) printMsg(lvlError, run.program + ": " + err); - return res; - } - res += c; - } while (run.fromBuf.hasData()); - } - } -} - - -template<class T> T LocalStore::getIntLineFromSubstituter(RunningSubstituter & run) -{ - string s = getLineFromSubstituter(run); - T res; - if (!string2Int(s, res)) throw Error("integer expected from stream"); - return res; } PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) { PathSet res; - foreach (Paths::iterator, i, settings.substituters) { - if (res.size() == paths.size()) break; - RunningSubstituter & run(runningSubstituters[*i]); - startSubstituter(*i, run); - if (run.disabled) continue; - string s = "have "; - foreach (PathSet::const_iterator, j, paths) - if (res.find(*j) == res.end()) { s += *j; s += " "; } - writeLine(run.to, s); - while (true) { - /* FIXME: we only read stderr when an error occurs, so - substituters should only write (short) messages to - stderr when they fail. I.e. they shouldn't write debug - output. */ - Path path = getLineFromSubstituter(run); - if (path == "") break; - res.insert(path); + for (auto & sub : getDefaultSubstituters()) { + if (!sub->wantMassQuery()) continue; + for (auto & path : paths) { + if (res.count(path)) continue; + debug(format("checking substituter ‘%s’ for path ‘%s’") + % sub->getUri() % path); + if (sub->isValidPath(path)) + res.insert(path); } } return res; } -void LocalStore::querySubstitutablePathInfos(const Path & substituter, - PathSet & paths, SubstitutablePathInfos & infos) -{ - RunningSubstituter & run(runningSubstituters[substituter]); - startSubstituter(substituter, run); - if (run.disabled) return; - - string s = "info "; - foreach (PathSet::const_iterator, i, paths) - if (infos.find(*i) == infos.end()) { s += *i; s += " "; } - writeLine(run.to, s); - - while (true) { - Path path = getLineFromSubstituter(run); - if (path == "") break; - if (paths.find(path) == paths.end()) - throw Error(format("got unexpected path ‘%1%’ from substituter") % path); - paths.erase(path); - SubstitutablePathInfo & info(infos[path]); - info.deriver = getLineFromSubstituter(run); - if (info.deriver != "") assertStorePath(info.deriver); - int nrRefs = getIntLineFromSubstituter<int>(run); - while (nrRefs--) { - Path p = getLineFromSubstituter(run); - assertStorePath(p); - info.references.insert(p); - } - info.downloadSize = getIntLineFromSubstituter<long long>(run); - info.narSize = getIntLineFromSubstituter<long long>(run); - } -} - - void LocalStore::querySubstitutablePathInfos(const PathSet & paths, SubstitutablePathInfos & infos) { - PathSet todo = paths; - foreach (Paths::iterator, i, settings.substituters) { - if (todo.empty()) break; - querySubstitutablePathInfos(*i, todo, infos); + for (auto & sub : getDefaultSubstituters()) { + for (auto & path : paths) { + if (infos.count(path)) continue; + debug(format("checking substituter ‘%s’ for path ‘%s’") + % sub->getUri() % path); + try { + auto info = sub->queryPathInfo(path); + auto narInfo = std::dynamic_pointer_cast<const NarInfo>( + std::shared_ptr<const ValidPathInfo>(info)); + infos[path] = SubstitutablePathInfo{ + info->deriver, + info->references, + narInfo ? narInfo->fileSize : 0, + info->narSize}; + } catch (InvalidPath) { + } + } } } -Hash LocalStore::queryPathHash(const Path & path) -{ - return queryPathInfo(path).hash; -} - - void LocalStore::registerValidPath(const ValidPathInfo & info) { ValidPathInfos infos; @@ -1295,69 +837,112 @@ void LocalStore::registerValidPath(const ValidPathInfo & info) void LocalStore::registerValidPaths(const ValidPathInfos & infos) { - /* SQLite will fsync by default, but the new valid paths may not be fsync-ed. - * So some may want to fsync them before registering the validity, at the - * expense of some speed of the path registering operation. */ + /* SQLite will fsync by default, but the new valid paths may not + be fsync-ed. So some may want to fsync them before registering + the validity, at the expense of some speed of the path + registering operation. */ if (settings.syncBeforeRegistering) sync(); - retry_sqlite { - SQLiteTxn txn(db); + return retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); PathSet paths; - foreach (ValidPathInfos::const_iterator, i, infos) { - assert(i->hash.type == htSHA256); - if (isValidPath_(i->path)) - updatePathInfo(*i); + for (auto & i : infos) { + assert(i.narHash.type == htSHA256); + if (isValidPath_(*state, i.path)) + updatePathInfo(*state, i); else - addValidPath(*i, false); - paths.insert(i->path); + addValidPath(*state, i, false); + paths.insert(i.path); } - foreach (ValidPathInfos::const_iterator, i, infos) { - unsigned long long referrer = queryValidPathId(i->path); - foreach (PathSet::iterator, j, i->references) - addReference(referrer, queryValidPathId(*j)); + for (auto & i : infos) { + auto referrer = queryValidPathId(*state, i.path); + for (auto & j : i.references) + state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } /* Check that the derivation outputs are correct. We can't do this in addValidPath() above, because the references might not be valid yet. */ - foreach (ValidPathInfos::const_iterator, i, infos) - if (isDerivation(i->path)) { + for (auto & i : infos) + if (isDerivation(i.path)) { // FIXME: inefficient; we already loaded the // derivation in addValidPath(). - Derivation drv = readDerivation(i->path); - checkDerivationOutputs(i->path, drv); + Derivation drv = readDerivation(i.path); + checkDerivationOutputs(i.path, drv); } /* Do a topological sort of the paths. This will throw an error if a cycle is detected and roll back the transaction. Cycles can only occur when a derivation has multiple outputs. */ - topoSortPaths(*this, paths); + topoSortPaths(paths); txn.commit(); - } end_retry_sqlite; + }); } /* Invalidate a path. The caller is responsible for checking that there are no referrers. */ -void LocalStore::invalidatePath(const Path & path) +void LocalStore::invalidatePath(State & state, const Path & path) { debug(format("invalidating path ‘%1%’") % path); - drvHashes.erase(path); + state.stmtInvalidatePath.use()(path).exec(); - SQLiteStmtUse use(stmtInvalidatePath); + /* Note that the foreign key constraints on the Refs table take + care of deleting the references entries for `path'. */ - stmtInvalidatePath.bind(path); + { + auto state_(Store::state.lock()); + state_->pathInfoCache.erase(storePathToHash(path)); + } +} - if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE) - throwSQLiteError(db, format("invalidating path ‘%1%’ in database") % path); - /* Note that the foreign key constraints on the Refs table take - care of deleting the references entries for `path'. */ +void LocalStore::addToStore(const ValidPathInfo & info, const std::string & nar, bool repair) +{ + Hash h = hashString(htSHA256, nar); + if (h != info.narHash) + throw Error(format("hash mismatch importing path ‘%s’; expected hash ‘%s’, got ‘%s’") % + info.path % info.narHash.to_string() % h.to_string()); + + if (requireSigs && !info.checkSignatures(publicKeys)) + throw Error(format("cannot import path ‘%s’ because it lacks a valid signature") % info.path); + + addTempRoot(info.path); + + if (repair || !isValidPath(info.path)) { + + PathLocks outputLock; + + /* Lock the output path. But don't lock if we're being called + from a build hook (whose parent process already acquired a + lock on this path). */ + Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS")); + if (find(locksHeld.begin(), locksHeld.end(), info.path) == locksHeld.end()) + outputLock.lockPaths({info.path}); + + if (repair || !isValidPath(info.path)) { + + deletePath(info.path); + + StringSource source(nar); + restorePath(info.path, source); + + canonicalisePathMetaData(info.path, -1); + + optimisePath(info.path); // FIXME: combine with hashPath() + + registerValidPath(info); + } + + outputLock.setDeletion(true); + } } @@ -1375,11 +960,11 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, /* The first check above is an optimisation to prevent unnecessary lock acquisition. */ - PathLocks outputLock(singleton<PathSet, Path>(dstPath)); + PathLocks outputLock({dstPath}); if (repair || !isValidPath(dstPath)) { - if (pathExists(dstPath)) deletePath(dstPath); + deletePath(dstPath); if (recursive) { StringSource source(dump); @@ -1404,8 +989,9 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, ValidPathInfo info; info.path = dstPath; - info.hash = hash.first; + info.narHash = hash.first; info.narSize = hash.second; + info.ultimate = true; registerValidPath(info); } @@ -1420,7 +1006,6 @@ Path LocalStore::addToStore(const string & name, const Path & _srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) { Path srcPath(absPath(_srcPath)); - debug(format("adding ‘%1%’ to the store") % srcPath); /* Read the whole path into memory. This is not a very scalable method for very large paths, but `copyPath' is mainly used for @@ -1429,9 +1014,9 @@ Path LocalStore::addToStore(const string & name, const Path & _srcPath, if (recursive) dumpPath(srcPath, sink, filter); else - sink.s = readFile(srcPath); + sink.s = make_ref<std::string>(readFile(srcPath)); - return addToStoreFromDump(sink.s, name, recursive, hashAlgo, repair); + return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair); } @@ -1444,25 +1029,28 @@ Path LocalStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(dstPath)) { - PathLocks outputLock(singleton<PathSet, Path>(dstPath)); + PathLocks outputLock({dstPath}); if (repair || !isValidPath(dstPath)) { - if (pathExists(dstPath)) deletePath(dstPath); + deletePath(dstPath); writeFile(dstPath, s); canonicalisePathMetaData(dstPath, -1); - HashResult hash = hashPath(htSHA256, dstPath); + StringSink sink; + dumpString(s, sink); + auto hash = hashString(htSHA256, *sink.s); optimisePath(dstPath); ValidPathInfo info; info.path = dstPath; - info.hash = hash.first; - info.narSize = hash.second; + info.narHash = hash; + info.narSize = sink.s->size(); info.references = references; + info.ultimate = true; registerValidPath(info); } @@ -1473,119 +1061,6 @@ Path LocalStore::addTextToStore(const string & name, const string & s, } -struct HashAndWriteSink : Sink -{ - Sink & writeSink; - HashSink hashSink; - HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) - { - } - virtual void operator () (const unsigned char * data, size_t len) - { - writeSink(data, len); - hashSink(data, len); - } - Hash currentHash() - { - return hashSink.currentHash().first; - } -}; - - -#define EXPORT_MAGIC 0x4558494e - - -static void checkSecrecy(const Path & path) -{ - struct stat st; - if (stat(path.c_str(), &st)) - throw SysError(format("getting status of ‘%1%’") % path); - if ((st.st_mode & (S_IRWXG | S_IRWXO)) != 0) - throw Error(format("file ‘%1%’ should be secret (inaccessible to everybody else)!") % path); -} - - -void LocalStore::exportPath(const Path & path, bool sign, - Sink & sink) -{ - assertStorePath(path); - - printMsg(lvlInfo, format("exporting path ‘%1%’") % path); - - if (!isValidPath(path)) - throw Error(format("path ‘%1%’ is not valid") % path); - - HashAndWriteSink hashAndWriteSink(sink); - - dumpPath(path, hashAndWriteSink); - - /* Refuse to export paths that have changed. This prevents - filesystem corruption from spreading to other machines. - Don't complain if the stored hash is zero (unknown). */ - Hash hash = hashAndWriteSink.currentHash(); - Hash storedHash = queryPathHash(path); - if (hash != storedHash && storedHash != Hash(storedHash.type)) - throw Error(format("hash of path ‘%1%’ has changed from ‘%2%’ to ‘%3%’!") % path - % printHash(storedHash) % printHash(hash)); - - writeInt(EXPORT_MAGIC, hashAndWriteSink); - - writeString(path, hashAndWriteSink); - - PathSet references; - queryReferences(path, references); - writeStrings(references, hashAndWriteSink); - - Path deriver = queryDeriver(path); - writeString(deriver, hashAndWriteSink); - - if (sign) { - Hash hash = hashAndWriteSink.currentHash(); - - writeInt(1, hashAndWriteSink); - - Path tmpDir = createTempDir(); - AutoDelete delTmp(tmpDir); - Path hashFile = tmpDir + "/hash"; - writeFile(hashFile, printHash(hash)); - - Path secretKey = settings.nixConfDir + "/signing-key.sec"; - checkSecrecy(secretKey); - - Strings args; - args.push_back("rsautl"); - args.push_back("-sign"); - args.push_back("-inkey"); - args.push_back(secretKey); - args.push_back("-in"); - args.push_back(hashFile); - string signature = runProgram(OPENSSL_PATH, true, args); - - writeString(signature, hashAndWriteSink); - - } else - writeInt(0, hashAndWriteSink); -} - - -struct HashAndReadSource : Source -{ - Source & readSource; - HashSink hashSink; - bool hashing; - HashAndReadSource(Source & readSource) : readSource(readSource), hashSink(htSHA256) - { - hashing = true; - } - size_t read(unsigned char * data, size_t len) - { - size_t n = readSource.read(data, len); - if (hashing) hashSink(data, n); - return n; - } -}; - - /* Create a temporary directory in the store that won't be garbage-collected. */ Path LocalStore::createTempDirInStore() @@ -1602,145 +1077,26 @@ Path LocalStore::createTempDirInStore() } -Path LocalStore::importPath(bool requireSignature, Source & source) -{ - HashAndReadSource hashAndReadSource(source); - - /* We don't yet know what store path this archive contains (the - store path follows the archive data proper), and besides, we - don't know yet whether the signature is valid. */ - Path tmpDir = createTempDirInStore(); - AutoDelete delTmp(tmpDir); - Path unpacked = tmpDir + "/unpacked"; - - restorePath(unpacked, hashAndReadSource); - - unsigned int magic = readInt(hashAndReadSource); - if (magic != EXPORT_MAGIC) - throw Error("Nix archive cannot be imported; wrong format"); - - Path dstPath = readStorePath(hashAndReadSource); - - PathSet references = readStorePaths<PathSet>(hashAndReadSource); - - Path deriver = readString(hashAndReadSource); - if (deriver != "") assertStorePath(deriver); - - Hash hash = hashAndReadSource.hashSink.finish().first; - hashAndReadSource.hashing = false; - - bool haveSignature = readInt(hashAndReadSource) == 1; - - if (requireSignature && !haveSignature) - throw Error(format("imported archive of ‘%1%’ lacks a signature") % dstPath); - - if (haveSignature) { - string signature = readString(hashAndReadSource); - - if (requireSignature) { - Path sigFile = tmpDir + "/sig"; - writeFile(sigFile, signature); - - Strings args; - args.push_back("rsautl"); - args.push_back("-verify"); - args.push_back("-inkey"); - args.push_back(settings.nixConfDir + "/signing-key.pub"); - args.push_back("-pubin"); - args.push_back("-in"); - args.push_back(sigFile); - string hash2 = runProgram(OPENSSL_PATH, true, args); - - /* Note: runProgram() throws an exception if the signature - is invalid. */ - - if (printHash(hash) != hash2) - throw Error( - "signed hash doesn't match actual contents of imported " - "archive; archive could be corrupt, or someone is trying " - "to import a Trojan horse"); - } - } - - /* Do the actual import. */ - - /* !!! way too much code duplication with addTextToStore() etc. */ - addTempRoot(dstPath); - - if (!isValidPath(dstPath)) { - - PathLocks outputLock; - - /* Lock the output path. But don't lock if we're being called - from a build hook (whose parent process already acquired a - lock on this path). */ - Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS")); - if (find(locksHeld.begin(), locksHeld.end(), dstPath) == locksHeld.end()) - outputLock.lockPaths(singleton<PathSet, Path>(dstPath)); - - if (!isValidPath(dstPath)) { - - if (pathExists(dstPath)) deletePath(dstPath); - - if (rename(unpacked.c_str(), dstPath.c_str()) == -1) - throw SysError(format("cannot move ‘%1%’ to ‘%2%’") - % unpacked % dstPath); - - canonicalisePathMetaData(dstPath, -1); - - /* !!! if we were clever, we could prevent the hashPath() - here. */ - HashResult hash = hashPath(htSHA256, dstPath); - - optimisePath(dstPath); // FIXME: combine with hashPath() - - ValidPathInfo info; - info.path = dstPath; - info.hash = hash.first; - info.narSize = hash.second; - info.references = references; - info.deriver = deriver != "" && isValidPath(deriver) ? deriver : ""; - registerValidPath(info); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - - -Paths LocalStore::importPaths(bool requireSignature, Source & source) -{ - Paths res; - while (true) { - unsigned long long n = readLongLong(source); - if (n == 0) break; - if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’"); - res.push_back(importPath(requireSignature, source)); - } - return res; -} - - void LocalStore::invalidatePathChecked(const Path & path) { assertStorePath(path); - retry_sqlite { - SQLiteTxn txn(db); + retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); - if (isValidPath_(path)) { - PathSet referrers; queryReferrers_(path, referrers); + if (isValidPath_(*state, path)) { + PathSet referrers; queryReferrers(*state, path, referrers); referrers.erase(path); /* ignore self-references */ if (!referrers.empty()) throw PathInUse(format("cannot delete path ‘%1%’ because it is in use by %2%") % path % showPaths(referrers)); - invalidatePath(path); + invalidatePath(*state, path); } txn.commit(); - } end_retry_sqlite; + }); } @@ -1761,8 +1117,8 @@ bool LocalStore::verifyStore(bool checkContents, bool repair) PathSet validPaths2 = queryAllValidPaths(), validPaths, done; - foreach (PathSet::iterator, i, validPaths2) - verifyPath(*i, store, done, validPaths, repair, errors); + for (auto & i : validPaths2) + verifyPath(i, store, done, validPaths, repair, errors); /* Release the GC lock so that checking content hashes (which can take ages) doesn't block the GC or builds. */ @@ -1774,45 +1130,48 @@ bool LocalStore::verifyStore(bool checkContents, bool repair) Hash nullHash(htSHA256); - foreach (PathSet::iterator, i, validPaths) { + for (auto & i : validPaths) { try { - ValidPathInfo info = queryPathInfo(*i); + auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(i))); /* Check the content hash (optionally - slow). */ - printMsg(lvlTalkative, format("checking contents of ‘%1%’") % *i); - HashResult current = hashPath(info.hash.type, *i); + printMsg(lvlTalkative, format("checking contents of ‘%1%’") % i); + HashResult current = hashPath(info->narHash.type, i); - if (info.hash != nullHash && info.hash != current.first) { + if (info->narHash != nullHash && info->narHash != current.first) { printMsg(lvlError, format("path ‘%1%’ was modified! " "expected hash ‘%2%’, got ‘%3%’") - % *i % printHash(info.hash) % printHash(current.first)); - if (repair) repairPath(*i); else errors = true; + % i % printHash(info->narHash) % printHash(current.first)); + if (repair) repairPath(i); else errors = true; } else { bool update = false; /* Fill in missing hashes. */ - if (info.hash == nullHash) { - printMsg(lvlError, format("fixing missing hash on ‘%1%’") % *i); - info.hash = current.first; + if (info->narHash == nullHash) { + printMsg(lvlError, format("fixing missing hash on ‘%1%’") % i); + info->narHash = current.first; update = true; } /* Fill in missing narSize fields (from old stores). */ - if (info.narSize == 0) { - printMsg(lvlError, format("updating size field on ‘%1%’ to %2%") % *i % current.second); - info.narSize = current.second; + if (info->narSize == 0) { + printMsg(lvlError, format("updating size field on ‘%1%’ to %2%") % i % current.second); + info->narSize = current.second; update = true; } - if (update) updatePathInfo(info); + if (update) { + auto state(_state.lock()); + updatePathInfo(*state, *info); + } } } catch (Error & e) { /* It's possible that the path got GC'ed, so ignore errors on invalid paths. */ - if (isValidPath(*i)) + if (isValidPath(i)) printMsg(lvlError, format("error: %1%") % e.msg()); else printMsg(lvlError, format("warning: %1%") % e.msg()); @@ -1835,7 +1194,8 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, if (!isStorePath(path)) { printMsg(lvlError, format("path ‘%1%’ is not in the Nix store") % path); - invalidatePath(path); + auto state(_state.lock()); + invalidatePath(*state, path); return; } @@ -1844,16 +1204,17 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, first, then we can invalidate this path as well. */ bool canInvalidate = true; PathSet referrers; queryReferrers(path, referrers); - foreach (PathSet::iterator, i, referrers) - if (*i != path) { - verifyPath(*i, store, done, validPaths, repair, errors); - if (validPaths.find(*i) != validPaths.end()) + for (auto & i : referrers) + if (i != path) { + verifyPath(i, store, done, validPaths, repair, errors); + if (validPaths.find(i) != validPaths.end()) canInvalidate = false; } if (canInvalidate) { printMsg(lvlError, format("path ‘%1%’ disappeared, removing from database...") % path); - invalidatePath(path); + auto state(_state.lock()); + invalidatePath(*state, path); } else { printMsg(lvlError, format("path ‘%1%’ disappeared, but it still has valid referrers!") % path); if (repair) @@ -1873,114 +1234,6 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, } -bool LocalStore::pathContentsGood(const Path & path) -{ - std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); - if (i != pathContentsGoodCache.end()) return i->second; - printMsg(lvlInfo, format("checking path ‘%1%’...") % path); - ValidPathInfo info = queryPathInfo(path); - bool res; - if (!pathExists(path)) - res = false; - else { - HashResult current = hashPath(info.hash.type, path); - Hash nullHash(htSHA256); - res = info.hash == nullHash || info.hash == current.first; - } - pathContentsGoodCache[path] = res; - if (!res) printMsg(lvlError, format("path ‘%1%’ is corrupted or missing!") % path); - return res; -} - - -void LocalStore::markContentsGood(const Path & path) -{ - pathContentsGoodCache[path] = true; -} - - -/* Functions for upgrading from the pre-SQLite database. */ - -PathSet LocalStore::queryValidPathsOld() -{ - PathSet paths; - for (auto & i : readDirectory(settings.nixDBPath + "/info")) - if (i.name.at(0) != '.') paths.insert(settings.nixStore + "/" + i.name); - return paths; -} - - -ValidPathInfo LocalStore::queryPathInfoOld(const Path & path) -{ - ValidPathInfo res; - res.path = path; - - /* Read the info file. */ - string baseName = baseNameOf(path); - Path infoFile = (format("%1%/info/%2%") % settings.nixDBPath % baseName).str(); - if (!pathExists(infoFile)) - throw Error(format("path ‘%1%’ is not valid") % path); - string info = readFile(infoFile); - - /* Parse it. */ - Strings lines = tokenizeString<Strings>(info, "\n"); - - foreach (Strings::iterator, i, lines) { - string::size_type p = i->find(':'); - if (p == string::npos) - throw Error(format("corrupt line in ‘%1%’: %2%") % infoFile % *i); - string name(*i, 0, p); - string value(*i, p + 2); - if (name == "References") { - Strings refs = tokenizeString<Strings>(value, " "); - res.references = PathSet(refs.begin(), refs.end()); - } else if (name == "Deriver") { - res.deriver = value; - } else if (name == "Hash") { - res.hash = parseHashField(path, value); - } else if (name == "Registered-At") { - int n = 0; - string2Int(value, n); - res.registrationTime = n; - } - } - - return res; -} - - -/* Upgrade from schema 5 (Nix 0.12) to schema 6 (Nix >= 0.15). */ -void LocalStore::upgradeStore6() -{ - printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); - - openDB(true); - - PathSet validPaths = queryValidPathsOld(); - - SQLiteTxn txn(db); - - foreach (PathSet::iterator, i, validPaths) { - addValidPath(queryPathInfoOld(*i), false); - std::cerr << "."; - } - - std::cerr << "|"; - - foreach (PathSet::iterator, i, validPaths) { - ValidPathInfo info = queryPathInfoOld(*i); - unsigned long long referrer = queryValidPathId(*i); - foreach (PathSet::iterator, j, info.references) - addReference(referrer, queryValidPathId(*j)); - std::cerr << "."; - } - - std::cerr << "\n"; - - txn.commit(); -} - - #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) static void makeMutable(const Path & path) @@ -2035,8 +1288,41 @@ void LocalStore::upgradeStore7() void LocalStore::vacuumDB() { - if (sqlite3_exec(db, "vacuum;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "vacuuming SQLite database"); + auto state(_state.lock()); + + if (sqlite3_exec(state->db, "vacuum;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "vacuuming SQLite database"); +} + + +void LocalStore::addSignatures(const Path & storePath, const StringSet & sigs) +{ + retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); + + auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(storePath))); + + info->sigs.insert(sigs.begin(), sigs.end()); + + updatePathInfo(*state, *info); + + txn.commit(); + }); +} + + +void LocalStore::signPathInfo(ValidPathInfo & info) +{ + // FIXME: keep secret keys in memory. + + auto secretKeyFiles = settings.get("secret-key-files", Strings()); + + for (auto & secretKeyFile : secretKeyFiles) { + SecretKey secretKey(readFile(secretKeyFile)); + info.sign(secretKey); + } } |