diff options
Diffstat (limited to 'third_party/nix/src/libstore/local-store.cc')
| -rw-r--r-- | third_party/nix/src/libstore/local-store.cc | 1534 |
1 files changed, 1534 insertions, 0 deletions
diff --git a/third_party/nix/src/libstore/local-store.cc b/third_party/nix/src/libstore/local-store.cc new file mode 100644 index 000000000000..1d162ad6be10 --- /dev/null +++ b/third_party/nix/src/libstore/local-store.cc @@ -0,0 +1,1534 @@ +#include "local-store.hh" + +#include <algorithm> +#include <cerrno> +#include <cstdio> +#include <cstring> +#include <ctime> +#include <iostream> + +#include <fcntl.h> +#include <glog/logging.h> +#include <grp.h> +#include <sys/select.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <sys/types.h> +#include <unistd.h> +#include <utime.h> + +#include "archive.hh" +#include "derivations.hh" +#include "globals.hh" +#include "nar-info.hh" +#include "pathlocks.hh" +#include "worker-protocol.hh" + +#if __linux__ +#include <sched.h> +#include <sys/ioctl.h> +#include <sys/mount.h> +#include <sys/statvfs.h> +#include <sys/xattr.h> +#endif + +#ifdef __CYGWIN__ +#include <windows.h> +#endif + +#include <sqlite3.h> + +namespace nix { + +LocalStore::LocalStore(const Params& params) + : Store(params), + LocalFSStore(params), + realStoreDir_{this, false, + rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", + "physical path to the Nix store"}, + realStoreDir(realStoreDir_), + dbDir(stateDir + "/db"), + linksDir(realStoreDir + "/.links"), + reservedPath(dbDir + "/reserved"), + schemaPath(dbDir + "/schema"), + trashDir(realStoreDir + "/trash"), + tempRootsDir(stateDir + "/temproots"), + fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) { + auto state(_state.lock()); + + /* Create missing state directories if they don't already exist. */ + createDirs(realStoreDir); + makeStoreWritable(); + createDirs(linksDir); + Path profilesDir = stateDir + "/profiles"; + createDirs(profilesDir); + createDirs(tempRootsDir); + createDirs(dbDir); + Path gcRootsDir = stateDir + "/gcroots"; + if (!pathExists(gcRootsDir)) { + createDirs(gcRootsDir); + createSymlink(profilesDir, gcRootsDir + "/profiles"); + } + + for (auto& perUserDir : + {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { + createDirs(perUserDir); + if (chmod(perUserDir.c_str(), 0755) == -1) { + throw SysError("could not set permissions on '%s' to 755", perUserDir); + } + } + + createUser(getUserName(), getuid()); + + /* Optionally, create directories and set permissions for a + multi-user install. */ + if (getuid() == 0 && settings.buildUsersGroup != "") { + mode_t perm = 01775; + + struct group* gr = getgrnam(settings.buildUsersGroup.get().c_str()); + if (gr == nullptr) { + LOG(ERROR) << "warning: the group '" << settings.buildUsersGroup + << "' specified in 'build-users-group' does not exist"; + } else { + struct stat st; + if (stat(realStoreDir.c_str(), &st) != 0) { + throw SysError(format("getting attributes of path '%1%'") % + realStoreDir); + } + + if (st.st_uid != 0 || st.st_gid != gr->gr_gid || + (st.st_mode & ~S_IFMT) != perm) { + if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1) { + throw SysError(format("changing ownership of path '%1%'") % + realStoreDir); + } + if (chmod(realStoreDir.c_str(), perm) == -1) { + throw SysError(format("changing permissions on path '%1%'") % + realStoreDir); + } + } + } + } + + /* Ensure that the store and its parents are not symlinks. */ + if (getEnv("NIX_IGNORE_SYMLINK_STORE") != "1") { + Path path = realStoreDir; + struct stat st; + while (path != "/") { + if (lstat(path.c_str(), &st) != 0) { + throw SysError(format("getting status of '%1%'") % path); + } + if (S_ISLNK(st.st_mode)) { + throw Error(format("the path '%1%' is a symlink; " + "this is not allowed for the Nix store and its " + "parent directories") % + path); + } + path = dirOf(path); + } + } + + /* We can't open a SQLite database if the disk is full. Since + this prevents the garbage collector from running when it's most + needed, we reserve some dummy space that we can free just + before doing a garbage collection. */ + try { + struct stat st; + if (stat(reservedPath.c_str(), &st) == -1 || + st.st_size != settings.reservedSize) { + AutoCloseFD fd = + open(reservedPath.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0600); + int res = -1; +#if HAVE_POSIX_FALLOCATE + res = posix_fallocate(fd.get(), 0, settings.reservedSize); +#endif + if (res == -1) { + writeFull(fd.get(), string(settings.reservedSize, 'X')); + [[gnu::unused]] auto res2 = ftruncate(fd.get(), settings.reservedSize); + } + } + } catch (SysError& e) { /* don't care about errors */ + } + + /* Acquire the big fat lock in shared mode to make sure that no + schema upgrade is in progress. */ + Path globalLockPath = dbDir + "/big-lock"; + globalLock = openLockFile(globalLockPath, true); + + if (!lockFile(globalLock.get(), ltRead, false)) { + LOG(INFO) << "waiting for the big Nix store lock..."; + lockFile(globalLock.get(), ltRead, true); + } + + /* Check the current database schema and if necessary do an + upgrade. */ + int curSchema = getSchema(); + if (curSchema > nixSchemaVersion) { + throw Error( + format( + "current Nix store schema is version %1%, but I only support %2%") % + curSchema % nixSchemaVersion); + } + if (curSchema == 0) { /* new store */ + curSchema = nixSchemaVersion; + openDB(*state, true); + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + } else if (curSchema < nixSchemaVersion) { + if (curSchema < 5) { + throw Error( + "Your Nix store has a database in Berkeley DB format,\n" + "which is no longer supported. To convert to the new format,\n" + "please upgrade Nix to version 0.12 first."); + } + + if (curSchema < 6) { + throw Error( + "Your Nix store has a database in flat file format,\n" + "which is no longer supported. To convert to the new format,\n" + "please upgrade Nix to version 1.11 first."); + } + + if (!lockFile(globalLock.get(), ltWrite, false)) { + LOG(INFO) << "waiting for exclusive access to the Nix store..."; + lockFile(globalLock.get(), ltWrite, true); + } + + /* Get the schema version again, because another process may + have performed the upgrade already. */ + curSchema = getSchema(); + + if (curSchema < 7) { + upgradeStore7(); + } + + openDB(*state, false); + + if (curSchema < 8) { + SQLiteTxn txn(state->db); + state->db.exec("alter table ValidPaths add column ultimate integer"); + state->db.exec("alter table ValidPaths add column sigs text"); + txn.commit(); + } + + if (curSchema < 9) { + SQLiteTxn txn(state->db); + state->db.exec("drop table FailedPaths"); + txn.commit(); + } + + if (curSchema < 10) { + SQLiteTxn txn(state->db); + state->db.exec("alter table ValidPaths add column ca text"); + txn.commit(); + } + + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + + lockFile(globalLock.get(), ltRead, true); + } else { + openDB(*state, false); + } + + /* Prepare SQL statements. */ + state->stmtRegisterValidPath.create( + state->db, + "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, " + "ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);"); + state->stmtUpdatePathInfo.create( + state->db, + "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca " + "= ? where path = ?;"); + state->stmtAddReference.create( + state->db, + "insert or replace into Refs (referrer, reference) values (?, ?);"); + state->stmtQueryPathInfo.create( + state->db, + "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca " + "from ValidPaths where path = ?;"); + state->stmtQueryReferences.create(state->db, + "select path from Refs join ValidPaths on " + "reference = id where referrer = ?;"); + state->stmtQueryReferrers.create( + state->db, + "select path from Refs join ValidPaths on referrer = id where reference " + "= (select id from ValidPaths where path = ?);"); + state->stmtInvalidatePath.create(state->db, + "delete from ValidPaths where path = ?;"); + state->stmtAddDerivationOutput.create( + state->db, + "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, " + "?);"); + state->stmtQueryValidDerivers.create( + state->db, + "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv " + "= v.id where d.path = ?;"); + state->stmtQueryDerivationOutputs.create( + state->db, "select id, path from DerivationOutputs where drv = ?;"); + // Use "path >= ?" with limit 1 rather than "path like '?%'" to + // ensure efficient lookup. + state->stmtQueryPathFromHashPart.create( + state->db, "select path from ValidPaths where path >= ? limit 1;"); + state->stmtQueryValidPaths.create(state->db, "select path from ValidPaths"); +} + +LocalStore::~LocalStore() { + std::shared_future<void> future; + + { + auto state(_state.lock()); + if (state->gcRunning) { + future = state->gcFuture; + } + } + + if (future.valid()) { + LOG(INFO) << "waiting for auto-GC to finish on exit..."; + future.get(); + } + + try { + auto state(_state.lock()); + if (state->fdTempRoots) { + state->fdTempRoots = -1; + unlink(fnTempRoots.c_str()); + } + } catch (...) { + ignoreException(); + } +} + +std::string LocalStore::getUri() { return "local"; } + +int LocalStore::getSchema() { + int curSchema = 0; + if (pathExists(schemaPath)) { + string s = readFile(schemaPath); + if (!string2Int(s, curSchema)) { + throw Error(format("'%1%' is corrupt") % schemaPath); + } + } + return curSchema; +} + +void LocalStore::openDB(State& state, bool create) { + if (access(dbDir.c_str(), R_OK | W_OK) != 0) { + throw SysError(format("Nix database directory '%1%' is not writable") % + dbDir); + } + + /* Open the Nix database. */ + string dbPath = dbDir + "/db.sqlite"; + auto& db(state.db); + if (sqlite3_open_v2(dbPath.c_str(), &db.db, + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), + nullptr) != SQLITE_OK) { + throw Error(format("cannot open Nix database '%1%'") % dbPath); + } + +#ifdef __CYGWIN__ + /* The cygwin version of sqlite3 has a patch which calls + SetDllDirectory("/usr/bin") on init. It was intended to fix extension + loading, which we don't use, and the effect of SetDllDirectory is + inherited by child processes, and causes libraries to be loaded from + /usr/bin instead of $PATH. This breaks quite a few things (e.g. + checkPhase on openssh), so we set it back to default behaviour. */ + SetDllDirectoryW(L""); +#endif + + if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) { + throwSQLiteError(db, "setting timeout"); + } + + db.exec("pragma foreign_keys = 1"); + + /* !!! check whether sqlite has been built with foreign key + support */ + + /* Whether SQLite should fsync(). "Normal" synchronous mode + should be safe enough. If the user asks for it, don't sync at + all. This can cause database corruption if the system + crashes. */ + string syncMode = settings.fsyncMetadata ? "normal" : "off"; + db.exec("pragma synchronous = " + syncMode); + + /* Set the SQLite journal mode. WAL mode is fastest, so it's the + default. */ + string mode = settings.useSQLiteWAL ? "wal" : "truncate"; + string prevMode; + { + SQLiteStmt stmt; + stmt.create(db, "pragma main.journal_mode;"); + if (sqlite3_step(stmt) != SQLITE_ROW) { + throwSQLiteError(db, "querying journal mode"); + } + prevMode = string((const char*)sqlite3_column_text(stmt, 0)); + } + if (prevMode != mode && + sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), + nullptr, nullptr, nullptr) != SQLITE_OK) { + throwSQLiteError(db, "setting journal mode"); + } + + /* Increase the auto-checkpoint interval to 40000 pages. This + seems enough to ensure that instantiating the NixOS system + derivation is done in a single fsync(). */ + if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 40000;", + nullptr, nullptr, nullptr) != SQLITE_OK) { + throwSQLiteError(db, "setting autocheckpoint interval"); + } + + /* Initialise the database schema, if necessary. */ + if (create) { + const char* schema = +#include "schema.sql.gen.hh" + ; + db.exec(schema); + } +} + +/* To improve purity, users may want to make the Nix store a read-only + bind mount. So make the Nix store writable for this process. */ +void LocalStore::makeStoreWritable() { +#if __linux__ + if (getuid() != 0) { + return; + } + /* Check if /nix/store is on a read-only mount. */ + struct statvfs stat; + if (statvfs(realStoreDir.c_str(), &stat) != 0) { + throw SysError("getting info about the Nix store mount point"); + } + + if ((stat.f_flag & ST_RDONLY) != 0u) { + if (unshare(CLONE_NEWNS) == -1) { + throw SysError("setting up a private mount namespace"); + } + + if (mount(nullptr, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, + nullptr) == -1) { + throw SysError(format("remounting %1% writable") % realStoreDir); + } + } +#endif +} + +const time_t mtimeStore = 1; /* 1 second into the epoch */ + +static void canonicaliseTimestampAndPermissions(const Path& path, + const struct stat& st) { + if (!S_ISLNK(st.st_mode)) { + /* Mask out all type related bits. */ + mode_t mode = st.st_mode & ~S_IFMT; + + if (mode != 0444 && mode != 0555) { + mode = (st.st_mode & S_IFMT) | 0444 | + ((st.st_mode & S_IXUSR) != 0u ? 0111 : 0); + if (chmod(path.c_str(), mode) == -1) { + throw SysError(format("changing mode of '%1%' to %2$o") % path % mode); + } + } + } + + if (st.st_mtime != mtimeStore) { + struct timeval times[2]; + times[0].tv_sec = st.st_atime; + times[0].tv_usec = 0; + times[1].tv_sec = mtimeStore; + times[1].tv_usec = 0; +#if HAVE_LUTIMES + if (lutimes(path.c_str(), times) == -1) { + if (errno != ENOSYS || + (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) { +#else + if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) +#endif + throw SysError(format("changing modification time of '%1%'") % path); + } + } + } // namespace nix +} + +void canonicaliseTimestampAndPermissions(const Path& path) { + struct stat st; + if (lstat(path.c_str(), &st) != 0) { + throw SysError(format("getting attributes of path '%1%'") % path); + } + canonicaliseTimestampAndPermissions(path, st); +} + +static void canonicalisePathMetaData_(const Path& path, uid_t fromUid, + InodesSeen& inodesSeen) { + checkInterrupt(); + +#if __APPLE__ + /* Remove flags, in particular UF_IMMUTABLE which would prevent + the file from being garbage-collected. FIXME: Use + setattrlist() to remove other attributes as well. */ + if (lchflags(path.c_str(), 0)) { + if (errno != ENOTSUP) + throw SysError(format("clearing flags of path '%1%'") % path); + } +#endif + + struct stat st; + if (lstat(path.c_str(), &st) != 0) { + throw SysError(format("getting attributes of path '%1%'") % path); + } + + /* Really make sure that the path is of a supported type. */ + if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) { + throw Error(format("file '%1%' has an unsupported type") % path); + } + +#if __linux__ + /* Remove extended attributes / ACLs. */ + ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); + + if (eaSize < 0) { + if (errno != ENOTSUP && errno != ENODATA) { + throw SysError("querying extended attributes of '%s'", path); + } + } else if (eaSize > 0) { + std::vector<char> eaBuf(eaSize); + + if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) { + throw SysError("querying extended attributes of '%s'", path); + } + + for (auto& eaName : tokenizeString<Strings>( + std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { + /* Ignore SELinux security labels since these cannot be + removed even by root. */ + if (eaName == "security.selinux") { + continue; + } + if (lremovexattr(path.c_str(), eaName.c_str()) == -1) { + throw SysError("removing extended attribute '%s' from '%s'", eaName, + path); + } + } + } +#endif + + /* Fail if the file is not owned by the build user. This prevents + us from messing up the ownership/permissions of files + hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). + However, ignore files that we chown'ed ourselves previously to + ensure that we don't fail on hard links within the same build + (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ + if (fromUid != (uid_t)-1 && st.st_uid != fromUid) { + assert(!S_ISDIR(st.st_mode)); + if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) { + throw BuildError(format("invalid ownership on file '%1%'") % path); + } + mode_t mode = st.st_mode & ~S_IFMT; + assert(S_ISLNK(st.st_mode) || + (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && + st.st_mtime == mtimeStore)); + return; + } + + inodesSeen.insert(Inode(st.st_dev, st.st_ino)); + + canonicaliseTimestampAndPermissions(path, st); + + /* Change ownership to the current uid. If it's a symlink, use + lchown if available, otherwise don't bother. Wrong ownership + of a symlink doesn't matter, since the owning user can't change + the symlink and can't delete it because the directory is not + writable. The only exception is top-level paths in the Nix + store (since that directory is group-writable for the Nix build + users group); we check for this case below. */ + if (st.st_uid != geteuid()) { +#if HAVE_LCHOWN + if (lchown(path.c_str(), geteuid(), getegid()) == -1) { +#else + if (!S_ISLNK(st.st_mode) && chown(path.c_str(), geteuid(), getegid()) == -1) +#endif + throw SysError(format("changing owner of '%1%' to %2%") % path % + geteuid()); + } + } + + if (S_ISDIR(st.st_mode)) { + DirEntries entries = readDirectory(path); + for (auto& i : entries) { + canonicalisePathMetaData_(path + "/" + i.name, fromUid, inodesSeen); + } + } +} + +void canonicalisePathMetaData(const Path& path, uid_t fromUid, + InodesSeen& inodesSeen) { + canonicalisePathMetaData_(path, fromUid, inodesSeen); + + /* On platforms that don't have lchown(), the top-level path can't + be a symlink, since we can't change its ownership. */ + struct stat st; + if (lstat(path.c_str(), &st) != 0) { + throw SysError(format("getting attributes of path '%1%'") % path); + } + + if (st.st_uid != geteuid()) { + assert(S_ISLNK(st.st_mode)); + throw Error(format("wrong ownership of top-level store path '%1%'") % path); + } +} + +void canonicalisePathMetaData(const Path& path, uid_t fromUid) { + InodesSeen inodesSeen; + canonicalisePathMetaData(path, fromUid, inodesSeen); +} + +void LocalStore::checkDerivationOutputs(const Path& drvPath, + const Derivation& drv) { + string drvName = storePathToName(drvPath); + assert(isDerivation(drvName)); + drvName = string(drvName, 0, drvName.size() - drvExtension.size()); + + if (drv.isFixedOutput()) { + auto out = drv.outputs.find("out"); + if (out == drv.outputs.end()) { + throw Error( + format("derivation '%1%' does not have an output named 'out'") % + drvPath); + } + + bool recursive; + Hash h; + out->second.parseHashInfo(recursive, h); + Path outPath = makeFixedOutputPath(recursive, h, drvName); + + auto j = drv.env.find("out"); + if (out->second.path != outPath || j == drv.env.end() || + j->second != outPath) { + throw Error( + format( + "derivation '%1%' has incorrect output '%2%', should be '%3%'") % + drvPath % out->second.path % outPath); + } + } + + else { + Derivation drvCopy(drv); + for (auto& i : drvCopy.outputs) { + i.second.path = ""; + drvCopy.env[i.first] = ""; + } + + Hash h = hashDerivationModulo(*this, drvCopy); + + for (auto& i : drv.outputs) { + Path outPath = makeOutputPath(i.first, h, drvName); + auto j = drv.env.find(i.first); + if (i.second.path != outPath || j == drv.env.end() || + j->second != outPath) { + throw Error(format("derivation '%1%' has incorrect output '%2%', " + "should be '%3%'") % + drvPath % i.second.path % outPath); + } + } + } +} + +uint64_t LocalStore::addValidPath(State& state, const ValidPathInfo& info, + bool checkOutputs) { + if (!info.ca.empty() && !info.isContentAddressed(*this)) { + throw Error( + "cannot add path '%s' to the Nix store because it claims to be " + "content-addressed but isn't", + info.path); + } + + state.stmtRegisterValidPath + .use()(info.path)(info.narHash.to_string(Base16))( + info.registrationTime == 0 ? time(nullptr) : info.registrationTime)( + info.deriver, !info.deriver.empty())(info.narSize, info.narSize != 0)( + info.ultimate ? 1 : 0, info.ultimate)( + concatStringsSep(" ", info.sigs), !info.sigs.empty())( + info.ca, !info.ca.empty()) + .exec(); + uint64_t id = sqlite3_last_insert_rowid(state.db); + + /* If this is a derivation, then store the derivation outputs in + the database. This is useful for the garbage collector: it can + efficiently query whether a path is an output of some + derivation. */ + if (isDerivation(info.path)) { + Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(info.path)); + + /* Verify that the output paths in the derivation are correct + (i.e., follow the scheme for computing output paths from + derivations). Note that if this throws an error, then the + DB transaction is rolled back, so the path validity + registration above is undone. */ + if (checkOutputs) { + checkDerivationOutputs(info.path, drv); + } + + for (auto& i : drv.outputs) { + state.stmtAddDerivationOutput.use()(id)(i.first)(i.second.path).exec(); + } + } + + { + auto state_(Store::state.lock()); + state_->pathInfoCache.upsert(storePathToHash(info.path), + std::make_shared<ValidPathInfo>(info)); + } + + return id; +} + +void LocalStore::queryPathInfoUncached( + const Path& path, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept { + try { + auto info = std::make_shared<ValidPathInfo>(); + info->path = path; + + assertStorePath(path); + + callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { + auto state(_state.lock()); + + /* Get the path info. */ + auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path)); + + if (!useQueryPathInfo.next()) { + return std::shared_ptr<ValidPathInfo>(); + } + + info->id = useQueryPathInfo.getInt(0); + + try { + info->narHash = Hash(useQueryPathInfo.getStr(1)); + } catch (BadHash& e) { + throw Error("in valid-path entry for '%s': %s", path, e.what()); + } + + info->registrationTime = useQueryPathInfo.getInt(2); + + auto s = (const char*)sqlite3_column_text(state->stmtQueryPathInfo, 3); + if (s != nullptr) { + info->deriver = s; + } + + /* Note that narSize = NULL yields 0. */ + info->narSize = useQueryPathInfo.getInt(4); + + info->ultimate = useQueryPathInfo.getInt(5) == 1; + + s = (const char*)sqlite3_column_text(state->stmtQueryPathInfo, 6); + if (s != nullptr) { + info->sigs = tokenizeString<StringSet>(s, " "); + } + + s = (const char*)sqlite3_column_text(state->stmtQueryPathInfo, 7); + if (s != nullptr) { + info->ca = s; + } + + /* Get the references. */ + auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); + + while (useQueryReferences.next()) { + info->references.insert(useQueryReferences.getStr(0)); + } + + return info; + })); + + } catch (...) { + callback.rethrow(); + } +} + +/* Update path info in the database. */ +void LocalStore::updatePathInfo(State& state, const ValidPathInfo& info) { + state.stmtUpdatePathInfo + .use()(info.narSize, info.narSize != 0)(info.narHash.to_string(Base16))( + info.ultimate ? 1 : 0, info.ultimate)( + concatStringsSep(" ", info.sigs), !info.sigs.empty())( + info.ca, !info.ca.empty())(info.path) + .exec(); +} + +uint64_t LocalStore::queryValidPathId(State& state, const Path& path) { + auto use(state.stmtQueryPathInfo.use()(path)); + if (!use.next()) { + throw Error(format("path '%1%' is not valid") % path); + } + return use.getInt(0); +} + +bool LocalStore::isValidPath_(State& state, const Path& path) { + return state.stmtQueryPathInfo.use()(path).next(); +} + +bool LocalStore::isValidPathUncached(const Path& path) { + return retrySQLite<bool>([&]() { + auto state(_state.lock()); + return isValidPath_(*state, path); + }); +} + +PathSet LocalStore::queryValidPaths(const PathSet& paths, + SubstituteFlag maybeSubstitute) { + PathSet res; + for (auto& i : paths) { + if (isValidPath(i)) { + res.insert(i); + } + } + return res; +} + +PathSet LocalStore::queryAllValidPaths() { + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); + auto use(state->stmtQueryValidPaths.use()); + PathSet res; + while (use.next()) { + res.insert(use.getStr(0)); + } + return res; + }); +} + +void LocalStore::queryReferrers(State& state, const Path& path, + PathSet& referrers) { + auto useQueryReferrers(state.stmtQueryReferrers.use()(path)); + + while (useQueryReferrers.next()) { + referrers.insert(useQueryReferrers.getStr(0)); + } +} + +void LocalStore::queryReferrers(const Path& path, PathSet& referrers) { + assertStorePath(path); + return retrySQLite<void>([&]() { + auto state(_state.lock()); + queryReferrers(*state, path, referrers); + }); +} + +PathSet LocalStore::queryValidDerivers(const Path& path) { + assertStorePath(path); + + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); + + auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(path)); + + PathSet derivers; + while (useQueryValidDerivers.next()) { + derivers.insert(useQueryValidDerivers.getStr(1)); + } + + return derivers; + }); +} + +PathSet LocalStore::queryDerivationOutputs(const Path& path) { + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); + + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()( + queryValidPathId(*state, path))); + + PathSet outputs; + while (useQueryDerivationOutputs.next()) { + outputs.insert(useQueryDerivationOutputs.getStr(1)); + } + + return outputs; + }); +} + +StringSet LocalStore::queryDerivationOutputNames(const Path& path) { + return retrySQLite<StringSet>([&]() { + auto state(_state.lock()); + + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()( + queryValidPathId(*state, path))); + + StringSet outputNames; + while (useQueryDerivationOutputs.next()) { + outputNames.insert(useQueryDerivationOutputs.getStr(0)); + } + + return outputNames; + }); +} + +Path LocalStore::queryPathFromHashPart(const string& hashPart) { + if (hashPart.size() != storePathHashLen) { + throw Error("invalid hash part"); + } + + Path prefix = storeDir + "/" + hashPart; + + return retrySQLite<Path>([&]() -> std::string { + auto state(_state.lock()); + + auto useQueryPathFromHashPart( + state->stmtQueryPathFromHashPart.use()(prefix)); + + if (!useQueryPathFromHashPart.next()) { + return ""; + } + + const char* s = + (const char*)sqlite3_column_text(state->stmtQueryPathFromHashPart, 0); + return (s != nullptr) && + prefix.compare(0, prefix.size(), s, prefix.size()) == 0 + ? s + : ""; + }); +} + +PathSet LocalStore::querySubstitutablePaths(const PathSet& paths) { + if (!settings.useSubstitutes) { + return PathSet(); + } + + auto remaining = paths; + PathSet res; + + for (auto& sub : getDefaultSubstituters()) { + if (remaining.empty()) { + break; + } + if (sub->storeDir != storeDir) { + continue; + } + if (!sub->wantMassQuery()) { + continue; + } + + auto valid = sub->queryValidPaths(remaining); + + PathSet remaining2; + for (auto& path : remaining) { + if (valid.count(path) != 0u) { + res.insert(path); + } else { + remaining2.insert(path); + } + } + + std::swap(remaining, remaining2); + } + + return res; +} + +void LocalStore::querySubstitutablePathInfos(const PathSet& paths, + SubstitutablePathInfos& infos) { + if (!settings.useSubstitutes) { + return; + } + for (auto& sub : getDefaultSubstituters()) { + if (sub->storeDir != storeDir) { + continue; + } + for (auto& path : paths) { + if (infos.count(path) != 0u) { + continue; + } + DLOG(INFO) << "checking substituter '" << sub->getUri() << "' for path '" + << path << "'"; + try { + auto info = sub->queryPathInfo(path); + auto narInfo = std::dynamic_pointer_cast<const NarInfo>( + std::shared_ptr<const ValidPathInfo>(info)); + infos[path] = SubstitutablePathInfo{info->deriver, info->references, + narInfo ? narInfo->fileSize : 0, + info->narSize}; + } catch (InvalidPath&) { + } catch (SubstituterDisabled&) { + } catch (Error& e) { + if (settings.tryFallback) { + LOG(ERROR) << e.what(); + } else { + throw; + } + } + } + } +} + +void LocalStore::registerValidPath(const ValidPathInfo& info) { + ValidPathInfos infos; + infos.push_back(info); + registerValidPaths(infos); +} + +void LocalStore::registerValidPaths(const ValidPathInfos& infos) { + /* SQLite will fsync by default, but the new valid paths may not + be fsync-ed. So some may want to fsync them before registering + the validity, at the expense of some speed of the path + registering operation. */ + if (settings.syncBeforeRegistering) { + sync(); + } + + return retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); + PathSet paths; + + for (auto& i : infos) { + assert(i.narHash.type == htSHA256); + if (isValidPath_(*state, i.path)) { + updatePathInfo(*state, i); + } else { + addValidPath(*state, i, false); + } + paths.insert(i.path); + } + + for (auto& i : infos) { + auto referrer = queryValidPathId(*state, i.path); + for (auto& j : i.references) { + state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)) + .exec(); + } + } + + /* Check that the derivation outputs are correct. We can't do + this in addValidPath() above, because the references might + not be valid yet. */ + for (auto& i : infos) { + if (isDerivation(i.path)) { + // FIXME: inefficient; we already loaded the + // derivation in addValidPath(). + Derivation drv = + readDerivation(realStoreDir + "/" + baseNameOf(i.path)); + checkDerivationOutputs(i.path, drv); + } + } + + /* Do a topological sort of the paths. This will throw an + error if a cycle is detected and roll back the + transaction. Cycles can only occur when a derivation + has multiple outputs. */ + topoSortPaths(paths); + + txn.commit(); + }); +} + +/* Invalidate a path. The caller is responsible for checking that + there are no referrers. */ +void LocalStore::invalidatePath(State& state, const Path& path) { + LOG(INFO) << "invalidating path '" << path << "'"; + + state.stmtInvalidatePath.use()(path).exec(); + + /* Note that the foreign key constraints on the Refs table take + care of deleting the references entries for `path'. */ + + { + auto state_(Store::state.lock()); + state_->pathInfoCache.erase(storePathToHash(path)); + } +} + +const PublicKeys& LocalStore::getPublicKeys() { + auto state(_state.lock()); + if (!state->publicKeys) { + state->publicKeys = std::make_unique<PublicKeys>(getDefaultPublicKeys()); + } + return *state->publicKeys; +} + +void LocalStore::addToStore(const ValidPathInfo& info, Source& source, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) { + if (!info.narHash) { + throw Error("cannot add path '%s' because it lacks a hash", info.path); + } + + if (requireSigs && (checkSigs != 0u) && + (info.checkSignatures(*this, getPublicKeys()) == 0u)) { + throw Error("cannot add path '%s' because it lacks a valid signature", + info.path); + } + + addTempRoot(info.path); + + if ((repair != 0u) || !isValidPath(info.path)) { + PathLocks outputLock; + + Path realPath = realStoreDir + "/" + baseNameOf(info.path); + + /* Lock the output path. But don't lock if we're being called + from a build hook (whose parent process already acquired a + lock on this path). */ + if (locksHeld.count(info.path) == 0u) { + outputLock.lockPaths({realPath}); + } + + if ((repair != 0u) || !isValidPath(info.path)) { + deletePath(realPath); + + /* While restoring the path from the NAR, compute the hash + of the NAR. */ + HashSink hashSink(htSHA256); + + LambdaSource wrapperSource( + [&](unsigned char* data, size_t len) -> size_t { + size_t n = source.read(data, len); + hashSink(data, n); + return n; + }); + + restorePath(realPath, wrapperSource); + + auto hashResult = hashSink.finish(); + + if (hashResult.first != info.narHash) { + throw Error( + "hash mismatch importing path '%s';\n wanted: %s\n got: %s", + info.path, info.narHash.to_string(), hashResult.first.to_string()); + } + + if (hashResult.second != info.narSize) { + throw Error( + "size mismatch importing path '%s';\n wanted: %s\n got: %s", + info.path, info.narSize, hashResult.second); + } + + autoGC(); + + canonicalisePathMetaData(realPath, -1); + + optimisePath(realPath); // FIXME: combine with hashPath() + + registerValidPath(info); + } + + outputLock.setDeletion(true); + } +} + +Path LocalStore::addToStoreFromDump(const string& dump, const string& name, + bool recursive, HashType hashAlgo, + RepairFlag repair) { + Hash h = hashString(hashAlgo, dump); + + Path dstPath = makeFixedOutputPath(recursive, h, name); + + addTempRoot(dstPath); + + if ((repair != 0u) || !isValidPath(dstPath)) { + /* The first check above is an optimisation to prevent + unnecessary lock acquisition. */ + + Path realPath = realStoreDir + "/" + baseNameOf(dstPath); + + PathLocks outputLock({realPath}); + + if ((repair != 0u) || !isValidPath(dstPath)) { + deletePath(realPath); + + autoGC(); + + if (recursive) { + StringSource source(dump); + restorePath(realPath, source); + } else { + writeFile(realPath, dump); + } + + canonicalisePathMetaData(realPath, -1); + + /* Register the SHA-256 hash of the NAR serialisation of + the path in the database. We may just have computed it + above (if called with recursive == true and hashAlgo == + sha256); otherwise, compute it here. */ + HashResult hash; + if (recursive) { + hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); + hash.second = dump.size(); + } else { + hash = hashPath(htSHA256, realPath); + } + + optimisePath(realPath); // FIXME: combine with hashPath() + + ValidPathInfo info; + info.path = dstPath; + info.narHash = hash.first; + info.narSize = hash.second; + info.ca = makeFixedOutputCA(recursive, h); + registerValidPath(info); + } + + outputLock.setDeletion(true); + } + + return dstPath; +} + +Path LocalStore::addToStore(const string& name, const Path& _srcPath, + bool recursive, HashType hashAlgo, + PathFilter& filter, RepairFlag repair) { + Path srcPath(absPath(_srcPath)); + + /* Read the whole path into memory. This is not a very scalable + method for very large paths, but `copyPath' is mainly used for + small files. */ + StringSink sink; + if (recursive) { + dumpPath(srcPath, sink, filter); + } else { + sink.s = make_ref<std::string>(readFile(srcPath)); + } + + return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair); +} + +Path LocalStore::addTextToStore(const string& name, const string& s, + const PathSet& references, RepairFlag repair) { + auto hash = hashString(htSHA256, s); + auto dstPath = makeTextPath(name, hash, references); + + addTempRoot(dstPath); + + if ((repair != 0u) || !isValidPath(dstPath)) { + Path realPath = realStoreDir + "/" + baseNameOf(dstPath); + + PathLocks outputLock({realPath}); + + if ((repair != 0u) || !isValidPath(dstPath)) { + deletePath(realPath); + + autoGC(); + + writeFile(realPath, s); + + canonicalisePathMetaData(realPath, -1); + + StringSink sink; + dumpString(s, sink); + auto narHash = hashString(htSHA256, *sink.s); + + optimisePath(realPath); + + ValidPathInfo info; + info.path = dstPath; + info.narHash = narHash; + info.narSize = sink.s->size(); + info.references = references; + info.ca = "text:" + hash.to_string(); + registerValidPath(info); + } + + outputLock.setDeletion(true); + } + + return dstPath; +} + +/* Create a temporary directory in the store that won't be + garbage-collected. */ +Path LocalStore::createTempDirInStore() { + Path tmpDir; + do { + /* There is a slight possibility that `tmpDir' gets deleted by + the GC between createTempDir() and addTempRoot(), so repeat + until `tmpDir' exists. */ + tmpDir = createTempDir(realStoreDir); + addTempRoot(tmpDir); + } while (!pathExists(tmpDir)); + return tmpDir; +} + +void LocalStore::invalidatePathChecked(const Path& path) { + assertStorePath(path); + + retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); + + if (isValidPath_(*state, path)) { + PathSet referrers; + queryReferrers(*state, path, referrers); + referrers.erase(path); /* ignore self-references */ + if (!referrers.empty()) { + throw PathInUse( + format("cannot delete path '%1%' because it is in use by %2%") % + path % showPaths(referrers)); + } + invalidatePath(*state, path); + } + + txn.commit(); + }); +} + +bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { + LOG(INFO) << "reading the Nix store..."; + + bool errors = false; + + /* Acquire the global GC lock to get a consistent snapshot of + existing and valid paths. */ + AutoCloseFD fdGCLock = openGCLock(ltWrite); + + PathSet store; + for (auto& i : readDirectory(realStoreDir)) { + store.insert(i.name); + } + + /* Check whether all valid paths actually exist. */ + LOG(INFO) << "checking path existence..."; + + PathSet validPaths2 = queryAllValidPaths(); + PathSet validPaths; + PathSet done; + + fdGCLock = -1; + + for (auto& i : validPaths2) { + verifyPath(i, store, done, validPaths, repair, errors); + } + + /* Optionally, check the content hashes (slow). */ + if (checkContents) { + LOG(INFO) << "checking hashes..."; + + Hash nullHash(htSHA256); + + for (auto& i : validPaths) { + try { + auto info = std::const_pointer_cast<ValidPathInfo>( + std::shared_ptr<const ValidPathInfo>(queryPathInfo(i))); + + /* Check the content hash (optionally - slow). */ + DLOG(INFO) << "checking contents of '" << i << "'"; + HashResult current = hashPath(info->narHash.type, toRealPath(i)); + + if (info->narHash != nullHash && info->narHash != current.first) { + LOG(ERROR) << "path '" << i << "' was modified! expected hash '" + << info->narHash.to_string() << "', got '" + << current.first.to_string() << "'"; + if (repair != 0u) { + repairPath(i); + } else { + errors = true; + } + } else { + bool update = false; + + /* Fill in missing hashes. */ + if (info->narHash == nullHash) { + LOG(WARNING) << "fixing missing hash on '" << i << "'"; + info->narHash = current.first; + update = true; + } + + /* Fill in missing narSize fields (from old stores). */ + if (info->narSize == 0) { + LOG(ERROR) << "updating size field on '" << i << "' to " + << current.second; + info->narSize = current.second; + update = true; + } + + if (update) { + auto state(_state.lock()); + updatePathInfo(*state, *info); + } + } + + } catch (Error& e) { + /* It's possible that the path got GC'ed, so ignore + errors on invalid paths. */ + if (isValidPath(i)) { + LOG(ERROR) << e.msg(); + } else { + LOG(WARNING) << e.msg(); + } + errors = true; + } + } + } + + return errors; +} + +void LocalStore::verifyPath(const Path& path, const PathSet& store, + PathSet& done, PathSet& validPaths, + RepairFlag repair, bool& errors) { + checkInterrupt(); + + if (done.find(path) != done.end()) { + return; + } + done.insert(path); + + if (!isStorePath(path)) { + LOG(ERROR) << "path '" << path << "' is not in the Nix store"; + auto state(_state.lock()); + invalidatePath(*state, path); + return; + } + + if (store.find(baseNameOf(path)) == store.end()) { + /* Check any referrers first. If we can invalidate them + first, then we can invalidate this path as well. */ + bool canInvalidate = true; + PathSet referrers; + queryReferrers(path, referrers); + for (auto& i : referrers) { + if (i != path) { + verifyPath(i, store, done, validPaths, repair, errors); + if (validPaths.find(i) != validPaths.end()) { + canInvalidate = false; + } + } + } + + if (canInvalidate) { + LOG(WARNING) << "path '" << path + << "' disappeared, removing from database..."; + auto state(_state.lock()); + invalidatePath(*state, path); + } else { + LOG(ERROR) << "path '" << path + << "' disappeared, but it still has valid referrers!"; + if (repair != 0u) { + try { + repairPath(path); + } catch (Error& e) { + LOG(WARNING) << e.msg(); + errors = true; + } + } else { + errors = true; + } + } + + return; + } + + validPaths.insert(path); +} + +unsigned int LocalStore::getProtocol() { return PROTOCOL_VERSION; } + +#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && \ + defined(FS_IMMUTABLE_FL) + +static void makeMutable(const Path& path) { + checkInterrupt(); + + struct stat st = lstat(path); + + if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) { + return; + } + + if (S_ISDIR(st.st_mode)) { + for (auto& i : readDirectory(path)) { + makeMutable(path + "/" + i.name); + } + } + + /* The O_NOFOLLOW is important to prevent us from changing the + mutable bit on the target of a symlink (which would be a + security hole). */ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (fd == -1) { + if (errno == ELOOP) { + return; + } // it's a symlink + throw SysError(format("opening file '%1%'") % path); + } + + unsigned int flags = 0, old; + + /* Silently ignore errors getting/setting the immutable flag so + that we work correctly on filesystems that don't support it. */ + if (ioctl(fd, FS_IOC_GETFLAGS, &flags)) { + return; + } + old = flags; + flags &= ~FS_IMMUTABLE_FL; + if (old == flags) { + return; + } + if (ioctl(fd, FS_IOC_SETFLAGS, &flags)) { + return; + } +} + +/* Upgrade from schema 6 (Nix 0.15) to schema 7 (Nix >= 1.3). */ +void LocalStore::upgradeStore7() { + if (getuid() != 0) { + return; + } + printError( + "removing immutable bits from the Nix store (this may take a while)..."); + makeMutable(realStoreDir); +} + +#else + +void LocalStore::upgradeStore7() {} + +#endif + +void LocalStore::vacuumDB() { + auto state(_state.lock()); + state->db.exec("vacuum"); +} + +void LocalStore::addSignatures(const Path& storePath, const StringSet& sigs) { + retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); + + auto info = std::const_pointer_cast<ValidPathInfo>( + std::shared_ptr<const ValidPathInfo>(queryPathInfo(storePath))); + + info->sigs.insert(sigs.begin(), sigs.end()); + + updatePathInfo(*state, *info); + + txn.commit(); + }); +} + +void LocalStore::signPathInfo(ValidPathInfo& info) { + // FIXME: keep secret keys in memory. + + auto secretKeyFiles = settings.secretKeyFiles; + + for (auto& secretKeyFile : secretKeyFiles.get()) { + SecretKey secretKey(readFile(secretKeyFile)); + info.sign(secretKey); + } +} + +void LocalStore::createUser(const std::string& userName, uid_t userId) { + for (auto& dir : {fmt("%s/profiles/per-user/%s", stateDir, userName), + fmt("%s/gcroots/per-user/%s", stateDir, userName)}) { + createDirs(dir); + if (chmod(dir.c_str(), 0755) == -1) { + throw SysError("changing permissions of directory '%s'", dir); + } + if (chown(dir.c_str(), userId, getgid()) == -1) { + throw SysError("changing owner of directory '%s'", dir); + } + } +} + +} // namespace nix |