diff options
Diffstat (limited to 'third_party/nix/src/libstore/local-store.cc')
| -rw-r--r-- | third_party/nix/src/libstore/local-store.cc | 1519 |
1 files changed, 0 insertions, 1519 deletions
diff --git a/third_party/nix/src/libstore/local-store.cc b/third_party/nix/src/libstore/local-store.cc deleted file mode 100644 index aca305e1a52f..000000000000 --- a/third_party/nix/src/libstore/local-store.cc +++ /dev/null @@ -1,1519 +0,0 @@ -#include "libstore/local-store.hh" - -#include <algorithm> -#include <cerrno> -#include <cstdio> -#include <cstring> -#include <ctime> -#include <iostream> - -#include <absl/strings/numbers.h> -#include <absl/strings/str_cat.h> -#include <absl/strings/str_split.h> -#include <fcntl.h> -#include <glog/logging.h> -#include <grp.h> -#include <sched.h> -#include <sqlite3.h> -#include <sys/ioctl.h> -#include <sys/mount.h> -#include <sys/select.h> -#include <sys/stat.h> -#include <sys/statvfs.h> -#include <sys/time.h> -#include <sys/types.h> -#include <sys/xattr.h> -#include <unistd.h> -#include <utime.h> - -#include "generated/schema.sql.hh" -#include "libstore/derivations.hh" -#include "libstore/globals.hh" -#include "libstore/nar-info.hh" -#include "libstore/pathlocks.hh" -#include "libstore/worker-protocol.hh" -#include "libutil/archive.hh" - -namespace nix { - -LocalStore::LocalStore(const Params& params) - : Store(params), - LocalFSStore(params), - realStoreDir_{this, false, - rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", - "physical path to the Nix store"}, - realStoreDir(realStoreDir_), - dbDir(stateDir + "/db"), - linksDir(realStoreDir + "/.links"), - reservedPath(dbDir + "/reserved"), - schemaPath(dbDir + "/schema"), - trashDir(realStoreDir + "/trash"), - tempRootsDir(stateDir + "/temproots"), - fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) { - auto state(_state.lock()); - - /* Create missing state directories if they don't already exist. */ - createDirs(realStoreDir); - makeStoreWritable(); - createDirs(linksDir); - Path profilesDir = stateDir + "/profiles"; - createDirs(profilesDir); - createDirs(tempRootsDir); - createDirs(dbDir); - Path gcRootsDir = stateDir + "/gcroots"; - if (!pathExists(gcRootsDir)) { - createDirs(gcRootsDir); - createSymlink(profilesDir, gcRootsDir + "/profiles"); - } - - for (auto& perUserDir : - {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { - createDirs(perUserDir); - if (chmod(perUserDir.c_str(), 0755) == -1) { - throw SysError("could not set permissions on '%s' to 755", perUserDir); - } - } - - // TODO(kanepyork): migrate to external constructor, this bypasses virtual - // dispatch - // NOLINTNEXTLINE clang-analyzer-optin.cplusplus.VirtualCall - createUser(getUserName(), getuid()); - - /* Optionally, create directories and set permissions for a - multi-user install. */ - if (getuid() == 0 && settings.buildUsersGroup != "") { - mode_t perm = 01775; - - struct group* gr = getgrnam(settings.buildUsersGroup.get().c_str()); - if (gr == nullptr) { - LOG(ERROR) << "warning: the group '" << settings.buildUsersGroup - << "' specified in 'build-users-group' does not exist"; - } else { - struct stat st; - if (stat(realStoreDir.c_str(), &st) != 0) { - throw SysError(format("getting attributes of path '%1%'") % - realStoreDir); - } - - if (st.st_uid != 0 || st.st_gid != gr->gr_gid || - (st.st_mode & ~S_IFMT) != perm) { - if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1) { - throw SysError(format("changing ownership of path '%1%'") % - realStoreDir); - } - if (chmod(realStoreDir.c_str(), perm) == -1) { - throw SysError(format("changing permissions on path '%1%'") % - realStoreDir); - } - } - } - } - - /* Ensure that the store and its parents are not symlinks. */ - if (getEnv("NIX_IGNORE_SYMLINK_STORE") != "1") { - Path path = realStoreDir; - struct stat st; - while (path != "/") { - if (lstat(path.c_str(), &st) != 0) { - throw SysError(format("getting status of '%1%'") % path); - } - if (S_ISLNK(st.st_mode)) { - throw Error(format("the path '%1%' is a symlink; " - "this is not allowed for the Nix store and its " - "parent directories") % - path); - } - path = dirOf(path); - } - } - - /* We can't open a SQLite database if the disk is full. Since - this prevents the garbage collector from running when it's most - needed, we reserve some dummy space that we can free just - before doing a garbage collection. */ - try { - struct stat st; - if (stat(reservedPath.c_str(), &st) == -1 || - st.st_size != settings.reservedSize) { - AutoCloseFD fd( - open(reservedPath.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0600)); - int res = -1; -#if HAVE_POSIX_FALLOCATE - res = posix_fallocate(fd.get(), 0, settings.reservedSize); -#endif - if (res == -1) { - writeFull(fd.get(), std::string(settings.reservedSize, 'X')); - [[gnu::unused]] auto res2 = ftruncate(fd.get(), settings.reservedSize); - } - } - } catch (SysError& e) { /* don't care about errors */ - } - - /* Acquire the big fat lock in shared mode to make sure that no - schema upgrade is in progress. */ - Path globalLockPath = dbDir + "/big-lock"; - globalLock = openLockFile(globalLockPath, true); - - if (!lockFile(globalLock.get(), ltRead, false)) { - LOG(INFO) << "waiting for the big Nix store lock..."; - lockFile(globalLock.get(), ltRead, true); - } - - /* Check the current database schema and if necessary do an - upgrade. */ - int curSchema = getSchema(); - if (curSchema > nixSchemaVersion) { - throw Error( - format( - "current Nix store schema is version %1%, but I only support %2%") % - curSchema % nixSchemaVersion); - } - if (curSchema == 0) { /* new store */ - curSchema = nixSchemaVersion; - openDB(*state, true); - writeFile(schemaPath, (format("%1%") % curSchema).str()); - } else if (curSchema < nixSchemaVersion) { - if (curSchema < 5) { - throw Error( - "Your Nix store has a database in Berkeley DB format,\n" - "which is no longer supported. To convert to the new format,\n" - "please upgrade Nix to version 0.12 first."); - } - - if (curSchema < 6) { - throw Error( - "Your Nix store has a database in flat file format,\n" - "which is no longer supported. To convert to the new format,\n" - "please upgrade Nix to version 1.11 first."); - } - - if (!lockFile(globalLock.get(), ltWrite, false)) { - LOG(INFO) << "waiting for exclusive access to the Nix store..."; - lockFile(globalLock.get(), ltWrite, true); - } - - /* Get the schema version again, because another process may - have performed the upgrade already. */ - curSchema = getSchema(); - - if (curSchema < 7) { - upgradeStore7(); - } - - openDB(*state, false); - - if (curSchema < 8) { - SQLiteTxn txn(state->db); - state->db.exec("alter table ValidPaths add column ultimate integer"); - state->db.exec("alter table ValidPaths add column sigs text"); - txn.commit(); - } - - if (curSchema < 9) { - SQLiteTxn txn(state->db); - state->db.exec("drop table FailedPaths"); - txn.commit(); - } - - if (curSchema < 10) { - SQLiteTxn txn(state->db); - state->db.exec("alter table ValidPaths add column ca text"); - txn.commit(); - } - - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); - - lockFile(globalLock.get(), ltRead, true); - } else { - openDB(*state, false); - } - - /* Prepare SQL statements. */ - state->stmtRegisterValidPath.create( - state->db, - "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, " - "ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);"); - state->stmtUpdatePathInfo.create( - state->db, - "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca " - "= ? where path = ?;"); - state->stmtAddReference.create( - state->db, - "insert or replace into Refs (referrer, reference) values (?, ?);"); - state->stmtQueryPathInfo.create( - state->db, - "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca " - "from ValidPaths where path = ?;"); - state->stmtQueryReferences.create(state->db, - "select path from Refs join ValidPaths on " - "reference = id where referrer = ?;"); - state->stmtQueryReferrers.create( - state->db, - "select path from Refs join ValidPaths on referrer = id where reference " - "= (select id from ValidPaths where path = ?);"); - state->stmtInvalidatePath.create(state->db, - "delete from ValidPaths where path = ?;"); - state->stmtAddDerivationOutput.create( - state->db, - "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, " - "?);"); - state->stmtQueryValidDerivers.create( - state->db, - "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv " - "= v.id where d.path = ?;"); - state->stmtQueryDerivationOutputs.create( - state->db, "select id, path from DerivationOutputs where drv = ?;"); - // Use "path >= ?" with limit 1 rather than "path like '?%'" to - // ensure efficient lookup. - state->stmtQueryPathFromHashPart.create( - state->db, "select path from ValidPaths where path >= ? limit 1;"); - state->stmtQueryValidPaths.create(state->db, "select path from ValidPaths"); -} - -LocalStore::~LocalStore() { - std::shared_future<void> future; - - { - auto state(_state.lock()); - if (state->gcRunning) { - future = state->gcFuture; - } - } - - if (future.valid()) { - LOG(INFO) << "waiting for auto-GC to finish on exit..."; - future.get(); - } - - try { - auto state(_state.lock()); - if (state->fdTempRoots) { - state->fdTempRoots = AutoCloseFD(-1); - unlink(fnTempRoots.c_str()); - } - } catch (...) { - ignoreException(); - } -} - -std::string LocalStore::getUri() { return "local"; } - -int LocalStore::getSchema() { - int curSchema = 0; - if (pathExists(schemaPath)) { - std::string s = readFile(schemaPath); - if (!absl::SimpleAtoi(s, &curSchema)) { - throw Error(format("'%1%' is corrupt") % schemaPath); - } - } - return curSchema; -} - -void LocalStore::openDB(State& state, bool create) { - if (access(dbDir.c_str(), R_OK | W_OK) != 0) { - throw SysError(format("Nix database directory '%1%' is not writable") % - dbDir); - } - - /* Open the Nix database. */ - std::string dbPath = dbDir + "/db.sqlite"; - auto& db(state.db); - if (sqlite3_open_v2(dbPath.c_str(), &db.db, - SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), - nullptr) != SQLITE_OK) { - throw Error(format("cannot open Nix database '%1%'") % dbPath); - } - - if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) { - throwSQLiteError(db, "setting timeout"); - } - - db.exec("pragma foreign_keys = 1"); - - /* !!! check whether sqlite has been built with foreign key - support */ - - /* Whether SQLite should fsync(). "Normal" synchronous mode - should be safe enough. If the user asks for it, don't sync at - all. This can cause database corruption if the system - crashes. */ - std::string syncMode = settings.fsyncMetadata ? "normal" : "off"; - db.exec("pragma synchronous = " + syncMode); - - /* Set the SQLite journal mode. WAL mode is fastest, so it's the - default. */ - std::string mode = settings.useSQLiteWAL ? "wal" : "truncate"; - std::string prevMode; - { - SQLiteStmt stmt; - stmt.create(db, "pragma main.journal_mode;"); - if (sqlite3_step(stmt) != SQLITE_ROW) { - throwSQLiteError(db, "querying journal mode"); - } - prevMode = std::string( - reinterpret_cast<const char*>(sqlite3_column_text(stmt, 0))); - } - if (prevMode != mode && - sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), - nullptr, nullptr, nullptr) != SQLITE_OK) { - throwSQLiteError(db, "setting journal mode"); - } - - /* Increase the auto-checkpoint interval to 40000 pages. This - seems enough to ensure that instantiating the NixOS system - derivation is done in a single fsync(). */ - if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 40000;", - nullptr, nullptr, nullptr) != SQLITE_OK) { - throwSQLiteError(db, "setting autocheckpoint interval"); - } - - /* Initialise the database schema, if necessary. */ - if (create) { - db.exec(kNixSqlSchema); - } -} - -/* To improve purity, users may want to make the Nix store a read-only - bind mount. So make the Nix store writable for this process. */ -void LocalStore::makeStoreWritable() { - if (getuid() != 0) { - return; - } - /* Check if /nix/store is on a read-only mount. */ - struct statvfs stat; - if (statvfs(realStoreDir.c_str(), &stat) != 0) { - throw SysError("getting info about the Nix store mount point"); - } - - if ((stat.f_flag & ST_RDONLY) != 0u) { - if (unshare(CLONE_NEWNS) == -1) { - throw SysError("setting up a private mount namespace"); - } - - if (mount(nullptr, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, - nullptr) == -1) { - throw SysError(format("remounting %1% writable") % realStoreDir); - } - } -} - -const time_t mtimeStore = 1; /* 1 second into the epoch */ - -static void canonicaliseTimestampAndPermissions(const Path& path, - const struct stat& st) { - if (!S_ISLNK(st.st_mode)) { - /* Mask out all type related bits. */ - mode_t mode = st.st_mode & ~S_IFMT; - - if (mode != 0444 && mode != 0555) { - mode = (st.st_mode & S_IFMT) | 0444 | - ((st.st_mode & S_IXUSR) != 0u ? 0111 : 0); - if (chmod(path.c_str(), mode) == -1) { - throw SysError(format("changing mode of '%1%' to %2$o") % path % mode); - } - } - } - - if (st.st_mtime != mtimeStore) { - struct timeval times[2]; - times[0].tv_sec = st.st_atime; - times[0].tv_usec = 0; - times[1].tv_sec = mtimeStore; - times[1].tv_usec = 0; -#if HAVE_LUTIMES - if (lutimes(path.c_str(), times) == -1) { - if (errno != ENOSYS || - (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) { -#else - if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) { -#endif - throw SysError(format("changing modification time of '%1%'") % path); - } - } - } // namespace nix -} // namespace nix - -void canonicaliseTimestampAndPermissions(const Path& path) { - struct stat st; - if (lstat(path.c_str(), &st) != 0) { - throw SysError(format("getting attributes of path '%1%'") % path); - } - canonicaliseTimestampAndPermissions(path, st); -} - -static void canonicalisePathMetaData_(const Path& path, uid_t fromUid, - InodesSeen& inodesSeen) { - checkInterrupt(); - - struct stat st; - if (lstat(path.c_str(), &st) != 0) { - throw SysError(format("getting attributes of path '%1%'") % path); - } - - /* Really make sure that the path is of a supported type. */ - if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) { - throw Error(format("file '%1%' has an unsupported type") % path); - } - - /* Remove extended attributes / ACLs. */ - ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); - - if (eaSize < 0) { - if (errno != ENOTSUP && errno != ENODATA) { - throw SysError("querying extended attributes of '%s'", path); - } - } else if (eaSize > 0) { - std::vector<char> eaBuf(eaSize); - - if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) { - throw SysError("querying extended attributes of '%s'", path); - } - - for (auto& eaName : absl::StrSplit(std::string(eaBuf.data(), eaSize), - absl::ByString(std::string("\000", 1)), - absl::SkipEmpty())) { - /* Ignore SELinux security labels since these cannot be - removed even by root. */ - if (eaName == "security.selinux") { - continue; - } - if (lremovexattr(path.c_str(), std::string(eaName).c_str()) == -1) { - throw SysError("removing extended attribute '%s' from '%s'", eaName, - path); - } - } - } - - /* Fail if the file is not owned by the build user. This prevents - us from messing up the ownership/permissions of files - hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). - However, ignore files that we chown'ed ourselves previously to - ensure that we don't fail on hard links within the same build - (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ - if (fromUid != static_cast<uid_t>(-1) && st.st_uid != fromUid) { - if (S_ISDIR(st.st_mode)) { - throw BuildError(format("invalid file '%1%': is a directory") % path); - } - if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) { - throw BuildError(format("invalid ownership on file '%1%'") % path); - } - if (!(S_ISLNK(st.st_mode) || - (st.st_uid == geteuid() && - ((st.st_mode & ~S_IFMT) == 0444 || (st.st_mode & ~S_IFMT) == 0555) && - st.st_mtime == mtimeStore))) { - throw BuildError( - format("invalid permissions on file '%1%', should be 0444/0555") % - path); - } - - return; - } - - inodesSeen.insert(Inode(st.st_dev, st.st_ino)); - - canonicaliseTimestampAndPermissions(path, st); - - /* Change ownership to the current uid. If it's a symlink, use - lchown if available, otherwise don't bother. Wrong ownership - of a symlink doesn't matter, since the owning user can't change - the symlink and can't delete it because the directory is not - writable. The only exception is top-level paths in the Nix - store (since that directory is group-writable for the Nix build - users group); we check for this case below. */ - if (st.st_uid != geteuid()) { -#if HAVE_LCHOWN - if (lchown(path.c_str(), geteuid(), getegid()) == -1) { -#else - if (!S_ISLNK(st.st_mode) && chown(path.c_str(), geteuid(), getegid()) == -1) -#endif - throw SysError(format("changing owner of '%1%' to %2%") % path % - geteuid()); - } - } - - if (S_ISDIR(st.st_mode)) { - DirEntries entries = readDirectory(path); - for (auto& i : entries) { - canonicalisePathMetaData_(path + "/" + i.name, fromUid, inodesSeen); - } - } -} - -void canonicalisePathMetaData(const Path& path, uid_t fromUid, - InodesSeen& inodesSeen) { - canonicalisePathMetaData_(path, fromUid, inodesSeen); - - /* On platforms that don't have lchown(), the top-level path can't - be a symlink, since we can't change its ownership. */ - struct stat st; - if (lstat(path.c_str(), &st) != 0) { - throw SysError(format("getting attributes of path '%1%'") % path); - } - - if (st.st_uid != geteuid()) { - assert(S_ISLNK(st.st_mode)); - throw Error(format("wrong ownership of top-level store path '%1%'") % path); - } -} - -void canonicalisePathMetaData(const Path& path, uid_t fromUid) { - InodesSeen inodesSeen; - canonicalisePathMetaData(path, fromUid, inodesSeen); -} - -void LocalStore::checkDerivationOutputs(const Path& drvPath, - const Derivation& drv) { - std::string drvName = storePathToName(drvPath); - assert(isDerivation(drvName)); - drvName = std::string(drvName, 0, drvName.size() - drvExtension.size()); - - if (drv.isFixedOutput()) { - auto out = drv.outputs.find("out"); - if (out == drv.outputs.end()) { - throw Error( - format("derivation '%1%' does not have an output named 'out'") % - drvPath); - } - - bool recursive; - Hash h; - out->second.parseHashInfo(recursive, h); - Path outPath = makeFixedOutputPath(recursive, h, drvName); - - auto j = drv.env.find("out"); - if (out->second.path != outPath || j == drv.env.end() || - j->second != outPath) { - throw Error( - format( - "derivation '%1%' has incorrect output '%2%', should be '%3%'") % - drvPath % out->second.path % outPath); - } - } - - else { - Derivation drvCopy(drv); - for (auto& i : drvCopy.outputs) { - i.second.path = ""; - drvCopy.env[i.first] = ""; - } - - Hash h = hashDerivationModulo(*this, drvCopy); - - for (auto& i : drv.outputs) { - Path outPath = makeOutputPath(i.first, h, drvName); - auto j = drv.env.find(i.first); - if (i.second.path != outPath || j == drv.env.end() || - j->second != outPath) { - throw Error(format("derivation '%1%' has incorrect output '%2%', " - "should be '%3%'") % - drvPath % i.second.path % outPath); - } - } - } -} - -uint64_t LocalStore::addValidPath(State& state, const ValidPathInfo& info, - bool checkOutputs) { - if (!info.ca.empty() && !info.isContentAddressed(*this)) { - throw Error( - "cannot add path '%s' to the Nix store because it claims to be " - "content-addressed but isn't", - info.path); - } - - state.stmtRegisterValidPath - .use()(info.path)(info.narHash.to_string(Base16))( - info.registrationTime == 0 ? time(nullptr) : info.registrationTime)( - info.deriver, !info.deriver.empty())(info.narSize, info.narSize != 0)( - info.ultimate ? 1 : 0, info.ultimate)( - concatStringsSep(" ", info.sigs), !info.sigs.empty())( - info.ca, !info.ca.empty()) - .exec(); - uint64_t id = sqlite3_last_insert_rowid(state.db); - - /* If this is a derivation, then store the derivation outputs in - the database. This is useful for the garbage collector: it can - efficiently query whether a path is an output of some - derivation. */ - if (isDerivation(info.path)) { - Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(info.path)); - - /* Verify that the output paths in the derivation are correct - (i.e., follow the scheme for computing output paths from - derivations). Note that if this throws an error, then the - DB transaction is rolled back, so the path validity - registration above is undone. */ - if (checkOutputs) { - checkDerivationOutputs(info.path, drv); - } - - for (auto& i : drv.outputs) { - state.stmtAddDerivationOutput.use()(id)(i.first)(i.second.path).exec(); - } - } - - { - auto state_(Store::state.lock()); - state_->pathInfoCache.upsert(storePathToHash(info.path), - std::make_shared<ValidPathInfo>(info)); - } - - return id; -} - -void LocalStore::queryPathInfoUncached( - const Path& path, - Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept { - try { - auto info = std::make_shared<ValidPathInfo>(); - info->path = path; - - assertStorePath(path); - - callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { - auto state(_state.lock()); - - /* Get the path info. */ - auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path)); - - if (!useQueryPathInfo.next()) { - return std::shared_ptr<ValidPathInfo>(); - } - - info->id = useQueryPathInfo.getInt(0); - - auto hash_ = Hash::deserialize(useQueryPathInfo.getStr(1)); - if (!hash_.ok()) { - throw Error(absl::StrCat("in valid-path entry for '", path, - "': ", hash_.status().ToString())); - } - info->narHash = *hash_; - - info->registrationTime = useQueryPathInfo.getInt(2); - - auto s = reinterpret_cast<const char*>( - sqlite3_column_text(state->stmtQueryPathInfo, 3)); - if (s != nullptr) { - info->deriver = s; - } - - /* Note that narSize = NULL yields 0. */ - info->narSize = useQueryPathInfo.getInt(4); - - info->ultimate = useQueryPathInfo.getInt(5) == 1; - - s = reinterpret_cast<const char*>( - sqlite3_column_text(state->stmtQueryPathInfo, 6)); - if (s != nullptr) { - info->sigs = absl::StrSplit(s, absl::ByChar(' '), absl::SkipEmpty()); - } - - s = reinterpret_cast<const char*>( - sqlite3_column_text(state->stmtQueryPathInfo, 7)); - if (s != nullptr) { - info->ca = s; - } - - /* Get the references. */ - auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); - - while (useQueryReferences.next()) { - info->references.insert(useQueryReferences.getStr(0)); - } - - return info; - })); - - } catch (...) { - callback.rethrow(); - } -} - -/* Update path info in the database. */ -void LocalStore::updatePathInfo(State& state, const ValidPathInfo& info) { - state.stmtUpdatePathInfo - .use()(info.narSize, info.narSize != 0)(info.narHash.to_string(Base16))( - info.ultimate ? 1 : 0, info.ultimate)( - concatStringsSep(" ", info.sigs), !info.sigs.empty())( - info.ca, !info.ca.empty())(info.path) - .exec(); -} - -uint64_t LocalStore::queryValidPathId(State& state, const Path& path) { - auto use(state.stmtQueryPathInfo.use()(path)); - if (!use.next()) { - throw Error(format("path '%1%' is not valid") % path); - } - return use.getInt(0); -} - -bool LocalStore::isValidPath_(State& state, const Path& path) { - return state.stmtQueryPathInfo.use()(path).next(); -} - -bool LocalStore::isValidPathUncached(const Path& path) { - return retrySQLite<bool>([&]() { - auto state(_state.lock()); - return isValidPath_(*state, path); - }); -} - -PathSet LocalStore::queryValidPaths(const PathSet& paths, - SubstituteFlag maybeSubstitute) { - PathSet res; - for (auto& i : paths) { - if (isValidPath(i)) { - res.insert(i); - } - } - return res; -} - -PathSet LocalStore::queryAllValidPaths() { - return retrySQLite<PathSet>([&]() { - auto state(_state.lock()); - auto use(state->stmtQueryValidPaths.use()); - PathSet res; - while (use.next()) { - res.insert(use.getStr(0)); - } - return res; - }); -} - -void LocalStore::queryReferrers(State& state, const Path& path, - PathSet& referrers) { - auto useQueryReferrers(state.stmtQueryReferrers.use()(path)); - - while (useQueryReferrers.next()) { - referrers.insert(useQueryReferrers.getStr(0)); - } -} - -void LocalStore::queryReferrers(const Path& path, PathSet& referrers) { - assertStorePath(path); - return retrySQLite<void>([&]() { - auto state(_state.lock()); - queryReferrers(*state, path, referrers); - }); -} - -PathSet LocalStore::queryValidDerivers(const Path& path) { - assertStorePath(path); - - return retrySQLite<PathSet>([&]() { - auto state(_state.lock()); - - auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(path)); - - PathSet derivers; - while (useQueryValidDerivers.next()) { - derivers.insert(useQueryValidDerivers.getStr(1)); - } - - return derivers; - }); -} - -PathSet LocalStore::queryDerivationOutputs(const Path& path) { - return retrySQLite<PathSet>([&]() { - auto state(_state.lock()); - - auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()( - queryValidPathId(*state, path))); - - PathSet outputs; - while (useQueryDerivationOutputs.next()) { - outputs.insert(useQueryDerivationOutputs.getStr(1)); - } - - return outputs; - }); -} - -StringSet LocalStore::queryDerivationOutputNames(const Path& path) { - return retrySQLite<StringSet>([&]() { - auto state(_state.lock()); - - auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()( - queryValidPathId(*state, path))); - - StringSet outputNames; - while (useQueryDerivationOutputs.next()) { - outputNames.insert(useQueryDerivationOutputs.getStr(0)); - } - - return outputNames; - }); -} - -Path LocalStore::queryPathFromHashPart(const std::string& hashPart) { - if (hashPart.size() != storePathHashLen) { - throw Error("invalid hash part"); - } - - Path prefix = storeDir + "/" + hashPart; - - return retrySQLite<Path>([&]() -> std::string { - auto state(_state.lock()); - - auto useQueryPathFromHashPart( - state->stmtQueryPathFromHashPart.use()(prefix)); - - if (!useQueryPathFromHashPart.next()) { - return ""; - } - - const char* s = reinterpret_cast<const char*>( - sqlite3_column_text(state->stmtQueryPathFromHashPart, 0)); - return (s != nullptr) && - prefix.compare(0, prefix.size(), s, prefix.size()) == 0 - ? s - : ""; - }); -} - -PathSet LocalStore::querySubstitutablePaths(const PathSet& paths) { - if (!settings.useSubstitutes) { - return PathSet(); - } - - auto remaining = paths; - PathSet res; - - for (auto& sub : getDefaultSubstituters()) { - if (remaining.empty()) { - break; - } - if (sub->storeDir != storeDir) { - continue; - } - if (!sub->wantMassQuery()) { - continue; - } - - auto valid = sub->queryValidPaths(remaining); - - PathSet remaining2; - for (auto& path : remaining) { - if (valid.count(path) != 0u) { - res.insert(path); - } else { - remaining2.insert(path); - } - } - - std::swap(remaining, remaining2); - } - - return res; -} - -void LocalStore::querySubstitutablePathInfos(const PathSet& paths, - SubstitutablePathInfos& infos) { - if (!settings.useSubstitutes) { - return; - } - for (auto& sub : getDefaultSubstituters()) { - if (sub->storeDir != storeDir) { - continue; - } - for (auto& path : paths) { - if (infos.count(path) != 0u) { - continue; - } - DLOG(INFO) << "checking substituter '" << sub->getUri() << "' for path '" - << path << "'"; - try { - auto info = sub->queryPathInfo(path); - auto narInfo = std::dynamic_pointer_cast<const NarInfo>( - std::shared_ptr<const ValidPathInfo>(info)); - infos[path] = SubstitutablePathInfo{info->deriver, info->references, - narInfo ? narInfo->fileSize : 0, - info->narSize}; - } catch (InvalidPath&) { - } catch (SubstituterDisabled&) { - } catch (Error& e) { - if (settings.tryFallback) { - LOG(ERROR) << e.what(); - } else { - throw; - } - } - } - } -} - -void LocalStore::registerValidPath(const ValidPathInfo& info) { - ValidPathInfos infos; - infos.push_back(info); - registerValidPaths(infos); -} - -void LocalStore::registerValidPaths(const ValidPathInfos& infos) { - /* SQLite will fsync by default, but the new valid paths may not - be fsync-ed. So some may want to fsync them before registering - the validity, at the expense of some speed of the path - registering operation. */ - if (settings.syncBeforeRegistering) { - sync(); - } - - return retrySQLite<void>([&]() { - auto state(_state.lock()); - - SQLiteTxn txn(state->db); - PathSet paths; - - for (auto& i : infos) { - assert(i.narHash.type == htSHA256); - if (isValidPath_(*state, i.path)) { - updatePathInfo(*state, i); - } else { - addValidPath(*state, i, false); - } - paths.insert(i.path); - } - - for (auto& i : infos) { - auto referrer = queryValidPathId(*state, i.path); - for (auto& j : i.references) { - state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)) - .exec(); - } - } - - /* Check that the derivation outputs are correct. We can't do - this in addValidPath() above, because the references might - not be valid yet. */ - for (auto& i : infos) { - if (isDerivation(i.path)) { - // FIXME: inefficient; we already loaded the - // derivation in addValidPath(). - Derivation drv = - readDerivation(realStoreDir + "/" + baseNameOf(i.path)); - checkDerivationOutputs(i.path, drv); - } - } - - /* Do a topological sort of the paths. This will throw an - error if a cycle is detected and roll back the - transaction. Cycles can only occur when a derivation - has multiple outputs. */ - topoSortPaths(paths); - - txn.commit(); - }); -} - -/* Invalidate a path. The caller is responsible for checking that - there are no referrers. */ -void LocalStore::invalidatePath(State& state, const Path& path) { - LOG(INFO) << "invalidating path '" << path << "'"; - - state.stmtInvalidatePath.use()(path).exec(); - - /* Note that the foreign key constraints on the Refs table take - care of deleting the references entries for `path'. */ - - { - auto state_(Store::state.lock()); - state_->pathInfoCache.erase(storePathToHash(path)); - } -} - -const PublicKeys& LocalStore::getPublicKeys() { - auto state(_state.lock()); - if (!state->publicKeys) { - state->publicKeys = std::make_unique<PublicKeys>(getDefaultPublicKeys()); - } - return *state->publicKeys; -} - -void LocalStore::addToStore(const ValidPathInfo& info, Source& source, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) { - if (!info.narHash) { - throw Error("cannot add path '%s' because it lacks a hash", info.path); - } - - if (requireSigs && (checkSigs != 0u) && - (info.checkSignatures(*this, getPublicKeys()) == 0u)) { - throw Error("cannot add path '%s' because it lacks a valid signature", - info.path); - } - - addTempRoot(info.path); - - if ((repair != 0u) || !isValidPath(info.path)) { - PathLocks outputLock; - - Path realPath = realStoreDir + "/" + baseNameOf(info.path); - - /* Lock the output path. But don't lock if we're being called - from a build hook (whose parent process already acquired a - lock on this path). */ - if (locksHeld.count(info.path) == 0u) { - outputLock.lockPaths({realPath}); - } - - if ((repair != 0u) || !isValidPath(info.path)) { - deletePath(realPath); - - /* While restoring the path from the NAR, compute the hash - of the NAR. */ - HashSink hashSink(htSHA256); - - LambdaSource wrapperSource( - [&](unsigned char* data, size_t len) -> size_t { - size_t n = source.read(data, len); - hashSink(data, n); - return n; - }); - - restorePath(realPath, wrapperSource); - - auto hashResult = hashSink.finish(); - - if (hashResult.first != info.narHash) { - throw Error( - "hash mismatch importing path '%s';\n wanted: %s\n got: %s", - info.path, info.narHash.to_string(), hashResult.first.to_string()); - } - - if (hashResult.second != info.narSize) { - throw Error( - "size mismatch importing path '%s';\n wanted: %s\n got: %s", - info.path, info.narSize, hashResult.second); - } - - autoGC(); - - canonicalisePathMetaData(realPath, -1); - - optimisePath(realPath); // FIXME: combine with hashPath() - - registerValidPath(info); - } - - outputLock.setDeletion(true); - } -} - -Path LocalStore::addToStoreFromDump(const std::string& dump, - const std::string& name, bool recursive, - HashType hashAlgo, RepairFlag repair) { - Hash h = hashString(hashAlgo, dump); - - Path dstPath = makeFixedOutputPath(recursive, h, name); - - addTempRoot(dstPath); - - if ((repair != 0u) || !isValidPath(dstPath)) { - /* The first check above is an optimisation to prevent - unnecessary lock acquisition. */ - - Path realPath = realStoreDir + "/" + baseNameOf(dstPath); - - PathLocks outputLock({realPath}); - - if ((repair != 0u) || !isValidPath(dstPath)) { - deletePath(realPath); - - autoGC(); - - if (recursive) { - StringSource source(dump); - restorePath(realPath, source); - } else { - writeFile(realPath, dump); - } - - canonicalisePathMetaData(realPath, -1); - - /* Register the SHA-256 hash of the NAR serialisation of - the path in the database. We may just have computed it - above (if called with recursive == true and hashAlgo == - sha256); otherwise, compute it here. */ - HashResult hash; - if (recursive) { - hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); - hash.second = dump.size(); - } else { - hash = hashPath(htSHA256, realPath); - } - - optimisePath(realPath); // FIXME: combine with hashPath() - - ValidPathInfo info; - info.path = dstPath; - info.narHash = hash.first; - info.narSize = hash.second; - info.ca = makeFixedOutputCA(recursive, h); - registerValidPath(info); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - -Path LocalStore::addToStore(const std::string& name, const Path& _srcPath, - bool recursive, HashType hashAlgo, - PathFilter& filter, RepairFlag repair) { - Path srcPath(absPath(_srcPath)); - - /* Read the whole path into memory. This is not a very scalable - method for very large paths, but `copyPath' is mainly used for - small files. */ - StringSink sink; - if (recursive) { - dumpPath(srcPath, sink, filter); - } else { - sink.s = make_ref<std::string>(readFile(srcPath)); - } - - return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair); -} - -Path LocalStore::addTextToStore(const std::string& name, const std::string& s, - const PathSet& references, RepairFlag repair) { - auto hash = hashString(htSHA256, s); - auto dstPath = makeTextPath(name, hash, references); - - addTempRoot(dstPath); - - if ((repair != 0u) || !isValidPath(dstPath)) { - Path realPath = realStoreDir + "/" + baseNameOf(dstPath); - - PathLocks outputLock({realPath}); - - if ((repair != 0u) || !isValidPath(dstPath)) { - deletePath(realPath); - - autoGC(); - - writeFile(realPath, s); - - canonicalisePathMetaData(realPath, -1); - - StringSink sink; - dumpString(s, sink); - auto narHash = hashString(htSHA256, *sink.s); - - optimisePath(realPath); - - ValidPathInfo info; - info.path = dstPath; - info.narHash = narHash; - info.narSize = sink.s->size(); - info.references = references; - info.ca = "text:" + hash.to_string(); - registerValidPath(info); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - -/* Create a temporary directory in the store that won't be - garbage-collected. */ -Path LocalStore::createTempDirInStore() { - Path tmpDir; - do { - /* There is a slight possibility that `tmpDir' gets deleted by - the GC between createTempDir() and addTempRoot(), so repeat - until `tmpDir' exists. */ - tmpDir = createTempDir(realStoreDir); - addTempRoot(tmpDir); - } while (!pathExists(tmpDir)); - return tmpDir; -} - -void LocalStore::invalidatePathChecked(const Path& path) { - assertStorePath(path); - - retrySQLite<void>([&]() { - auto state(_state.lock()); - - SQLiteTxn txn(state->db); - - if (isValidPath_(*state, path)) { - PathSet referrers; - queryReferrers(*state, path, referrers); - referrers.erase(path); /* ignore self-references */ - if (!referrers.empty()) { - throw PathInUse( - format("cannot delete path '%1%' because it is in use by %2%") % - path % showPaths(referrers)); - } - invalidatePath(*state, path); - } - - txn.commit(); - }); -} - -bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { - LOG(INFO) << "reading the Nix store..."; - - bool errors = false; - - /* Acquire the global GC lock to get a consistent snapshot of - existing and valid paths. */ - AutoCloseFD fdGCLock = openGCLock(ltWrite); - - PathSet store; - for (auto& i : readDirectory(realStoreDir)) { - store.insert(i.name); - } - - /* Check whether all valid paths actually exist. */ - LOG(INFO) << "checking path existence..."; - - PathSet validPaths2 = queryAllValidPaths(); - PathSet validPaths; - PathSet done; - - fdGCLock = AutoCloseFD(-1); - - for (auto& i : validPaths2) { - verifyPath(i, store, done, validPaths, repair, errors); - } - - /* Optionally, check the content hashes (slow). */ - if (checkContents) { - LOG(INFO) << "checking hashes..."; - - Hash nullHash(htSHA256); - - for (auto& i : validPaths) { - try { - auto info = std::const_pointer_cast<ValidPathInfo>( - std::shared_ptr<const ValidPathInfo>(queryPathInfo(i))); - - /* Check the content hash (optionally - slow). */ - DLOG(INFO) << "checking contents of '" << i << "'"; - HashResult current = hashPath(info->narHash.type, toRealPath(i)); - - if (info->narHash != nullHash && info->narHash != current.first) { - LOG(ERROR) << "path '" << i << "' was modified! expected hash '" - << info->narHash.to_string() << "', got '" - << current.first.to_string() << "'"; - if (repair != 0u) { - repairPath(i); - } else { - errors = true; - } - } else { - bool update = false; - - /* Fill in missing hashes. */ - if (info->narHash == nullHash) { - LOG(WARNING) << "fixing missing hash on '" << i << "'"; - info->narHash = current.first; - update = true; - } - - /* Fill in missing narSize fields (from old stores). */ - if (info->narSize == 0) { - LOG(ERROR) << "updating size field on '" << i << "' to " - << current.second; - info->narSize = current.second; - update = true; - } - - if (update) { - auto state(_state.lock()); - updatePathInfo(*state, *info); - } - } - - } catch (Error& e) { - /* It's possible that the path got GC'ed, so ignore - errors on invalid paths. */ - if (isValidPath(i)) { - LOG(ERROR) << e.msg(); - } else { - LOG(WARNING) << e.msg(); - } - errors = true; - } - } - } - - return errors; -} - -void LocalStore::verifyPath(const Path& path, const PathSet& store, - PathSet& done, PathSet& validPaths, - RepairFlag repair, bool& errors) { - checkInterrupt(); - - if (done.find(path) != done.end()) { - return; - } - done.insert(path); - - if (!isStorePath(path)) { - LOG(ERROR) << "path '" << path << "' is not in the Nix store"; - auto state(_state.lock()); - invalidatePath(*state, path); - return; - } - - if (store.find(baseNameOf(path)) == store.end()) { - /* Check any referrers first. If we can invalidate them - first, then we can invalidate this path as well. */ - bool canInvalidate = true; - PathSet referrers; - queryReferrers(path, referrers); - for (auto& i : referrers) { - if (i != path) { - verifyPath(i, store, done, validPaths, repair, errors); - if (validPaths.find(i) != validPaths.end()) { - canInvalidate = false; - } - } - } - - if (canInvalidate) { - LOG(WARNING) << "path '" << path - << "' disappeared, removing from database..."; - auto state(_state.lock()); - invalidatePath(*state, path); - } else { - LOG(ERROR) << "path '" << path - << "' disappeared, but it still has valid referrers!"; - if (repair != 0u) { - try { - repairPath(path); - } catch (Error& e) { - LOG(WARNING) << e.msg(); - errors = true; - } - } else { - errors = true; - } - } - - return; - } - - validPaths.insert(path); -} - -unsigned int LocalStore::getProtocol() { return PROTOCOL_VERSION; } - -#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && \ - defined(FS_IMMUTABLE_FL) - -static void makeMutable(const Path& path) { - checkInterrupt(); - - struct stat st = lstat(path); - - if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) { - return; - } - - if (S_ISDIR(st.st_mode)) { - for (auto& i : readDirectory(path)) { - makeMutable(path + "/" + i.name); - } - } - - /* The O_NOFOLLOW is important to prevent us from changing the - mutable bit on the target of a symlink (which would be a - security hole). */ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); - if (fd == -1) { - if (errno == ELOOP) { - return; - } // it's a symlink - throw SysError(format("opening file '%1%'") % path); - } - - unsigned int flags = 0, old; - - /* Silently ignore errors getting/setting the immutable flag so - that we work correctly on filesystems that don't support it. */ - if (ioctl(fd, FS_IOC_GETFLAGS, &flags)) { - return; - } - old = flags; - flags &= ~FS_IMMUTABLE_FL; - if (old == flags) { - return; - } - if (ioctl(fd, FS_IOC_SETFLAGS, &flags)) { - return; - } -} - -/* Upgrade from schema 6 (Nix 0.15) to schema 7 (Nix >= 1.3). */ -void LocalStore::upgradeStore7() { - if (getuid() != 0) { - return; - } - printError( - "removing immutable bits from the Nix store (this may take a while)..."); - makeMutable(realStoreDir); -} - -#else - -void LocalStore::upgradeStore7() {} - -#endif - -void LocalStore::vacuumDB() { - auto state(_state.lock()); - state->db.exec("vacuum"); -} - -void LocalStore::addSignatures(const Path& storePath, const StringSet& sigs) { - retrySQLite<void>([&]() { - auto state(_state.lock()); - - SQLiteTxn txn(state->db); - - auto info = std::const_pointer_cast<ValidPathInfo>( - std::shared_ptr<const ValidPathInfo>(queryPathInfo(storePath))); - - info->sigs.insert(sigs.begin(), sigs.end()); - - updatePathInfo(*state, *info); - - txn.commit(); - }); -} - -void LocalStore::signPathInfo(ValidPathInfo& info) { - // FIXME: keep secret keys in memory. - - auto secretKeyFiles = settings.secretKeyFiles; - - for (auto& secretKeyFile : secretKeyFiles.get()) { - SecretKey secretKey(readFile(secretKeyFile)); - info.sign(secretKey); - } -} - -void LocalStore::createUser(const std::string& userName, uid_t userId) { - for (auto& dir : {fmt("%s/profiles/per-user/%s", stateDir, userName), - fmt("%s/gcroots/per-user/%s", stateDir, userName)}) { - createDirs(dir); - if (chmod(dir.c_str(), 0755) == -1) { - throw SysError("changing permissions of directory '%s'", dir); - } - if (chown(dir.c_str(), userId, getgid()) == -1) { - throw SysError("changing owner of directory '%s'", dir); - } - } -} - -} // namespace nix |