From b0e92f6d474ce91d7f071f9ed62bbb2015009c58 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 9 Jun 2008 13:52:45 +0000 Subject: * Merged the no-bdb branch (-r10900:HEAD https://svn.nixos.org/repos/nix/nix/branches/no-bdb). --- src/libstore/local-store.cc | 978 +++++++++++++++++++------------------------- 1 file changed, 412 insertions(+), 566 deletions(-) (limited to 'src/libstore/local-store.cc') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6a8de7bc42eb..2e53d0dc6da4 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1,8 +1,6 @@ #include "config.h" #include "local-store.hh" -#include "util.hh" #include "globals.hh" -#include "db.hh" #include "archive.hh" #include "pathlocks.hh" #include "aterm.hh" @@ -16,50 +14,12 @@ #include #include #include +#include namespace nix { -/* Nix database. */ -static Database nixDB; - - -/* Database tables. */ - -/* dbValidPaths :: Path -> () - - The existence of a key $p$ indicates that path $p$ is valid (that - is, produced by a succesful build). */ -static TableId dbValidPaths = 0; - -/* dbReferences :: Path -> [Path] - - This table lists the outgoing file system references for each - output path that has been built by a Nix derivation. These are - found by scanning the path for the hash components of input - paths. */ -static TableId dbReferences = 0; - -/* dbReferrers :: Path -> Path - - This table is just the reverse mapping of dbReferences. This table - can have duplicate keys, each corresponding value denoting a single - referrer. */ -static TableId dbReferrers = 0; - -/* dbDerivers :: Path -> [Path] - - This table lists the derivation used to build a path. There can - only be multiple such paths for fixed-output derivations (i.e., - derivations specifying an expected hash). */ -static TableId dbDerivers = 0; - - -static void upgradeStore09(); -static void upgradeStore11(); - - void checkStoreNotSymlink() { if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; @@ -78,81 +38,62 @@ void checkStoreNotSymlink() } -LocalStore::LocalStore(bool reserveSpace) +LocalStore::LocalStore() { substitutablePathsLoaded = false; + schemaPath = nixDBPath + "/schema"; + if (readOnlyMode) return; checkStoreNotSymlink(); - try { - Path reservedPath = nixDBPath + "/reserved"; - string s = querySetting("gc-reserved-space", ""); - int reservedSize; - if (!string2Int(s, reservedSize)) reservedSize = 1024 * 1024; - if (reserveSpace) { - struct stat st; - if (stat(reservedPath.c_str(), &st) == -1 || - st.st_size != reservedSize) - writeFile(reservedPath, string(reservedSize, 'X')); - } - else - deletePath(reservedPath); - } catch (SysError & e) { /* don't care about errors */ + Path globalLockPath = nixDBPath + "/big-lock"; + globalLock = openLockFile(globalLockPath.c_str(), true); + + if (!lockFile(globalLock, ltRead, false)) { + printMsg(lvlError, "waiting for the big Nix store lock..."); + lockFile(globalLock, ltRead, true); } - try { - nixDB.open(nixDBPath); - } catch (DbNoPermission & e) { - printMsg(lvlTalkative, "cannot access Nix database; continuing anyway"); - readOnlyMode = true; - return; - } - dbValidPaths = nixDB.openTable("validpaths"); - dbReferences = nixDB.openTable("references"); - dbReferrers = nixDB.openTable("referrers", true); /* must be sorted */ - dbDerivers = nixDB.openTable("derivers"); + createDirs(nixDBPath + "/info"); + createDirs(nixDBPath + "/referrer"); - int curSchema = 0; - Path schemaFN = nixDBPath + "/schema"; - if (pathExists(schemaFN)) { - string s = readFile(schemaFN); - if (!string2Int(s, curSchema)) - throw Error(format("`%1%' is corrupt") % schemaFN); - } + //printMsg(lvlTalkative, "cannot access Nix database; continuing anyway"); + //readOnlyMode = true; + int curSchema = getSchema(); if (curSchema > nixSchemaVersion) throw Error(format("current Nix store schema is version %1%, but I only support %2%") % curSchema % nixSchemaVersion); - - if (curSchema < nixSchemaVersion) { - if (curSchema == 0) /* new store */ - curSchema = nixSchemaVersion; - if (curSchema <= 1) - throw Error("your Nix store is no longer supported"); - if (curSchema <= 2) upgradeStore09(); - if (curSchema <= 3) upgradeStore11(); - writeFile(schemaFN, (format("%1%") % nixSchemaVersion).str()); + if (curSchema == 0) { /* new store */ + curSchema = nixSchemaVersion; + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); } + if (curSchema == 1) throw Error("your Nix store is no longer supported"); + if (curSchema < nixSchemaVersion) upgradeStore12(); } LocalStore::~LocalStore() { - /* If the database isn't open, this is a NOP. */ try { - nixDB.close(); + flushDelayedUpdates(); } catch (...) { ignoreException(); } } -void createStoreTransaction(Transaction & txn) +int LocalStore::getSchema() { - Transaction txn2(nixDB); - txn2.moveTo(txn); + int curSchema = 0; + if (pathExists(schemaPath)) { + string s = readFile(schemaPath); + if (!string2Int(s, curSchema)) + throw Error(format("`%1%' is corrupt") % schemaPath); + } + return curSchema; } @@ -173,7 +114,7 @@ void copyPath(const Path & src, const Path & dst, PathFilter & filter) } -static void _canonicalisePathMetaData(const Path & path, bool recurse) +void canonicalisePathMetaData(const Path & path, bool recurse) { checkInterrupt(); @@ -181,7 +122,7 @@ static void _canonicalisePathMetaData(const Path & path, bool recurse) if (lstat(path.c_str(), &st)) throw SysError(format("getting attributes of path `%1%'") % path); - /* Change ownership to the current uid. If its a symlink, use + /* Change ownership to the current uid. If it's a symlink, use lchown if available, otherwise don't bother. Wrong ownership of a symlink doesn't matter, since the owning user can't change the symlink and can't delete it because the directory is not @@ -225,14 +166,14 @@ static void _canonicalisePathMetaData(const Path & path, bool recurse) if (recurse && S_ISDIR(st.st_mode)) { Strings names = readDirectory(path); for (Strings::iterator i = names.begin(); i != names.end(); ++i) - _canonicalisePathMetaData(path + "/" + *i, true); + canonicalisePathMetaData(path + "/" + *i, true); } } void canonicalisePathMetaData(const Path & path) { - _canonicalisePathMetaData(path, true); + canonicalisePathMetaData(path, true); /* On platforms that don't have lchown(), the top-level path can't be a symlink, since we can't change its ownership. */ @@ -247,149 +188,278 @@ void canonicalisePathMetaData(const Path & path) } -bool isValidPathTxn(const Transaction & txn, const Path & path) +static Path infoFileFor(const Path & path) { - string s; - return nixDB.queryString(txn, dbValidPaths, path, s); + string baseName = baseNameOf(path); + return (format("%1%/info/%2%") % nixDBPath % baseName).str(); } -bool LocalStore::isValidPath(const Path & path) +static Path referrersFileFor(const Path & path) { - return isValidPathTxn(noTxn, path); + string baseName = baseNameOf(path); + return (format("%1%/referrer/%2%") % nixDBPath % baseName).str(); } -PathSet LocalStore::queryValidPaths() +static Path tmpFileForAtomicUpdate(const Path & path) { - Paths paths; - nixDB.enumTable(noTxn, dbValidPaths, paths); - return PathSet(paths.begin(), paths.end()); + return (format("%1%/.%2%.%3%") % dirOf(path) % getpid() % baseNameOf(path)).str(); } -static string addPrefix(const string & prefix, const string & s) +static void appendReferrer(const Path & from, const Path & to, bool lock) { - return prefix + string(1, (char) 0) + s; + Path referrersFile = referrersFileFor(from); + + PathLocks referrersLock; + if (lock) { + referrersLock.lockPaths(singleton(referrersFile)); + referrersLock.setDeletion(true); + } + + AutoCloseFD fd = open(referrersFile.c_str(), O_WRONLY | O_APPEND | O_CREAT, 0666); + if (fd == -1) throw SysError(format("opening file `%1%'") % referrersFile); + + string s = " " + to; + writeFull(fd, (const unsigned char *) s.c_str(), s.size()); } -static string stripPrefix(const string & prefix, const string & s) +/* Atomically update the referrers file. If `purge' is true, the set + of referrers is set to `referrers'. Otherwise, the current set of + referrers is purged of invalid paths. */ +void LocalStore::rewriteReferrers(const Path & path, bool purge, PathSet referrers) { - if (s.size() <= prefix.size() || - string(s, 0, prefix.size()) != prefix || - s[prefix.size()] != 0) - throw Error(format("string `%1%' is missing prefix `%2%'") - % s % prefix); - return string(s, prefix.size() + 1); + Path referrersFile = referrersFileFor(path); + + PathLocks referrersLock(singleton(referrersFile)); + referrersLock.setDeletion(true); + + if (purge) + /* queryReferrers() purges invalid paths, so that's all we + need. */ + queryReferrers(path, referrers); + + Path tmpFile = tmpFileForAtomicUpdate(referrersFile); + + AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666); + if (fd == -1) throw SysError(format("opening file `%1%'") % referrersFile); + + string s; + foreach (PathSet::const_iterator, i, referrers) { + s += " "; s += *i; + } + + writeFull(fd, (const unsigned char *) s.c_str(), s.size()); + + fd.close(); /* for Windows; can't rename open file */ + + if (rename(tmpFile.c_str(), referrersFile.c_str()) == -1) + throw SysError(format("cannot rename `%1%' to `%2%'") % tmpFile % referrersFile); } -static PathSet getReferrers(const Transaction & txn, const Path & storePath) +void LocalStore::flushDelayedUpdates() { - PathSet referrers; - Strings keys; - nixDB.enumTable(txn, dbReferrers, keys, storePath + string(1, (char) 0)); - for (Strings::iterator i = keys.begin(); i != keys.end(); ++i) - referrers.insert(stripPrefix(storePath, *i)); - return referrers; + foreach (PathSet::iterator, i, delayedUpdates) { + rewriteReferrers(*i, true, PathSet()); + } + delayedUpdates.clear(); } -void setReferences(const Transaction & txn, const Path & storePath, - const PathSet & references) +void LocalStore::registerValidPath(const Path & path, + const Hash & hash, const PathSet & references, + const Path & deriver) +{ + ValidPathInfo info; + info.path = path; + info.hash = hash; + info.references = references; + info.deriver = deriver; + registerValidPath(info); +} + + +void LocalStore::registerValidPath(const ValidPathInfo & info, bool ignoreValidity) { - /* For invalid paths, we can only clear the references. */ - if (references.size() > 0 && !isValidPathTxn(txn, storePath)) - throw Error( - format("cannot set references for invalid path `%1%'") % storePath); + Path infoFile = infoFileFor(info.path); - Paths oldReferences; - nixDB.queryStrings(txn, dbReferences, storePath, oldReferences); + ValidPathInfo oldInfo; + if (pathExists(infoFile)) oldInfo = queryPathInfo(info.path); - PathSet oldReferences2(oldReferences.begin(), oldReferences.end()); - if (oldReferences2 == references) return; - - nixDB.setStrings(txn, dbReferences, storePath, - Paths(references.begin(), references.end())); - - /* Update the referrers mappings of all new referenced paths. */ - for (PathSet::const_iterator i = references.begin(); - i != references.end(); ++i) - if (oldReferences2.find(*i) == oldReferences2.end()) - nixDB.setString(txn, dbReferrers, addPrefix(*i, storePath), ""); - - /* Remove referrer mappings from paths that are no longer - references. */ - for (Paths::iterator i = oldReferences.begin(); - i != oldReferences.end(); ++i) - if (references.find(*i) == references.end()) - nixDB.delPair(txn, dbReferrers, addPrefix(*i, storePath)); + /* Note that it's possible for infoFile to already exist. */ + + /* Acquire a lock on each referrer file. This prevents those + paths from being invalidated. (It would be a violation of the + store invariants if we registered info.path as valid while some + of its references are invalid.) NB: there can be no deadlock + here since we're acquiring the locks in sorted order. */ + PathSet lockNames; + foreach (PathSet::const_iterator, i, info.references) + if (*i != info.path) lockNames.insert(referrersFileFor(*i)); + PathLocks referrerLocks(lockNames); + referrerLocks.setDeletion(true); + + string refs; + foreach (PathSet::const_iterator, i, info.references) { + if (!refs.empty()) refs += " "; + refs += *i; + + if (!ignoreValidity && *i != info.path && !isValidPath(*i)) + throw Error(format("cannot register `%1%' as valid, because its reference `%2%' isn't valid") + % info.path % *i); + + /* Update the referrer mapping for *i. This must be done + before the info file is written to maintain the invariant + that if `path' is a valid path, then all its references + have referrer mappings back to `path'. A " " is prefixed + to separate it from the previous entry. It's not suffixed + to deal with interrupted partial writes to this file. */ + if (oldInfo.references.find(*i) == oldInfo.references.end()) + appendReferrer(*i, info.path, false); + } + + string s = (format( + "Hash: sha256:%1%\n" + "References: %2%\n" + "Deriver: %3%\n" + "Registered-At: %4%\n") + % printHash(info.hash) % refs % info.deriver % + (oldInfo.registrationTime ? oldInfo.registrationTime : time(0))).str(); + + /* Atomically rewrite the info file. */ + Path tmpFile = tmpFileForAtomicUpdate(infoFile); + writeFile(tmpFile, s); + if (rename(tmpFile.c_str(), infoFile.c_str()) == -1) + throw SysError(format("cannot rename `%1%' to `%2%'") % tmpFile % infoFile); + + pathInfoCache[info.path] = info; } -void queryReferences(const Transaction & txn, - const Path & storePath, PathSet & references) +Hash parseHashField(const Path & path, const string & s) { - Paths references2; - if (!isValidPathTxn(txn, storePath)) - throw Error(format("path `%1%' is not valid") % storePath); - nixDB.queryStrings(txn, dbReferences, storePath, references2); - references.insert(references2.begin(), references2.end()); + string::size_type colon = s.find(':'); + if (colon == string::npos) + throw Error(format("corrupt hash `%1%' in valid-path entry for `%2%'") + % s % path); + HashType ht = parseHashType(string(s, 0, colon)); + if (ht == htUnknown) + throw Error(format("unknown hash type `%1%' in valid-path entry for `%2%'") + % string(s, 0, colon) % path); + return parseHash(ht, string(s, colon + 1)); } -void LocalStore::queryReferences(const Path & storePath, - PathSet & references) +ValidPathInfo LocalStore::queryPathInfo(const Path & path) { - nix::queryReferences(noTxn, storePath, references); + ValidPathInfo res; + res.path = path; + + assertStorePath(path); + + std::map::iterator lookup = pathInfoCache.find(path); + if (lookup != pathInfoCache.end()) return lookup->second; + + //printMsg(lvlError, "queryPathInfo: " + path); + + /* Read the info file. */ + Path infoFile = infoFileFor(path); + if (!pathExists(infoFile)) + throw Error(format("path `%1%' is not valid") % path); + string info = readFile(infoFile); + + /* Parse it. */ + Strings lines = tokenizeString(info, "\n"); + + for (Strings::iterator i = lines.begin(); i != lines.end(); ++i) { + unsigned int p = i->find(':'); + if (p == string::npos) continue; /* bad line */ + string name(*i, 0, p); + string value(*i, p + 2); + if (name == "References") { + Strings refs = tokenizeString(value, " "); + res.references = PathSet(refs.begin(), refs.end()); + } else if (name == "Deriver") { + res.deriver = value; + } else if (name == "Hash") { + res.hash = parseHashField(path, value); + } else if (name == "Registered-At") { + int n = 0; + string2Int(value, n); + res.registrationTime = n; + } + } + + return pathInfoCache[path] = res; } -void queryReferrers(const Transaction & txn, - const Path & storePath, PathSet & referrers) +bool LocalStore::isValidPath(const Path & path) { - if (!isValidPathTxn(txn, storePath)) - throw Error(format("path `%1%' is not valid") % storePath); - PathSet referrers2 = getReferrers(txn, storePath); - referrers.insert(referrers2.begin(), referrers2.end()); + return pathExists(infoFileFor(path)); } -void LocalStore::queryReferrers(const Path & storePath, - PathSet & referrers) +PathSet LocalStore::queryValidPaths() { - nix::queryReferrers(noTxn, storePath, referrers); + PathSet paths; + Strings entries = readDirectory(nixDBPath + "/info"); + for (Strings::iterator i = entries.begin(); i != entries.end(); ++i) + paths.insert(nixStore + "/" + *i); + return paths; } -void setDeriver(const Transaction & txn, const Path & storePath, - const Path & deriver) +void LocalStore::queryReferences(const Path & path, + PathSet & references) { - assertStorePath(storePath); - if (deriver == "") return; - assertStorePath(deriver); - if (!isValidPathTxn(txn, storePath)) - throw Error(format("path `%1%' is not valid") % storePath); - nixDB.setString(txn, dbDerivers, storePath, deriver); + ValidPathInfo info = queryPathInfo(path); + references.insert(info.references.begin(), info.references.end()); } -static Path queryDeriver(const Transaction & txn, const Path & storePath) +bool LocalStore::queryReferrersInternal(const Path & path, PathSet & referrers) { - if (!isValidPathTxn(txn, storePath)) - throw Error(format("path `%1%' is not valid") % storePath); - Path deriver; - if (nixDB.queryString(txn, dbDerivers, storePath, deriver)) - return deriver; - else - return ""; + bool allValid = true; + + if (!isValidPath(path)) + throw Error(format("path `%1%' is not valid") % path); + + /* No locking is necessary here: updates are only done by + appending or by atomically replacing the file. When appending, + there is a possibility that we see a partial entry, but it will + just be filtered out below (the partially written path will not + be valid, so it will be ignored). */ + + Path referrersFile = referrersFileFor(path); + if (!pathExists(referrersFile)) return true; + + AutoCloseFD fd = open(referrersFile.c_str(), O_RDONLY); + if (fd == -1) throw SysError(format("opening file `%1%'") % referrersFile); + + Paths refs = tokenizeString(readFile(fd), " "); + + for (Paths::iterator i = refs.begin(); i != refs.end(); ++i) + /* Referrers can be invalid (see registerValidPath() for the + invariant), so we only return one if it is valid. */ + if (isStorePath(*i) && isValidPath(*i)) referrers.insert(*i); else allValid = false; + + return allValid; +} + + +void LocalStore::queryReferrers(const Path & path, PathSet & referrers) +{ + queryReferrersInternal(path, referrers); } Path LocalStore::queryDeriver(const Path & path) { - return nix::queryDeriver(noTxn, path); + return queryPathInfo(path).deriver; } @@ -423,95 +493,92 @@ bool LocalStore::hasSubstitutes(const Path & path) } -static void setHash(const Transaction & txn, const Path & storePath, - const Hash & hash) -{ - assert(hash.type == htSHA256); - nixDB.setString(txn, dbValidPaths, storePath, "sha256:" + printHash(hash)); -} - - -static Hash queryHash(const Transaction & txn, const Path & storePath) +Hash LocalStore::queryPathHash(const Path & path) { - string s; - nixDB.queryString(txn, dbValidPaths, storePath, s); - string::size_type colon = s.find(':'); - if (colon == string::npos) - throw Error(format("corrupt hash `%1%' in valid-path entry for `%2%'") - % s % storePath); - HashType ht = parseHashType(string(s, 0, colon)); - if (ht == htUnknown) - throw Error(format("unknown hash type `%1%' in valid-path entry for `%2%'") - % string(s, 0, colon) % storePath); - return parseHash(ht, string(s, colon + 1)); + return queryPathInfo(path).hash; } -Hash LocalStore::queryPathHash(const Path & path) +static void dfsVisit(std::map & infos, + const Path & path, PathSet & visited, Paths & sorted) { - if (!isValidPath(path)) - throw Error(format("path `%1%' is not valid") % path); - return queryHash(noTxn, path); -} + if (visited.find(path) != visited.end()) return; + visited.insert(path); + ValidPathInfo & info(infos[path]); + + for (PathSet::iterator i = info.references.begin(); + i != info.references.end(); ++i) + if (infos.find(*i) != infos.end()) + dfsVisit(infos, *i, visited, sorted); -void registerValidPath(const Transaction & txn, - const Path & path, const Hash & hash, const PathSet & references, - const Path & deriver) -{ - ValidPathInfo info; - info.path = path; - info.hash = hash; - info.references = references; - info.deriver = deriver; - ValidPathInfos infos; - infos.push_back(info); - registerValidPaths(txn, infos); + sorted.push_back(path); } -void registerValidPaths(const Transaction & txn, - const ValidPathInfos & infos) +void LocalStore::registerValidPaths(const ValidPathInfos & infos) { - PathSet newPaths; - for (ValidPathInfos::const_iterator i = infos.begin(); - i != infos.end(); ++i) - newPaths.insert(i->path); - - for (ValidPathInfos::const_iterator i = infos.begin(); - i != infos.end(); ++i) - { - assertStorePath(i->path); + std::map infosMap; + + /* Sort the paths topologically under the references relation, so + that if path A is referenced by B, then A is registered before + B. */ + for (ValidPathInfos::const_iterator i = infos.begin(); i != infos.end(); ++i) + infosMap[i->path] = *i; - debug(format("registering path `%1%'") % i->path); - setHash(txn, i->path, i->hash); + PathSet visited; + Paths sorted; + for (ValidPathInfos::const_iterator i = infos.begin(); i != infos.end(); ++i) + dfsVisit(infosMap, i->path, visited, sorted); - setReferences(txn, i->path, i->references); - - /* Check that all referenced paths are also valid (or about to - become valid). */ - for (PathSet::iterator j = i->references.begin(); - j != i->references.end(); ++j) - if (!isValidPathTxn(txn, *j) && newPaths.find(*j) == newPaths.end()) - throw Error(format("cannot register path `%1%' as valid, since its reference `%2%' is invalid") - % i->path % *j); - - setDeriver(txn, i->path, i->deriver); - } + for (Paths::iterator i = sorted.begin(); i != sorted.end(); ++i) + registerValidPath(infosMap[*i]); } /* Invalidate a path. The caller is responsible for checking that there are no referrers. */ -static void invalidatePath(Transaction & txn, const Path & path) +void LocalStore::invalidatePath(const Path & path) { debug(format("invalidating path `%1%'") % path); - /* Clear the `references' entry for this path, as well as the - inverse `referrers' entries, and the `derivers' entry. */ - setReferences(txn, path, PathSet()); - nixDB.delPair(txn, dbDerivers, path); - nixDB.delPair(txn, dbValidPaths, path); + ValidPathInfo info; + + if (pathExists(infoFileFor(path))) { + info = queryPathInfo(path); + + /* Remove the info file. */ + Path p = infoFileFor(path); + if (unlink(p.c_str()) == -1) + throw SysError(format("unlinking `%1%'") % p); + } + + /* Remove the referrers file for `path'. */ + Path p = referrersFileFor(path); + if (pathExists(p) && unlink(p.c_str()) == -1) + throw SysError(format("unlinking `%1%'") % p); + + /* Clear `path' from the info cache. */ + pathInfoCache.erase(path); + delayedUpdates.erase(path); + + /* Cause the referrer files for each path referenced by this one + to be updated. This has to happen after removing the info file + to preserve the invariant (see registerValidPath()). + + The referrer files are updated lazily in flushDelayedUpdates() + to prevent quadratic performance in the garbage collector + (i.e., when N referrers to some path X are deleted, we have to + rewrite the referrers file for X N times, causing O(N^2) I/O). + + What happens if we die before the referrer file can be updated? + That's not a problem, because stale (invalid) entries in the + referrer file are ignored by queryReferrers(). Thus a referrer + file is allowed to have stale entries; removing them is just an + optimisation. verifyStore() gets rid of them eventually. + */ + foreach (PathSet::iterator, i, info.references) + if (*i != path) delayedUpdates.insert(*i); } @@ -548,9 +615,7 @@ Path LocalStore::addToStore(const Path & _srcPath, bool fixed, canonicalisePathMetaData(dstPath); - Transaction txn(nixDB); - registerValidPath(txn, dstPath, h, PathSet(), ""); - txn.commit(); + registerValidPath(dstPath, h, PathSet(), ""); } outputLock.setDeletion(true); @@ -579,10 +644,8 @@ Path LocalStore::addTextToStore(const string & suffix, const string & s, canonicalisePathMetaData(dstPath); - Transaction txn(nixDB); - registerValidPath(txn, dstPath, + registerValidPath(dstPath, hashPath(htSHA256, dstPath), references, ""); - txn.commit(); } outputLock.setDeletion(true); @@ -774,13 +837,11 @@ Path LocalStore::importPath(bool requireSignature, Source & source) canonicalisePathMetaData(dstPath); - Transaction txn(nixDB); /* !!! if we were clever, we could prevent the hashPath() here. */ - if (!isValidPath(deriver)) deriver = ""; - registerValidPath(txn, dstPath, + if (deriver != "" && !isValidPath(deriver)) deriver = ""; + registerValidPath(dstPath, hashPath(htSHA256, dstPath), references, deriver); - txn.commit(); } outputLock.setDeletion(true); @@ -790,357 +851,142 @@ Path LocalStore::importPath(bool requireSignature, Source & source) } -void deleteFromStore(const Path & _path, unsigned long long & bytesFreed) +void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFreed) { bytesFreed = 0; - Path path(canonPath(_path)); assertStorePath(path); - Transaction txn(nixDB); - if (isValidPathTxn(txn, path)) { - PathSet referrers = getReferrers(txn, path); - for (PathSet::iterator i = referrers.begin(); - i != referrers.end(); ++i) - if (*i != path && isValidPathTxn(txn, *i)) - throw PathInUse(format("cannot delete path `%1%' because it is in use by path `%2%'") % path % *i); - invalidatePath(txn, path); + if (isValidPath(path)) { + /* Acquire a lock on the referrers file to prevent new + referrers to this path from appearing while we're deleting + it. */ + PathLocks referrersLock(singleton(referrersFileFor(path))); + referrersLock.setDeletion(true); + PathSet referrers; queryReferrers(path, referrers); + referrers.erase(path); /* ignore self-references */ + if (!referrers.empty()) + throw PathInUse(format("cannot delete path `%1%' because it is in use by `%2%'") + % path % showPaths(referrers)); + invalidatePath(path); } - txn.commit(); deletePathWrapped(path, bytesFreed); } -void verifyStore(bool checkContents) +void LocalStore::verifyStore(bool checkContents) { - Transaction txn(nixDB); - - + /* Check whether all valid paths actually exist. */ printMsg(lvlInfo, "checking path existence"); - Paths paths; - PathSet validPaths; - nixDB.enumTable(txn, dbValidPaths, paths); - - for (Paths::iterator i = paths.begin(); i != paths.end(); ++i) { + PathSet validPaths2 = queryValidPaths(), validPaths; + + for (PathSet::iterator i = validPaths2.begin(); i != validPaths2.end(); ++i) { checkInterrupt(); - if (!pathExists(*i)) { - printMsg(lvlError, format("path `%1%' disappeared") % *i); - invalidatePath(txn, *i); - } else if (!isStorePath(*i)) { + if (!isStorePath(*i)) { printMsg(lvlError, format("path `%1%' is not in the Nix store") % *i); - invalidatePath(txn, *i); - } else { - if (checkContents) { - debug(format("checking contents of `%1%'") % *i); - Hash expected = queryHash(txn, *i); - Hash current = hashPath(expected.type, *i); - if (current != expected) { - printMsg(lvlError, format("path `%1%' was modified! " - "expected hash `%2%', got `%3%'") - % *i % printHash(expected) % printHash(current)); - } - } + invalidatePath(*i); + } else if (!pathExists(*i)) { + printMsg(lvlError, format("path `%1%' disappeared") % *i); + invalidatePath(*i); + } else validPaths.insert(*i); - } } - /* Check the cleanup invariant: only valid paths can have - `references', `referrers', or `derivers' entries. */ - + /* Check the store path meta-information. */ + printMsg(lvlInfo, "checking path meta-information"); - /* Check the `derivers' table. */ - printMsg(lvlInfo, "checking the derivers table"); - Paths deriversKeys; - nixDB.enumTable(txn, dbDerivers, deriversKeys); - for (Paths::iterator i = deriversKeys.begin(); - i != deriversKeys.end(); ++i) - { - if (validPaths.find(*i) == validPaths.end()) { - printMsg(lvlError, format("removing deriver entry for invalid path `%1%'") - % *i); - nixDB.delPair(txn, dbDerivers, *i); - } - else { - Path deriver = queryDeriver(txn, *i); - if (!isStorePath(deriver)) { - printMsg(lvlError, format("removing corrupt deriver `%1%' for `%2%'") - % deriver % *i); - nixDB.delPair(txn, dbDerivers, *i); + std::map referrersCache; + + for (PathSet::iterator i = validPaths.begin(); i != validPaths.end(); ++i) { + bool update = false; + ValidPathInfo info = queryPathInfo(*i); + + /* Check the references: each reference should be valid, and + it should have a matching referrer. */ + for (PathSet::iterator j = info.references.begin(); + j != info.references.end(); ++j) + { + if (referrersCache.find(*j) == referrersCache.end()) + queryReferrers(*j, referrersCache[*j]); + if (referrersCache[*j].find(*i) == referrersCache[*j].end()) { + printMsg(lvlError, format("adding missing referrer mapping from `%1%' to `%2%'") + % *j % *i); + appendReferrer(*j, *i, true); } - } - } - - - /* Check the `references' table. */ - printMsg(lvlInfo, "checking the references table"); - Paths referencesKeys; - nixDB.enumTable(txn, dbReferences, referencesKeys); - for (Paths::iterator i = referencesKeys.begin(); - i != referencesKeys.end(); ++i) - { - if (validPaths.find(*i) == validPaths.end()) { - printMsg(lvlError, format("removing references entry for invalid path `%1%'") - % *i); - setReferences(txn, *i, PathSet()); - } - else { - PathSet references; - queryReferences(txn, *i, references); - for (PathSet::iterator j = references.begin(); - j != references.end(); ++j) - { - string dummy; - if (!nixDB.queryString(txn, dbReferrers, addPrefix(*j, *i), dummy)) { - printMsg(lvlError, format("adding missing referrer mapping from `%1%' to `%2%'") - % *j % *i); - nixDB.setString(txn, dbReferrers, addPrefix(*j, *i), ""); - } - if (validPaths.find(*j) == validPaths.end()) { - printMsg(lvlError, format("incomplete closure: `%1%' needs missing `%2%'") - % *i % *j); - } + if (validPaths.find(*j) == validPaths.end()) { + printMsg(lvlError, format("incomplete closure: `%1%' needs missing `%2%'") + % *i % *j); + /* nothing we can do about it... */ } } - } - /* Check the `referrers' table. */ - printMsg(lvlInfo, "checking the referrers table"); - Strings referrers; - nixDB.enumTable(txn, dbReferrers, referrers); - for (Strings::iterator i = referrers.begin(); i != referrers.end(); ++i) { - - /* Decode the entry (it's a tuple of paths). */ - string::size_type nul = i->find((char) 0); - if (nul == string::npos) { - printMsg(lvlError, format("removing bad referrer table entry `%1%'") % *i); - nixDB.delPair(txn, dbReferrers, *i); - continue; - } - Path to(*i, 0, nul); - Path from(*i, nul + 1); - - if (validPaths.find(to) == validPaths.end()) { - printMsg(lvlError, format("removing referrer entry from `%1%' to invalid `%2%'") - % from % to); - nixDB.delPair(txn, dbReferrers, *i); + /* Check the deriver. (Note that the deriver doesn't have to + be a valid path.) */ + if (!info.deriver.empty() && !isStorePath(info.deriver)) { + info.deriver = ""; + update = true; } - else if (validPaths.find(from) == validPaths.end()) { - printMsg(lvlError, format("removing referrer entry from invalid `%1%' to `%2%'") - % from % to); - nixDB.delPair(txn, dbReferrers, *i); - } - - else { - PathSet references; - queryReferences(txn, from, references); - if (find(references.begin(), references.end(), to) == references.end()) { - printMsg(lvlError, format("adding missing referrer mapping from `%1%' to `%2%'") - % from % to); - references.insert(to); - setReferences(txn, from, references); + /* Check the content hash (optionally - slow). */ + if (checkContents) { + debug(format("checking contents of `%1%'") % *i); + Hash current = hashPath(info.hash.type, *i); + if (current != info.hash) { + printMsg(lvlError, format("path `%1%' was modified! " + "expected hash `%2%', got `%3%'") + % *i % printHash(info.hash) % printHash(current)); } } - - } - - txn.commit(); -} - - -typedef std::map > HashToPath; - - -static void makeWritable(const Path & path) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); - if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError(format("changing writability of `%1%'") % path); -} - - -static void hashAndLink(bool dryRun, HashToPath & hashToPath, - OptimiseStats & stats, const Path & path) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); - - /* Sometimes SNAFUs can cause files in the Nix store to be - modified, in particular when running programs as root under - NixOS (example: $fontconfig/var/cache being modified). Skip - those files. */ - if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { - printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path); - return; + if (update) registerValidPath(info); } - /* We can hard link regular files and symlinks. */ - if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) { - - /* Hash the file. Note that hashPath() returns the hash over - the NAR serialisation, which includes the execute bit on - the file. Thus, executable and non-executable files with - the same contents *won't* be linked (which is good because - otherwise the permissions would be screwed up). - - Also note that if `path' is a symlink, then we're hashing - the contents of the symlink (i.e. the result of - readlink()), not the contents of the target (which may not - even exist). */ - Hash hash = hashPath(htSHA256, path); - stats.totalFiles++; - printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); - - std::pair prevPath = hashToPath[hash]; - - if (prevPath.first == "") { - hashToPath[hash] = std::pair(path, st.st_ino); - return; - } - - /* Yes! We've seen a file with the same contents. Replace - the current file with a hard link to that file. */ - stats.sameContents++; - if (prevPath.second == st.st_ino) { - printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % prevPath.first); - return; - } - - if (!dryRun) { - - printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % prevPath.first); - - Path tempLink = (format("%1%.tmp-%2%-%3%") - % path % getpid() % rand()).str(); - - /* Make the containing directory writable, but only if - it's not the store itself (we don't want or need to - mess with its permissions). */ - bool mustToggle = !isStorePath(path); - if (mustToggle) makeWritable(dirOf(path)); - - if (link(prevPath.first.c_str(), tempLink.c_str()) == -1) - throw SysError(format("cannot link `%1%' to `%2%'") - % tempLink % prevPath.first); - - /* Atomically replace the old file with the new hard link. */ - if (rename(tempLink.c_str(), path.c_str()) == -1) - throw SysError(format("cannot rename `%1%' to `%2%'") - % tempLink % path); - - /* Make the directory read-only again and reset its - timestamp back to 0. */ - if (mustToggle) _canonicalisePathMetaData(dirOf(path), false); - - } else - printMsg(lvlTalkative, format("would link `%1%' to `%2%'") % path % prevPath.first); - - stats.filesLinked++; - stats.bytesFreed += st.st_size; - } - - if (S_ISDIR(st.st_mode)) { - Strings names = readDirectory(path); - for (Strings::iterator i = names.begin(); i != names.end(); ++i) - hashAndLink(dryRun, hashToPath, stats, path + "/" + *i); - } -} - - -void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats) -{ - HashToPath hashToPath; + referrersCache.clear(); - Paths paths; - PathSet validPaths; - nixDB.enumTable(noTxn, dbValidPaths, paths); - - for (Paths::iterator i = paths.begin(); i != paths.end(); ++i) { - addTempRoot(*i); - if (!isValidPath(*i)) continue; /* path was GC'ed, probably */ - startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i); - hashAndLink(dryRun, hashToPath, stats, *i); - } -} + /* Check the referrers. */ + printMsg(lvlInfo, "checking referrers"); -/* Upgrade from schema 2 (0.8 <= Nix <= 0.9) to schema 3 (Nix >= - 0.10). The only thing to do here is to upgrade the old `referer' - table (which causes quadratic complexity in some cases) to the new - (and properly spelled) `referrer' table. */ -static void upgradeStore09() -{ - /* !!! we should disallow concurrent upgrades */ + std::map referencesCache; - if (!pathExists(nixDBPath + "/referers")) return; - - printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); - - Transaction txn(nixDB); - - std::cerr << "converting referers to referrers..."; - - TableId dbReferers = nixDB.openTable("referers"); /* sic! */ - - Paths referersKeys; - nixDB.enumTable(txn, dbReferers, referersKeys); - - int n = 0; - for (Paths::iterator i = referersKeys.begin(); - i != referersKeys.end(); ++i) - { - Paths referers; - nixDB.queryStrings(txn, dbReferers, *i, referers); - for (Paths::iterator j = referers.begin(); - j != referers.end(); ++j) - nixDB.setString(txn, dbReferrers, addPrefix(*i, *j), ""); - if (++n % 1000 == 0) { - txn.commit(); - txn.begin(nixDB); - std::cerr << "|"; + Strings entries = readDirectory(nixDBPath + "/referrer"); + for (Strings::iterator i = entries.begin(); i != entries.end(); ++i) { + Path from = nixStore + "/" + *i; + + if (validPaths.find(from) == validPaths.end()) { + printMsg(lvlError, format("removing referrers file for invalid `%1%'") % from); + Path p = referrersFileFor(from); + if (unlink(p.c_str()) == -1) + throw SysError(format("unlinking `%1%'") % p); + continue; } - std::cerr << "."; - } - - txn.commit(); - - std::cerr << std::endl; - nixDB.closeTable(dbReferers); + PathSet referrers; + bool allValid = queryReferrersInternal(from, referrers); + bool update = false; - nixDB.deleteTable("referers"); -} - - -/* Upgrade from schema 3 (Nix 0.10) to schema 4 (Nix >= 0.11). The - only thing to do here is to delete the substitutes table and get - rid of invalid but substitutable references/referrers. */ -static void upgradeStore11() -{ - if (!pathExists(nixDBPath + "/substitutes")) return; - - printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); + if (!allValid) { + printMsg(lvlError, format("removing some stale referrers for `%1%'") % from); + update = true; + } - Transaction txn(nixDB); - TableId dbSubstitutes = nixDB.openTable("substitutes"); + /* Each referrer should have a matching reference. */ + PathSet referrersNew; + for (PathSet::iterator j = referrers.begin(); j != referrers.end(); ++j) { + if (referencesCache.find(*j) == referencesCache.end()) + queryReferences(*j, referencesCache[*j]); + if (referencesCache[*j].find(from) == referencesCache[*j].end()) { + printMsg(lvlError, format("removing unexpected referrer mapping from `%1%' to `%2%'") + % from % *j); + update = true; + } else referrersNew.insert(*j); + } - Paths subKeys; - nixDB.enumTable(txn, dbSubstitutes, subKeys); - for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) { - if (!isValidPathTxn(txn, *i)) - invalidatePath(txn, *i); + if (update) rewriteReferrers(from, false, referrersNew); } - - txn.commit(); - nixDB.closeTable(dbSubstitutes); - nixDB.deleteTable("substitutes"); } -- cgit 1.4.1