From e2ef5e07fdc142670f7f3161d3133ff04e99d342 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Nov 2006 17:43:04 +0000 Subject: * Refactoring. There is now an abstract interface class StoreAPI containing functions that operate on the Nix store. One implementation is LocalStore, which operates on the Nix store directly. The next step, to enable secure multi-user Nix, is to create a different implementation RemoteStore that talks to a privileged daemon process that uses LocalStore to perform the actual operations. --- src/libstore/Makefile.am | 8 +- src/libstore/build.cc | 32 +- src/libstore/derivations.cc | 4 +- src/libstore/gc.cc | 16 +- src/libstore/local-store.cc | 1037 ++++++++++++++++++++++++++++++++++++++++ src/libstore/local-store.hh | 136 ++++++ src/libstore/misc.cc | 18 +- src/libstore/store-api.cc | 107 +++++ src/libstore/store-api.hh | 115 +++++ src/libstore/store.cc | 1103 ------------------------------------------- src/libstore/store.hh | 178 ------- 11 files changed, 1436 insertions(+), 1318 deletions(-) create mode 100644 src/libstore/local-store.cc create mode 100644 src/libstore/local-store.hh create mode 100644 src/libstore/store-api.cc create mode 100644 src/libstore/store-api.hh delete mode 100644 src/libstore/store.cc delete mode 100644 src/libstore/store.hh (limited to 'src/libstore') diff --git a/src/libstore/Makefile.am b/src/libstore/Makefile.am index 515311efac78..209a1e69291c 100644 --- a/src/libstore/Makefile.am +++ b/src/libstore/Makefile.am @@ -1,12 +1,12 @@ pkglib_LTLIBRARIES = libstore.la libstore_la_SOURCES = \ - store.cc derivations.cc build.cc misc.cc globals.cc db.cc \ - references.cc pathlocks.cc gc.cc + store-api.cc local-store.cc derivations.cc build.cc misc.cc globals.cc \ + db.cc references.cc pathlocks.cc gc.cc pkginclude_HEADERS = \ - store.hh derivations.hh build.hh misc.hh globals.hh db.hh \ - references.hh pathlocks.hh gc.hh + store-api.hh local-store.cc derivations.hh build.hh misc.hh globals.hh \ + db.hh references.hh pathlocks.hh gc.hh libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 7058bd12b35b..a78d5010cb66 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4,7 +4,7 @@ #include "misc.hh" #include "globals.hh" #include "gc.hh" -#include "store.hh" +#include "local-store.hh" #include "db.hh" #include "util.hh" @@ -636,7 +636,7 @@ void DerivationGoal::haveStoreExpr() return; } - assert(isValidPath(drvPath)); + assert(store->isValidPath(drvPath)); /* Get the derivation. */ drv = derivationFromPath(drvPath); @@ -661,7 +661,7 @@ void DerivationGoal::haveStoreExpr() i != invalidOutputs.end(); ++i) /* Don't bother creating a substitution goal if there are no substitutes. */ - if (querySubstitutes(noTxn, *i).size() > 0) + if (store->querySubstitutes(*i).size() > 0) addWaitee(worker.makeSubstitutionGoal(*i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ @@ -916,7 +916,7 @@ static string makeValidityRegistration(const PathSet & paths, s += deriver + "\n"; PathSet references; - queryReferences(noTxn, *i, references); + store->queryReferences(*i, references); s += (format("%1%\n") % references.size()).str(); @@ -1130,7 +1130,7 @@ bool DerivationGoal::prepareBuild() /* Add the relevant output closures of the input derivation `*i' as input paths. Only add the closures of output paths that are specified as inputs. */ - assert(isValidPath(i->first)); + assert(store->isValidPath(i->first)); Derivation inDrv = derivationFromPath(i->first); for (StringSet::iterator j = i->second.begin(); j != i->second.end(); ++j) @@ -1172,7 +1172,7 @@ void DerivationGoal::startBuilder() i != drv.outputs.end(); ++i) { Path path = i->second.path; - if (isValidPath(path)) + if (store->isValidPath(path)) throw Error(format("obstructed build: path `%1%' exists") % path); if (pathExists(path)) { debug(format("removing unregistered path `%1%'") % path); @@ -1259,7 +1259,7 @@ void DerivationGoal::startBuilder() for (Strings::iterator i = ss.begin(); i != ss.end(); ) { string fileName = *i++; Path storePath = *i++; - if (!isValidPath(storePath)) + if (!store->isValidPath(storePath)) throw Error(format("`exportReferencesGraph' refers to an invalid path `%1%'") % storePath); checkStoreName(fileName); /* !!! abuse of this function */ @@ -1630,7 +1630,7 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid) PathSet result; for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i) - if (isValidPath(i->second.path)) { + if (store->isValidPath(i->second.path)) { if (returnValid) result.insert(i->second.path); } else { if (!returnValid) result.insert(i->second.path); @@ -1718,17 +1718,21 @@ void SubstitutionGoal::init() addTempRoot(storePath); /* If the path already exists we're done. */ - if (isValidPath(storePath)) { + if (store->isValidPath(storePath)) { amDone(); return; } + /* !!! race condition; should get the substitutes and the + references in a transaction (in case a clearSubstitutes() is + done simultaneously). */ + /* Read the substitutes. */ - subs = querySubstitutes(noTxn, storePath); + subs = store->querySubstitutes(storePath); /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ - queryReferences(noTxn, storePath, references); + store->queryReferences(storePath, references); for (PathSet::iterator i = references.begin(); i != references.end(); ++i) @@ -1752,7 +1756,7 @@ void SubstitutionGoal::referencesValid() for (PathSet::iterator i = references.begin(); i != references.end(); ++i) if (*i != storePath) /* ignore self-references */ - assert(isValidPath(*i)); + assert(store->isValidPath(*i)); tryNext(); } @@ -1796,7 +1800,7 @@ void SubstitutionGoal::tryToRun() (format("waiting for lock on `%1%'") % storePath).str()); /* Check again whether the path is invalid. */ - if (isValidPath(storePath)) { + if (store->isValidPath(storePath)) { debug(format("store path `%1%' has become valid") % storePath); outputLock->setDeletion(true); amDone(); @@ -2221,7 +2225,7 @@ void buildDerivations(const PathSet & drvPaths) void ensurePath(const Path & path) { /* If the path is already valid, we're done. */ - if (isValidPath(path)) return; + if (store->isValidPath(path)) return; Worker worker; GoalPtr goal = worker.makeSubstitutionGoal(path); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index aeab675b2354..d159d47a5e4a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1,5 +1,5 @@ #include "derivations.hh" -#include "store.hh" +#include "store-api.hh" #include "aterm.hh" #include "derivations-ast.hh" @@ -25,7 +25,7 @@ Path writeDerivation(const Derivation & drv, const string & name) /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be held during a garbage collection). */ - return addTextToStore(name + drvExtension, + return store->addTextToStore(name + drvExtension, atPrint(unparseDerivation(drv)), references); } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 3a626dedae98..05966ad4b076 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -2,7 +2,7 @@ #include "globals.hh" #include "misc.hh" #include "pathlocks.hh" -#include "store.hh" +#include "local-store.hh" #include "db.hh" #include "util.hh" @@ -305,7 +305,7 @@ static void findRoots(const Path & path, bool recurseSymlinks, debug(format("found root `%1%' in `%2%'") % target2 % path); Path target3 = toStorePath(target2); - if (isValidPath(target3)) + if (store->isValidPath(target3)) roots.insert(target3); else printMsg(lvlInfo, format("skipping invalid root from `%1%' to `%2%'") @@ -343,7 +343,7 @@ static void addAdditionalRoots(PathSet & roots) for (Strings::iterator i = paths.begin(); i != paths.end(); ++i) { if (isInStore(*i)) { Path path = toStorePath(*i); - if (roots.find(path) == roots.end() && isValidPath(path)) { + if (roots.find(path) == roots.end() && store->isValidPath(path)) { debug(format("found additional root `%1%'") % path); roots.insert(path); } @@ -359,8 +359,8 @@ static void dfsVisit(const PathSet & paths, const Path & path, visited.insert(path); PathSet references; - if (isValidPath(path)) - queryReferences(noTxn, path, references); + if (store->isValidPath(path)) + store->queryReferences(path, references); for (PathSet::iterator i = references.begin(); i != references.end(); ++i) @@ -431,7 +431,7 @@ void collectGarbage(GCAction action, const PathSet & pathsToDelete, previously ran the collector with `gcKeepDerivations' turned off). */ Path deriver = queryDeriver(noTxn, *i); - if (deriver != "" && isValidPath(deriver)) + if (deriver != "" && store->isValidPath(deriver)) computeFSClosure(deriver, livePaths); } } @@ -444,7 +444,7 @@ void collectGarbage(GCAction action, const PathSet & pathsToDelete, Derivation drv = derivationFromPath(*i); for (DerivationOutputs::iterator j = drv.outputs.begin(); j != drv.outputs.end(); ++j) - if (isValidPath(j->second.path)) + if (store->isValidPath(j->second.path)) computeFSClosure(j->second.path, livePaths); } } @@ -469,7 +469,7 @@ void collectGarbage(GCAction action, const PathSet & pathsToDelete, means that it has already been closed). */ PathSet tempRootsClosed; for (PathSet::iterator i = tempRoots.begin(); i != tempRoots.end(); ++i) - if (isValidPath(*i)) + if (store->isValidPath(*i)) computeFSClosure(*i, tempRootsClosed); else tempRootsClosed.insert(*i); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc new file mode 100644 index 000000000000..c8bb60bce6b0 --- /dev/null +++ b/src/libstore/local-store.cc @@ -0,0 +1,1037 @@ +#include "local-store.hh" +#include "util.hh" +#include "globals.hh" +#include "db.hh" +#include "archive.hh" +#include "pathlocks.hh" +#include "gc.hh" +#include "aterm.hh" +#include "derivations-ast.hh" + +#include +#include + +#include +#include +#include +#include + +namespace nix { + + +/* Nix database. */ +static Database nixDB; + + +/* Database tables. */ + +/* dbValidPaths :: Path -> () + + The existence of a key $p$ indicates that path $p$ is valid (that + is, produced by a succesful build). */ +static TableId dbValidPaths = 0; + +/* dbReferences :: Path -> [Path] + + This table lists the outgoing file system references for each + output path that has been built by a Nix derivation. These are + found by scanning the path for the hash components of input + paths. */ +static TableId dbReferences = 0; + +/* dbReferrers :: Path -> Path + + This table is just the reverse mapping of dbReferences. This table + can have duplicate keys, each corresponding value denoting a single + referrer. */ +static TableId dbReferrers = 0; + +/* dbSubstitutes :: Path -> [[Path]] + + Each pair $(p, subs)$ tells Nix that it can use any of the + substitutes in $subs$ to build path $p$. Each substitute defines a + command-line invocation of a program (i.e., the first list element + is the full path to the program, the remaining elements are + arguments). + + The main purpose of this is for distributed caching of derivates. + One system can compute a derivate and put it on a website (as a Nix + archive), for instance, and then another system can register a + substitute for that derivate. The substitute in this case might be + a Nix derivation that fetches the Nix archive. +*/ +static TableId dbSubstitutes = 0; + +/* dbDerivers :: Path -> [Path] + + This table lists the derivation used to build a path. There can + only be multiple such paths for fixed-output derivations (i.e., + derivations specifying an expected hash). */ +static TableId dbDerivers = 0; + + +bool Substitute::operator == (const Substitute & sub) const +{ + return program == sub.program + && args == sub.args; +} + + +static void upgradeStore07(); +static void upgradeStore09(); + + +void checkStoreNotSymlink() +{ + if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; + Path path = nixStore; + struct stat st; + while (path != "/") { + if (lstat(path.c_str(), &st)) + throw SysError(format("getting status of `%1%'") % path); + if (S_ISLNK(st.st_mode)) + throw Error(format( + "the path `%1%' is a symlink; " + "this is not allowed for the Nix store and its parent directories") + % path); + path = dirOf(path); + } +} + + +LocalStore::LocalStore(bool reserveSpace) +{ + if (readOnlyMode) return; + + checkStoreNotSymlink(); + + try { + Path reservedPath = nixDBPath + "/reserved"; + string s = querySetting("gc-reserved-space", ""); + int reservedSize; + if (!string2Int(s, reservedSize)) reservedSize = 1024 * 1024; + if (reserveSpace) { + struct stat st; + if (stat(reservedPath.c_str(), &st) == -1 || + st.st_size != reservedSize) + writeFile(reservedPath, string(reservedSize, 'X')); + } + else + deletePath(reservedPath); + } catch (SysError & e) { /* don't care about errors */ + } + + try { + nixDB.open(nixDBPath); + } catch (DbNoPermission & e) { + printMsg(lvlTalkative, "cannot access Nix database; continuing anyway"); + readOnlyMode = true; + return; + } + dbValidPaths = nixDB.openTable("validpaths"); + dbReferences = nixDB.openTable("references"); + dbReferrers = nixDB.openTable("referrers", true); /* must be sorted */ + dbSubstitutes = nixDB.openTable("substitutes"); + dbDerivers = nixDB.openTable("derivers"); + + int curSchema = 0; + Path schemaFN = nixDBPath + "/schema"; + if (pathExists(schemaFN)) { + string s = readFile(schemaFN); + if (!string2Int(s, curSchema)) + throw Error(format("`%1%' is corrupt") % schemaFN); + } + + if (curSchema > nixSchemaVersion) + throw Error(format("current Nix store schema is version %1%, but I only support %2%") + % curSchema % nixSchemaVersion); + + if (curSchema < nixSchemaVersion) { + if (curSchema <= 1) + upgradeStore07(); + if (curSchema == 2) + upgradeStore09(); + writeFile(schemaFN, (format("%1%") % nixSchemaVersion).str()); + } +} + + +LocalStore::~LocalStore() +{ + /* If the database isn't open, this is a NOP. */ + nixDB.close(); +} + + +void createStoreTransaction(Transaction & txn) +{ + Transaction txn2(nixDB); + txn2.moveTo(txn); +} + + +/* Path copying. */ + +struct CopySink : DumpSink +{ + string s; + virtual void operator () (const unsigned char * data, unsigned int len) + { + s.append((const char *) data, len); + } +}; + + +struct CopySource : RestoreSource +{ + string & s; + unsigned int pos; + CopySource(string & _s) : s(_s), pos(0) { } + virtual void operator () (unsigned char * data, unsigned int len) + { + s.copy((char *) data, len, pos); + pos += len; + assert(pos <= s.size()); + } +}; + + +void copyPath(const Path & src, const Path & dst) +{ + debug(format("copying `%1%' to `%2%'") % src % dst); + + /* Dump an archive of the path `src' into a string buffer, then + restore the archive to `dst'. This is not a very good method + for very large paths, but `copyPath' is mainly used for small + files. */ + + CopySink sink; + { + SwitchToOriginalUser sw; + dumpPath(src, sink); + } + + CopySource source(sink.s); + restorePath(dst, source); +} + + +void canonicalisePathMetaData(const Path & path) +{ + checkInterrupt(); + + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + + if (!S_ISLNK(st.st_mode)) { + + /* Mask out all type related bits. */ + mode_t mode = st.st_mode & ~S_IFMT; + + if (mode != 0444 && mode != 0555) { + mode = (st.st_mode & S_IFMT) + | 0444 + | (st.st_mode & S_IXUSR ? 0111 : 0); + if (chmod(path.c_str(), mode) == -1) + throw SysError(format("changing mode of `%1%' to %2$o") % path % mode); + } + + if (st.st_uid != getuid() || st.st_gid != getgid()) { + if (chown(path.c_str(), getuid(), getgid()) == -1) + throw SysError(format("changing owner/group of `%1%' to %2%/%3%") + % path % getuid() % getgid()); + } + + if (st.st_mtime != 0) { + struct utimbuf utimbuf; + utimbuf.actime = st.st_atime; + utimbuf.modtime = 0; + if (utime(path.c_str(), &utimbuf) == -1) + throw SysError(format("changing modification time of `%1%'") % path); + } + + } + + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + for (Strings::iterator i = names.begin(); i != names.end(); ++i) + canonicalisePathMetaData(path + "/" + *i); + } +} + + +bool isValidPathTxn(const Transaction & txn, const Path & path) +{ + string s; + return nixDB.queryString(txn, dbValidPaths, path, s); +} + + +bool LocalStore::isValidPath(const Path & path) +{ + return isValidPathTxn(noTxn, path); +} + + +static Substitutes readSubstitutes(const Transaction & txn, + const Path & srcPath); + + +static bool isRealisablePath(const Transaction & txn, const Path & path) +{ + return isValidPathTxn(txn, path) + || readSubstitutes(txn, path).size() > 0; +} + + +static string addPrefix(const string & prefix, const string & s) +{ + return prefix + string(1, (char) 0) + s; +} + + +static string stripPrefix(const string & prefix, const string & s) +{ + if (s.size() <= prefix.size() || + string(s, 0, prefix.size()) != prefix || + s[prefix.size()] != 0) + throw Error(format("string `%1%' is missing prefix `%2%'") + % s % prefix); + return string(s, prefix.size() + 1); +} + + +static PathSet getReferrers(const Transaction & txn, const Path & storePath) +{ + PathSet referrers; + Strings keys; + nixDB.enumTable(txn, dbReferrers, keys, storePath + string(1, (char) 0)); + for (Strings::iterator i = keys.begin(); i != keys.end(); ++i) + referrers.insert(stripPrefix(storePath, *i)); + return referrers; +} + + +void setReferences(const Transaction & txn, const Path & storePath, + const PathSet & references) +{ + /* For unrealisable paths, we can only clear the references. */ + if (references.size() > 0 && !isRealisablePath(txn, storePath)) + throw Error( + format("cannot set references for path `%1%' which is invalid and has no substitutes") + % storePath); + + Paths oldReferences; + nixDB.queryStrings(txn, dbReferences, storePath, oldReferences); + + PathSet oldReferences2(oldReferences.begin(), oldReferences.end()); + if (oldReferences2 == references) return; + + nixDB.setStrings(txn, dbReferences, storePath, + Paths(references.begin(), references.end())); + + /* Update the referrers mappings of all new referenced paths. */ + for (PathSet::const_iterator i = references.begin(); + i != references.end(); ++i) + if (oldReferences2.find(*i) == oldReferences2.end()) + nixDB.setString(txn, dbReferrers, addPrefix(*i, storePath), ""); + + /* Remove referrer mappings from paths that are no longer + references. */ + for (Paths::iterator i = oldReferences.begin(); + i != oldReferences.end(); ++i) + if (references.find(*i) == references.end()) + nixDB.delPair(txn, dbReferrers, addPrefix(*i, storePath)); +} + + +void queryReferences(const Transaction & txn, + const Path & storePath, PathSet & references) +{ + Paths references2; + if (!isRealisablePath(txn, storePath)) + throw Error(format("path `%1%' is not valid") % storePath); + nixDB.queryStrings(txn, dbReferences, storePath, references2); + references.insert(references2.begin(), references2.end()); +} + + +void LocalStore::queryReferences(const Path & storePath, + PathSet & references) +{ + nix::queryReferences(noTxn, storePath, references); +} + + +void queryReferrers(const Transaction & txn, + const Path & storePath, PathSet & referrers) +{ + if (!isRealisablePath(txn, storePath)) + throw Error(format("path `%1%' is not valid") % storePath); + PathSet referrers2 = getReferrers(txn, storePath); + referrers.insert(referrers2.begin(), referrers2.end()); +} + + +void LocalStore::queryReferrers(const Path & storePath, + PathSet & referrers) +{ + nix::queryReferrers(noTxn, storePath, referrers); +} + + +void setDeriver(const Transaction & txn, const Path & storePath, + const Path & deriver) +{ + assertStorePath(storePath); + if (deriver == "") return; + assertStorePath(deriver); + if (!isRealisablePath(txn, storePath)) + throw Error(format("path `%1%' is not valid") % storePath); + nixDB.setString(txn, dbDerivers, storePath, deriver); +} + + +Path queryDeriver(const Transaction & txn, const Path & storePath) +{ + if (!isRealisablePath(txn, storePath)) + throw Error(format("path `%1%' is not valid") % storePath); + Path deriver; + if (nixDB.queryString(txn, dbDerivers, storePath, deriver)) + return deriver; + else + return ""; +} + + +const int substituteVersion = 2; + + +static Substitutes readSubstitutes(const Transaction & txn, + const Path & srcPath) +{ + Strings ss; + nixDB.queryStrings(txn, dbSubstitutes, srcPath, ss); + + Substitutes subs; + + for (Strings::iterator i = ss.begin(); i != ss.end(); ++i) { + if (i->size() < 4 || (*i)[3] != 0) { + /* Old-style substitute. !!! remove this code + eventually? */ + break; + } + Strings ss2 = unpackStrings(*i); + if (ss2.size() == 0) continue; + int version; + if (!string2Int(ss2.front(), version)) continue; + if (version != substituteVersion) continue; + if (ss2.size() != 4) throw Error("malformed substitute"); + Strings::iterator j = ss2.begin(); + j++; + Substitute sub; + sub.deriver = *j++; + sub.program = *j++; + sub.args = unpackStrings(*j++); + subs.push_back(sub); + } + + return subs; +} + + +static void writeSubstitutes(const Transaction & txn, + const Path & srcPath, const Substitutes & subs) +{ + Strings ss; + + for (Substitutes::const_iterator i = subs.begin(); + i != subs.end(); ++i) + { + Strings ss2; + ss2.push_back((format("%1%") % substituteVersion).str()); + ss2.push_back(i->deriver); + ss2.push_back(i->program); + ss2.push_back(packStrings(i->args)); + ss.push_back(packStrings(ss2)); + } + + nixDB.setStrings(txn, dbSubstitutes, srcPath, ss); +} + + +void registerSubstitute(const Transaction & txn, + const Path & srcPath, const Substitute & sub) +{ + assertStorePath(srcPath); + + Substitutes subs = readSubstitutes(txn, srcPath); + + if (find(subs.begin(), subs.end(), sub) != subs.end()) + return; + + /* New substitutes take precedence over old ones. If the + substitute is already present, it's moved to the front. */ + remove(subs.begin(), subs.end(), sub); + subs.push_front(sub); + + writeSubstitutes(txn, srcPath, subs); +} + + +Substitutes querySubstitutes(const Transaction & txn, const Path & srcPath) +{ + return readSubstitutes(txn, srcPath); +} + + +Substitutes LocalStore::querySubstitutes(const Path & srcPath) +{ + return nix::querySubstitutes(noTxn, srcPath); +} + + +static void invalidatePath(Transaction & txn, const Path & path); + + +void clearSubstitutes() +{ + Transaction txn(nixDB); + + /* Iterate over all paths for which there are substitutes. */ + Paths subKeys; + nixDB.enumTable(txn, dbSubstitutes, subKeys); + for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) { + + /* Delete all substitutes for path *i. */ + nixDB.delPair(txn, dbSubstitutes, *i); + + /* Maintain the cleanup invariant. */ + if (!isValidPathTxn(txn, *i)) + invalidatePath(txn, *i); + } + + /* !!! there should be no referrers to any of the invalid + substitutable paths. This should be the case by construction + (the only referrers can be other invalid substitutable paths, + which have all been removed now). */ + + txn.commit(); +} + + +static void setHash(const Transaction & txn, const Path & storePath, + const Hash & hash) +{ + assert(hash.type == htSHA256); + nixDB.setString(txn, dbValidPaths, storePath, "sha256:" + printHash(hash)); +} + + +static Hash queryHash(const Transaction & txn, const Path & storePath) +{ + string s; + nixDB.queryString(txn, dbValidPaths, storePath, s); + string::size_type colon = s.find(':'); + if (colon == string::npos) + throw Error(format("corrupt hash `%1%' in valid-path entry for `%2%'") + % s % storePath); + HashType ht = parseHashType(string(s, 0, colon)); + if (ht == htUnknown) + throw Error(format("unknown hash type `%1%' in valid-path entry for `%2%'") + % string(s, 0, colon) % storePath); + return parseHash(ht, string(s, colon + 1)); +} + + +Hash LocalStore::queryPathHash(const Path & path) +{ + if (!isValidPath(path)) + throw Error(format("path `%1%' is not valid") % path); + return queryHash(noTxn, path); +} + + +void registerValidPath(const Transaction & txn, + const Path & path, const Hash & hash, const PathSet & references, + const Path & deriver) +{ + ValidPathInfo info; + info.path = path; + info.hash = hash; + info.references = references; + info.deriver = deriver; + ValidPathInfos infos; + infos.push_back(info); + registerValidPaths(txn, infos); +} + + +void registerValidPaths(const Transaction & txn, + const ValidPathInfos & infos) +{ + PathSet newPaths; + for (ValidPathInfos::const_iterator i = infos.begin(); + i != infos.end(); ++i) + newPaths.insert(i->path); + + for (ValidPathInfos::const_iterator i = infos.begin(); + i != infos.end(); ++i) + { + assertStorePath(i->path); + + debug(format("registering path `%1%'") % i->path); + setHash(txn, i->path, i->hash); + + setReferences(txn, i->path, i->references); + + /* Check that all referenced paths are also valid (or about to + become valid). */ + for (PathSet::iterator j = i->references.begin(); + j != i->references.end(); ++j) + if (!isValidPathTxn(txn, *j) && newPaths.find(*j) == newPaths.end()) + throw Error(format("cannot register path `%1%' as valid, since its reference `%2%' is invalid") + % i->path % *j); + + setDeriver(txn, i->path, i->deriver); + } +} + + +/* Invalidate a path. The caller is responsible for checking that + there are no referrers. */ +static void invalidatePath(Transaction & txn, const Path & path) +{ + debug(format("unregistering path `%1%'") % path); + + /* Clear the `references' entry for this path, as well as the + inverse `referrers' entries, and the `derivers' entry; but only + if there are no substitutes for this path. This maintains the + cleanup invariant. */ + if (querySubstitutes(txn, path).size() == 0) { + setReferences(txn, path, PathSet()); + nixDB.delPair(txn, dbDerivers, path); + } + + nixDB.delPair(txn, dbValidPaths, path); +} + + +Path LocalStore::_addToStore(bool fixed, bool recursive, + string hashAlgo, const Path & _srcPath) +{ + Path srcPath(absPath(_srcPath)); + debug(format("adding `%1%' to the store") % srcPath); + + Hash h(htSHA256); + { + SwitchToOriginalUser sw; + h = hashPath(htSHA256, srcPath); + } + + string baseName = baseNameOf(srcPath); + + Path dstPath; + + if (fixed) { + + HashType ht(parseHashType(hashAlgo)); + Hash h2(ht); + { + SwitchToOriginalUser sw; + h2 = recursive ? hashPath(ht, srcPath) : hashFile(ht, srcPath); + } + + dstPath = makeFixedOutputPath(recursive, hashAlgo, h2, baseName); + } + + else dstPath = makeStorePath("source", h, baseName); + + if (!readOnlyMode) addTempRoot(dstPath); + + if (!readOnlyMode && !isValidPath(dstPath)) { + + /* The first check above is an optimisation to prevent + unnecessary lock acquisition. */ + + PathLocks outputLock(singleton(dstPath)); + + if (!isValidPath(dstPath)) { + + if (pathExists(dstPath)) deletePath(dstPath); + + copyPath(srcPath, dstPath); + + Hash h2 = hashPath(htSHA256, dstPath); + if (h != h2) + throw Error(format("contents of `%1%' changed while copying it to `%2%' (%3% -> %4%)") + % srcPath % dstPath % printHash(h) % printHash(h2)); + + canonicalisePathMetaData(dstPath); + + Transaction txn(nixDB); + registerValidPath(txn, dstPath, h, PathSet(), ""); + txn.commit(); + } + + outputLock.setDeletion(true); + } + + return dstPath; +} + + +Path LocalStore::addToStore(const Path & srcPath) +{ + return _addToStore(false, false, "", srcPath); +} + + +Path LocalStore::addToStoreFixed(bool recursive, string hashAlgo, const Path & srcPath) +{ + return _addToStore(true, recursive, hashAlgo, srcPath); +} + + +Path LocalStore::addTextToStore(const string & suffix, const string & s, + const PathSet & references) +{ + Hash hash = hashString(htSHA256, s); + + Path dstPath = makeStorePath("text", hash, suffix); + + if (!readOnlyMode) addTempRoot(dstPath); + + if (!readOnlyMode && !isValidPath(dstPath)) { + + PathLocks outputLock(singleton(dstPath)); + + if (!isValidPath(dstPath)) { + + if (pathExists(dstPath)) deletePath(dstPath); + + writeStringToFile(dstPath, s); + + canonicalisePathMetaData(dstPath); + + Transaction txn(nixDB); + registerValidPath(txn, dstPath, + hashPath(htSHA256, dstPath), references, ""); + txn.commit(); + } + + outputLock.setDeletion(true); + } + + return dstPath; +} + + +void deleteFromStore(const Path & _path, unsigned long long & bytesFreed) +{ + bytesFreed = 0; + Path path(canonPath(_path)); + + assertStorePath(path); + + Transaction txn(nixDB); + if (isValidPathTxn(txn, path)) { + PathSet referrers = getReferrers(txn, path); + for (PathSet::iterator i = referrers.begin(); + i != referrers.end(); ++i) + if (*i != path && isValidPathTxn(txn, *i)) + throw Error(format("cannot delete path `%1%' because it is in use by path `%2%'") % path % *i); + invalidatePath(txn, path); + } + txn.commit(); + + deletePath(path, bytesFreed); +} + + +void verifyStore(bool checkContents) +{ + Transaction txn(nixDB); + + Paths paths; + PathSet validPaths; + nixDB.enumTable(txn, dbValidPaths, paths); + + for (Paths::iterator i = paths.begin(); i != paths.end(); ++i) { + if (!pathExists(*i)) { + printMsg(lvlError, format("path `%1%' disappeared") % *i); + invalidatePath(txn, *i); + } else if (!isStorePath(*i)) { + printMsg(lvlError, format("path `%1%' is not in the Nix store") % *i); + invalidatePath(txn, *i); + } else { + if (checkContents) { + debug(format("checking contents of `%1%'") % *i); + Hash expected = queryHash(txn, *i); + Hash current = hashPath(expected.type, *i); + if (current != expected) { + printMsg(lvlError, format("path `%1%' was modified! " + "expected hash `%2%', got `%3%'") + % *i % printHash(expected) % printHash(current)); + } + } + validPaths.insert(*i); + } + } + + /* "Usable" paths are those that are valid or have a + substitute. */ + PathSet usablePaths(validPaths); + + /* Check that the values of the substitute mappings are valid + paths. */ + Paths subKeys; + nixDB.enumTable(txn, dbSubstitutes, subKeys); + for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) { + Substitutes subs = readSubstitutes(txn, *i); + if (!isStorePath(*i)) { + printMsg(lvlError, format("found substitutes for non-store path `%1%'") % *i); + nixDB.delPair(txn, dbSubstitutes, *i); + } + else if (subs.size() == 0) + nixDB.delPair(txn, dbSubstitutes, *i); + else + usablePaths.insert(*i); + } + + /* Check the cleanup invariant: only usable paths can have + `references', `referrers', or `derivers' entries. */ + + /* Check the `derivers' table. */ + Paths deriversKeys; + nixDB.enumTable(txn, dbDerivers, deriversKeys); + for (Paths::iterator i = deriversKeys.begin(); + i != deriversKeys.end(); ++i) + { + if (usablePaths.find(*i) == usablePaths.end()) { + printMsg(lvlError, format("found deriver entry for unusable path `%1%'") + % *i); + nixDB.delPair(txn, dbDerivers, *i); + } + else { + Path deriver = queryDeriver(txn, *i); + if (!isStorePath(deriver)) { + printMsg(lvlError, format("found corrupt deriver `%1%' for `%2%'") + % deriver % *i); + nixDB.delPair(txn, dbDerivers, *i); + } + } + } + + /* Check the `references' table. */ + Paths referencesKeys; + nixDB.enumTable(txn, dbReferences, referencesKeys); + for (Paths::iterator i = referencesKeys.begin(); + i != referencesKeys.end(); ++i) + { + if (usablePaths.find(*i) == usablePaths.end()) { + printMsg(lvlError, format("found references entry for unusable path `%1%'") + % *i); + setReferences(txn, *i, PathSet()); + } + else { + bool isValid = validPaths.find(*i) != validPaths.end(); + PathSet references; + queryReferences(txn, *i, references); + for (PathSet::iterator j = references.begin(); + j != references.end(); ++j) + { + string dummy; + if (!nixDB.queryString(txn, dbReferrers, addPrefix(*j, *i), dummy)) { + printMsg(lvlError, format("missing referrer mapping from `%1%' to `%2%'") + % *j % *i); + nixDB.setString(txn, dbReferrers, addPrefix(*j, *i), ""); + } + if (isValid && validPaths.find(*j) == validPaths.end()) { + printMsg(lvlError, format("incomplete closure: `%1%' needs missing `%2%'") + % *i % *j); + } + } + } + } + +#if 0 // !!! + /* Check the `referrers' table. */ + Paths referrersKeys; + nixDB.enumTable(txn, dbReferrers, referrersKeys); + for (Paths::iterator i = referrersKeys.begin(); + i != referrersKeys.end(); ++i) + { + if (usablePaths.find(*i) == usablePaths.end()) { + printMsg(lvlError, format("found referrers entry for unusable path `%1%'") + % *i); + nixDB.delPair(txn, dbReferrers, *i); + } + else { + PathSet referrers, newReferrers; + queryReferrers(txn, *i, referrers); + for (PathSet::iterator j = referrers.begin(); + j != referrers.end(); ++j) + { + Paths references; + if (usablePaths.find(*j) == usablePaths.end()) { + printMsg(lvlError, format("referrer mapping from `%1%' to unusable `%2%'") + % *i % *j); + } else { + nixDB.queryStrings(txn, dbReferences, *j, references); + if (find(references.begin(), references.end(), *i) == references.end()) { + printMsg(lvlError, format("missing reference mapping from `%1%' to `%2%'") + % *j % *i); + /* !!! repair by inserting *i into references */ + } + else newReferrers.insert(*j); + } + } + if (referrers != newReferrers) + nixDB.setStrings(txn, dbReferrers, *i, + Paths(newReferrers.begin(), newReferrers.end())); + } + } +#endif + + txn.commit(); +} + + +/* Upgrade from schema 1 (Nix <= 0.7) to schema 2 (Nix >= 0.8). */ +static void upgradeStore07() +{ + printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); + + Transaction txn(nixDB); + + Paths validPaths2; + nixDB.enumTable(txn, dbValidPaths, validPaths2); + PathSet validPaths(validPaths2.begin(), validPaths2.end()); + + std::cerr << "hashing paths..."; + int n = 0; + for (PathSet::iterator i = validPaths.begin(); i != validPaths.end(); ++i) { + checkInterrupt(); + string s; + nixDB.queryString(txn, dbValidPaths, *i, s); + if (s == "") { + Hash hash = hashPath(htSHA256, *i); + setHash(txn, *i, hash); + std::cerr << "."; + if (++n % 1000 == 0) { + txn.commit(); + txn.begin(nixDB); + } + } + } + std::cerr << std::endl; + + txn.commit(); + + txn.begin(nixDB); + + std::cerr << "processing closures..."; + for (PathSet::iterator i = validPaths.begin(); i != validPaths.end(); ++i) { + checkInterrupt(); + if (i->size() > 6 && string(*i, i->size() - 6) == ".store") { + ATerm t = ATreadFromNamedFile(i->c_str()); + if (!t) throw Error(format("cannot read aterm from `%1%'") % *i); + + ATermList roots, elems; + if (!matchOldClosure(t, roots, elems)) continue; + + for (ATermIterator j(elems); j; ++j) { + + ATerm path2; + ATermList references2; + if (!matchOldClosureElem(*j, path2, references2)) continue; + + Path path = aterm2String(path2); + if (validPaths.find(path) == validPaths.end()) + /* Skip this path; it's invalid. This is a normal + condition (Nix <= 0.7 did not enforce closure + on closure store expressions). */ + continue; + + PathSet references; + for (ATermIterator k(references2); k; ++k) { + Path reference = aterm2String(*k); + if (validPaths.find(reference) == validPaths.end()) + /* Bad reference. Set it anyway and let the + user fix it. */ + printMsg(lvlError, format("closure `%1%' contains reference from `%2%' " + "to invalid path `%3%' (run `nix-store --verify')") + % *i % path % reference); + references.insert(reference); + } + + PathSet prevReferences; + queryReferences(txn, path, prevReferences); + if (prevReferences.size() > 0 && references != prevReferences) + printMsg(lvlError, format("warning: conflicting references for `%1%'") % path); + + if (references != prevReferences) + setReferences(txn, path, references); + } + + std::cerr << "."; + } + } + std::cerr << std::endl; + + /* !!! maybe this transaction is way too big */ + txn.commit(); +} + + +/* Upgrade from schema 2 (0.8 <= Nix <= 0.9) to schema 3 (Nix >= + 0.10). The only thing to do here is to upgrade the old `referer' + table (which causes quadratic complexity in some cases) to the new + (and properly spelled) `referrer' table. */ +static void upgradeStore09() +{ + /* !!! we should disallow concurrent upgrades */ + + printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); + + if (!pathExists(nixDBPath + "/referers")) return; + + Transaction txn(nixDB); + + std::cerr << "converting referers to referrers..."; + + TableId dbReferers = nixDB.openTable("referers"); /* sic! */ + + Paths referersKeys; + nixDB.enumTable(txn, dbReferers, referersKeys); + + int n = 0; + for (Paths::iterator i = referersKeys.begin(); + i != referersKeys.end(); ++i) + { + Paths referers; + nixDB.queryStrings(txn, dbReferers, *i, referers); + for (Paths::iterator j = referers.begin(); + j != referers.end(); ++j) + nixDB.setString(txn, dbReferrers, addPrefix(*i, *j), ""); + if (++n % 1000 == 0) { + txn.commit(); + txn.begin(nixDB); + std::cerr << "|"; + } + std::cerr << "."; + } + + txn.commit(); + + std::cerr << std::endl; + + nixDB.closeTable(dbReferers); + + nixDB.deleteTable("referers"); +} + + +} diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh new file mode 100644 index 000000000000..2fd45cd7a521 --- /dev/null +++ b/src/libstore/local-store.hh @@ -0,0 +1,136 @@ +#ifndef __LOCAL_STORE_H +#define __LOCAL_STORE_H + +#include + +#include "store-api.hh" + + +namespace nix { + + +class Transaction; + + +/* Nix store and database schema version. Version 1 (or 0) was Nix <= + 0.7. Version 2 was Nix 0.8 and 0.8. Version 3 is Nix 0.10 and + up. */ +const int nixSchemaVersion = 3; + + +class LocalStore : public StoreAPI +{ +public: + + /* Open the database environment. If `reserveSpace' is true, make + sure that a big empty file exists in /nix/var/nix/db/reserved. + If `reserveSpace' is false, delete this file if it exists. The + idea is that on normal operation, the file exists; but when we + run the garbage collector, it is deleted. This is to ensure + that the garbage collector has a small amount of disk space + available, which is required to open the Berkeley DB + environment. */ + LocalStore(bool reserveSpace); + + ~LocalStore(); + + /* Implementations of abstract store API methods. */ + + bool isValidPath(const Path & path); + + Substitutes querySubstitutes(const Path & srcPath); + + Hash queryPathHash(const Path & path); + + void queryReferences(const Path & storePath, + PathSet & references); + + void queryReferrers(const Path & storePath, + PathSet & referrers); + + Path addToStore(const Path & srcPath); + + Path addToStoreFixed(bool recursive, string hashAlgo, + const Path & srcPath); + + Path addTextToStore(const string & suffix, const string & s, + const PathSet & references); + +private: + Path _addToStore(bool fixed, bool recursive, + string hashAlgo, const Path & _srcPath); +}; + + +/* Get a transaction object. */ +void createStoreTransaction(Transaction & txn); + +/* Copy a path recursively. */ +void copyPath(const Path & src, const Path & dst); + +/* Register a substitute. */ +void registerSubstitute(const Transaction & txn, + const Path & srcPath, const Substitute & sub); + +/* Deregister all substitutes. */ +void clearSubstitutes(); + +/* Register the validity of a path, i.e., that `path' exists, that the + paths referenced by it exists, and in the case of an output path of + a derivation, that it has been produced by a succesful execution of + the derivation (or something equivalent). Also register the hash + of the file system contents of the path. The hash must be a + SHA-256 hash. */ +void registerValidPath(const Transaction & txn, + const Path & path, const Hash & hash, const PathSet & references, + const Path & deriver); + +struct ValidPathInfo +{ + Path path; + Path deriver; + Hash hash; + PathSet references; +}; + +typedef list ValidPathInfos; + +void registerValidPaths(const Transaction & txn, + const ValidPathInfos & infos); + +/* "Fix", or canonicalise, the meta-data of the files in a store path + after it has been built. In particular: + - the last modification date on each file is set to 0 (i.e., + 00:00:00 1/1/1970 UTC) + - the permissions are set of 444 or 555 (i.e., read-only with or + without execute permission; setuid bits etc. are cleared) + - the owner and group are set to the Nix user and group, if we're + in a setuid Nix installation. */ +void canonicalisePathMetaData(const Path & path); + +/* Checks whether a path is valid. */ +bool isValidPathTxn(const Transaction & txn, const Path & path); + +/* Sets the set of outgoing FS references for a store path. Use with + care! */ +void setReferences(const Transaction & txn, const Path & storePath, + const PathSet & references); + +/* Sets the deriver of a store path. Use with care! */ +void setDeriver(const Transaction & txn, const Path & storePath, + const Path & deriver); + +/* Query the deriver of a store path. Return the empty string if no + deriver has been set. */ +Path queryDeriver(const Transaction & txn, const Path & storePath); + +/* Delete a value from the nixStore directory. */ +void deleteFromStore(const Path & path, unsigned long long & bytesFreed); + +void verifyStore(bool checkContents); + + +} + + +#endif /* !__LOCAL_STORE_H */ diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index bcede901c5ae..b442bd4c2157 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,5 +1,5 @@ #include "misc.hh" -#include "store.hh" +#include "store-api.hh" #include "build.hh" #include "db.hh" @@ -27,9 +27,9 @@ void computeFSClosure(const Path & storePath, PathSet references; if (flipDirection) - queryReferrers(noTxn, storePath, references); + store->queryReferrers(storePath, references); else - queryReferences(noTxn, storePath, references); + store->queryReferences(storePath, references); for (PathSet::iterator i = references.begin(); i != references.end(); ++i) @@ -58,14 +58,14 @@ void queryMissing(const PathSet & targets, done.insert(p); if (isDerivation(p)) { - if (!isValidPath(p)) continue; + if (!store->isValidPath(p)) continue; Derivation drv = derivationFromPath(p); bool mustBuild = false; for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i) - if (!isValidPath(i->second.path) && - querySubstitutes(noTxn, i->second.path).size() == 0) + if (!store->isValidPath(i->second.path) && + store->querySubstitutes(i->second.path).size() == 0) mustBuild = true; if (mustBuild) { @@ -81,11 +81,11 @@ void queryMissing(const PathSet & targets, } else { - if (isValidPath(p)) continue; - if (querySubstitutes(noTxn, p).size() > 0) + if (store->isValidPath(p)) continue; + if (store->querySubstitutes(p).size() > 0) willSubstitute.insert(p); PathSet refs; - queryReferences(noTxn, p, todo); + store->queryReferences(p, todo); } } } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc new file mode 100644 index 000000000000..3a07d1c73ca2 --- /dev/null +++ b/src/libstore/store-api.cc @@ -0,0 +1,107 @@ +#include "store-api.hh" +#include "globals.hh" + + +namespace nix { + + +bool isInStore(const Path & path) +{ + return path[0] == '/' + && string(path, 0, nixStore.size()) == nixStore + && path.size() >= nixStore.size() + 2 + && path[nixStore.size()] == '/'; +} + + +bool isStorePath(const Path & path) +{ + return isInStore(path) + && path.find('/', nixStore.size() + 1) == Path::npos; +} + + +void assertStorePath(const Path & path) +{ + if (!isStorePath(path)) + throw Error(format("path `%1%' is not in the Nix store") % path); +} + + +Path toStorePath(const Path & path) +{ + if (!isInStore(path)) + throw Error(format("path `%1%' is not in the Nix store") % path); + Path::size_type slash = path.find('/', nixStore.size() + 1); + if (slash == Path::npos) + return path; + else + return Path(path, 0, slash); +} + + +void checkStoreName(const string & name) +{ + string validChars = "+-._?="; + /* Disallow names starting with a dot for possible security + reasons (e.g., "." and ".."). */ + if (string(name, 0, 1) == ".") + throw Error(format("illegal name: `%1%'") % name); + for (string::const_iterator i = name.begin(); i != name.end(); ++i) + if (!((*i >= 'A' && *i <= 'Z') || + (*i >= 'a' && *i <= 'z') || + (*i >= '0' && *i <= '9') || + validChars.find(*i) != string::npos)) + { + throw Error(format("invalid character `%1%' in name `%2%'") + % *i % name); + } +} + + +Path makeStorePath(const string & type, + const Hash & hash, const string & suffix) +{ + /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ + string s = type + ":sha256:" + printHash(hash) + ":" + + nixStore + ":" + suffix; + + checkStoreName(suffix); + + return nixStore + "/" + + printHash32(compressHash(hashString(htSHA256, s), 20)) + + "-" + suffix; +} + + +Path makeFixedOutputPath(bool recursive, + string hashAlgo, Hash hash, string name) +{ + /* !!! copy/paste from primops.cc */ + Hash h = hashString(htSHA256, "fixed:out:" + + (recursive ? (string) "r:" : "") + hashAlgo + ":" + + printHash(hash) + ":" + + ""); + return makeStorePath("output:out", h, name); +} + + +} + + +#include "local-store.hh" + + +namespace nix { + + +boost::shared_ptr store; + + +boost::shared_ptr openStore(bool reserveSpace) +{ + return boost::shared_ptr(new LocalStore(reserveSpace)); +} + + +} diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh new file mode 100644 index 000000000000..91beba50f7d2 --- /dev/null +++ b/src/libstore/store-api.hh @@ -0,0 +1,115 @@ +#ifndef __STOREAPI_H +#define __STOREAPI_H + +#include + +#include + +#include "hash.hh" + + +namespace nix { + + +/* A substitute is a program invocation that constructs some store + path (typically by fetching it from somewhere, e.g., from the + network). */ +struct Substitute +{ + /* The derivation that built this store path (empty if none). */ + Path deriver; + + /* Program to be executed to create the store path. Must be in + the output path of `storeExpr'. */ + Path program; + + /* Extra arguments to be passed to the program (the first argument + is the store path to be substituted). */ + Strings args; + + bool operator == (const Substitute & sub) const; +}; + +typedef list Substitutes; + + +class StoreAPI +{ +public: + + virtual ~StoreAPI() { } + + /* Checks whether a path is valid. */ + virtual bool isValidPath(const Path & path) = 0; + + /* Return the substitutes for the given path. */ + virtual Substitutes querySubstitutes(const Path & srcPath) = 0; + + /* Queries the hash of a valid path. */ + virtual Hash queryPathHash(const Path & path) = 0; + + /* Queries the set of outgoing FS references for a store path. + The result is not cleared. */ + virtual void queryReferences(const Path & storePath, + PathSet & references) = 0; + + /* Queries the set of incoming FS references for a store path. + The result is not cleared. */ + virtual void queryReferrers(const Path & storePath, + PathSet & referrers) = 0; + + /* Copy the contents of a path to the store and register the + validity the resulting path. The resulting path is + returned. */ + virtual Path addToStore(const Path & srcPath) = 0; + + /* Like addToStore(), but for pre-adding the outputs of + fixed-output derivations. */ + virtual Path addToStoreFixed(bool recursive, string hashAlgo, + const Path & srcPath) = 0; + + /* Like addToStore, but the contents written to the output path is + a regular file containing the given string. */ + virtual Path addTextToStore(const string & suffix, const string & s, + const PathSet & references) = 0; +}; + + +/* !!! These should be part of the store API, I guess. */ + +/* Throw an exception if `path' is not directly in the Nix store. */ +void assertStorePath(const Path & path); + +bool isInStore(const Path & path); +bool isStorePath(const Path & path); + +void checkStoreName(const string & name); + +/* Chop off the parts after the top-level store name, e.g., + /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */ +Path toStorePath(const Path & path); + + +/* Constructs a unique store path name. */ +Path makeStorePath(const string & type, + const Hash & hash, const string & suffix); + +Path makeFixedOutputPath(bool recursive, + string hashAlgo, Hash hash, string name); + + +/* For now, there is a single global store API object, but we'll + purify that in the future. */ +extern boost::shared_ptr store; + + +/* Factory method: open the Nix database, either through the local or + remote implementation. */ +boost::shared_ptr openStore(bool reserveSpace = true); + + + +} + + +#endif /* !__STOREAPI_H */ diff --git a/src/libstore/store.cc b/src/libstore/store.cc deleted file mode 100644 index e073d64adaff..000000000000 --- a/src/libstore/store.cc +++ /dev/null @@ -1,1103 +0,0 @@ -#include "store.hh" -#include "util.hh" -#include "globals.hh" -#include "db.hh" -#include "archive.hh" -#include "pathlocks.hh" -#include "gc.hh" -#include "aterm.hh" -#include "derivations-ast.hh" - -#include -#include - -#include -#include -#include -#include - -namespace nix { - - -/* Nix database. */ -static Database nixDB; - - -/* Database tables. */ - -/* dbValidPaths :: Path -> () - - The existence of a key $p$ indicates that path $p$ is valid (that - is, produced by a succesful build). */ -static TableId dbValidPaths = 0; - -/* dbReferences :: Path -> [Path] - - This table lists the outgoing file system references for each - output path that has been built by a Nix derivation. These are - found by scanning the path for the hash components of input - paths. */ -static TableId dbReferences = 0; - -/* dbReferrers :: Path -> Path - - This table is just the reverse mapping of dbReferences. This table - can have duplicate keys, each corresponding value denoting a single - referrer. */ -static TableId dbReferrers = 0; - -/* dbSubstitutes :: Path -> [[Path]] - - Each pair $(p, subs)$ tells Nix that it can use any of the - substitutes in $subs$ to build path $p$. Each substitute defines a - command-line invocation of a program (i.e., the first list element - is the full path to the program, the remaining elements are - arguments). - - The main purpose of this is for distributed caching of derivates. - One system can compute a derivate and put it on a website (as a Nix - archive), for instance, and then another system can register a - substitute for that derivate. The substitute in this case might be - a Nix derivation that fetches the Nix archive. -*/ -static TableId dbSubstitutes = 0; - -/* dbDerivers :: Path -> [Path] - - This table lists the derivation used to build a path. There can - only be multiple such paths for fixed-output derivations (i.e., - derivations specifying an expected hash). */ -static TableId dbDerivers = 0; - - -bool Substitute::operator == (const Substitute & sub) const -{ - return program == sub.program - && args == sub.args; -} - - -static void upgradeStore07(); -static void upgradeStore09(); - - -void checkStoreNotSymlink() -{ - if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; - Path path = nixStore; - struct stat st; - while (path != "/") { - if (lstat(path.c_str(), &st)) - throw SysError(format("getting status of `%1%'") % path); - if (S_ISLNK(st.st_mode)) - throw Error(format( - "the path `%1%' is a symlink; " - "this is not allowed for the Nix store and its parent directories") - % path); - path = dirOf(path); - } -} - - -void openDB(bool reserveSpace) -{ - if (readOnlyMode) return; - - checkStoreNotSymlink(); - - try { - Path reservedPath = nixDBPath + "/reserved"; - string s = querySetting("gc-reserved-space", ""); - int reservedSize; - if (!string2Int(s, reservedSize)) reservedSize = 1024 * 1024; - if (reserveSpace) { - struct stat st; - if (stat(reservedPath.c_str(), &st) == -1 || - st.st_size != reservedSize) - writeFile(reservedPath, string(reservedSize, 'X')); - } - else - deletePath(reservedPath); - } catch (SysError & e) { /* don't care about errors */ - } - - try { - nixDB.open(nixDBPath); - } catch (DbNoPermission & e) { - printMsg(lvlTalkative, "cannot access Nix database; continuing anyway"); - readOnlyMode = true; - return; - } - dbValidPaths = nixDB.openTable("validpaths"); - dbReferences = nixDB.openTable("references"); - dbReferrers = nixDB.openTable("referrers", true); /* must be sorted */ - dbSubstitutes = nixDB.openTable("substitutes"); - dbDerivers = nixDB.openTable("derivers"); - - int curSchema = 0; - Path schemaFN = nixDBPath + "/schema"; - if (pathExists(schemaFN)) { - string s = readFile(schemaFN); - if (!string2Int(s, curSchema)) - throw Error(format("`%1%' is corrupt") % schemaFN); - } - - if (curSchema > nixSchemaVersion) - throw Error(format("current Nix store schema is version %1%, but I only support %2%") - % curSchema % nixSchemaVersion); - - if (curSchema < nixSchemaVersion) { - if (curSchema <= 1) - upgradeStore07(); - if (curSchema == 2) - upgradeStore09(); - writeFile(schemaFN, (format("%1%") % nixSchemaVersion).str()); - } -} - - -void initDB() -{ -} - - -void closeDB() -{ - /* If the database isn't open, this is a NOP. */ - nixDB.close(); -} - - -void createStoreTransaction(Transaction & txn) -{ - Transaction txn2(nixDB); - txn2.moveTo(txn); -} - - -/* Path copying. */ - -struct CopySink : DumpSink -{ - string s; - virtual void operator () (const unsigned char * data, unsigned int len) - { - s.append((const char *) data, len); - } -}; - - -struct CopySource : RestoreSource -{ - string & s; - unsigned int pos; - CopySource(string & _s) : s(_s), pos(0) { } - virtual void operator () (unsigned char * data, unsigned int len) - { - s.copy((char *) data, len, pos); - pos += len; - assert(pos <= s.size()); - } -}; - - -void copyPath(const Path & src, const Path & dst) -{ - debug(format("copying `%1%' to `%2%'") % src % dst); - - /* Dump an archive of the path `src' into a string buffer, then - restore the archive to `dst'. This is not a very good method - for very large paths, but `copyPath' is mainly used for small - files. */ - - CopySink sink; - { - SwitchToOriginalUser sw; - dumpPath(src, sink); - } - - CopySource source(sink.s); - restorePath(dst, source); -} - - -bool isInStore(const Path & path) -{ - return path[0] == '/' - && string(path, 0, nixStore.size()) == nixStore - && path.size() >= nixStore.size() + 2 - && path[nixStore.size()] == '/'; -} - - -bool isStorePath(const Path & path) -{ - return isInStore(path) - && path.find('/', nixStore.size() + 1) == Path::npos; -} - - -void assertStorePath(const Path & path) -{ - if (!isStorePath(path)) - throw Error(format("path `%1%' is not in the Nix store") % path); -} - - -Path toStorePath(const Path & path) -{ - if (!isInStore(path)) - throw Error(format("path `%1%' is not in the Nix store") % path); - Path::size_type slash = path.find('/', nixStore.size() + 1); - if (slash == Path::npos) - return path; - else - return Path(path, 0, slash); -} - - -void checkStoreName(const string & name) -{ - string validChars = "+-._?="; - /* Disallow names starting with a dot for possible security - reasons (e.g., "." and ".."). */ - if (string(name, 0, 1) == ".") - throw Error(format("illegal name: `%1%'") % name); - for (string::const_iterator i = name.begin(); i != name.end(); ++i) - if (!((*i >= 'A' && *i <= 'Z') || - (*i >= 'a' && *i <= 'z') || - (*i >= '0' && *i <= '9') || - validChars.find(*i) != string::npos)) - { - throw Error(format("invalid character `%1%' in name `%2%'") - % *i % name); - } -} - - -void canonicalisePathMetaData(const Path & path) -{ - checkInterrupt(); - - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); - - if (!S_ISLNK(st.st_mode)) { - - /* Mask out all type related bits. */ - mode_t mode = st.st_mode & ~S_IFMT; - - if (mode != 0444 && mode != 0555) { - mode = (st.st_mode & S_IFMT) - | 0444 - | (st.st_mode & S_IXUSR ? 0111 : 0); - if (chmod(path.c_str(), mode) == -1) - throw SysError(format("changing mode of `%1%' to %2$o") % path % mode); - } - - if (st.st_uid != getuid() || st.st_gid != getgid()) { - if (chown(path.c_str(), getuid(), getgid()) == -1) - throw SysError(format("changing owner/group of `%1%' to %2%/%3%") - % path % getuid() % getgid()); - } - - if (st.st_mtime != 0) { - struct utimbuf utimbuf; - utimbuf.actime = st.st_atime; - utimbuf.modtime = 0; - if (utime(path.c_str(), &utimbuf) == -1) - throw SysError(format("changing modification time of `%1%'") % path); - } - - } - - if (S_ISDIR(st.st_mode)) { - Strings names = readDirectory(path); - for (Strings::iterator i = names.begin(); i != names.end(); ++i) - canonicalisePathMetaData(path + "/" + *i); - } -} - - -bool isValidPathTxn(const Transaction & txn, const Path & path) -{ - string s; - return nixDB.queryString(txn, dbValidPaths, path, s); -} - - -bool isValidPath(const Path & path) -{ - return isValidPathTxn(noTxn, path); -} - - -static Substitutes readSubstitutes(const Transaction & txn, - const Path & srcPath); - - -static bool isRealisablePath(const Transaction & txn, const Path & path) -{ - return isValidPathTxn(txn, path) - || readSubstitutes(txn, path).size() > 0; -} - - -static string addPrefix(const string & prefix, const string & s) -{ - return prefix + string(1, (char) 0) + s; -} - - -static string stripPrefix(const string & prefix, const string & s) -{ - if (s.size() <= prefix.size() || - string(s, 0, prefix.size()) != prefix || - s[prefix.size()] != 0) - throw Error(format("string `%1%' is missing prefix `%2%'") - % s % prefix); - return string(s, prefix.size() + 1); -} - - -static PathSet getReferrers(const Transaction & txn, const Path & storePath) -{ - PathSet referrers; - Strings keys; - nixDB.enumTable(txn, dbReferrers, keys, storePath + string(1, (char) 0)); - for (Strings::iterator i = keys.begin(); i != keys.end(); ++i) - referrers.insert(stripPrefix(storePath, *i)); - return referrers; -} - - -void setReferences(const Transaction & txn, const Path & storePath, - const PathSet & references) -{ - /* For unrealisable paths, we can only clear the references. */ - if (references.size() > 0 && !isRealisablePath(txn, storePath)) - throw Error( - format("cannot set references for path `%1%' which is invalid and has no substitutes") - % storePath); - - Paths oldReferences; - nixDB.queryStrings(txn, dbReferences, storePath, oldReferences); - - PathSet oldReferences2(oldReferences.begin(), oldReferences.end()); - if (oldReferences2 == references) return; - - nixDB.setStrings(txn, dbReferences, storePath, - Paths(references.begin(), references.end())); - - /* Update the referrers mappings of all new referenced paths. */ - for (PathSet::const_iterator i = references.begin(); - i != references.end(); ++i) - if (oldReferences2.find(*i) == oldReferences2.end()) - nixDB.setString(txn, dbReferrers, addPrefix(*i, storePath), ""); - - /* Remove referrer mappings from paths that are no longer - references. */ - for (Paths::iterator i = oldReferences.begin(); - i != oldReferences.end(); ++i) - if (references.find(*i) == references.end()) - nixDB.delPair(txn, dbReferrers, addPrefix(*i, storePath)); -} - - -void queryReferences(const Transaction & txn, - const Path & storePath, PathSet & references) -{ - Paths references2; - if (!isRealisablePath(txn, storePath)) - throw Error(format("path `%1%' is not valid") % storePath); - nixDB.queryStrings(txn, dbReferences, storePath, references2); - references.insert(references2.begin(), references2.end()); -} - - -void queryReferrers(const Transaction & txn, - const Path & storePath, PathSet & referrers) -{ - if (!isRealisablePath(txn, storePath)) - throw Error(format("path `%1%' is not valid") % storePath); - PathSet referrers2 = getReferrers(txn, storePath); - referrers.insert(referrers2.begin(), referrers2.end()); -} - - -void setDeriver(const Transaction & txn, const Path & storePath, - const Path & deriver) -{ - assertStorePath(storePath); - if (deriver == "") return; - assertStorePath(deriver); - if (!isRealisablePath(txn, storePath)) - throw Error(format("path `%1%' is not valid") % storePath); - nixDB.setString(txn, dbDerivers, storePath, deriver); -} - - -Path queryDeriver(const Transaction & txn, const Path & storePath) -{ - if (!isRealisablePath(txn, storePath)) - throw Error(format("path `%1%' is not valid") % storePath); - Path deriver; - if (nixDB.queryString(txn, dbDerivers, storePath, deriver)) - return deriver; - else - return ""; -} - - -const int substituteVersion = 2; - - -static Substitutes readSubstitutes(const Transaction & txn, - const Path & srcPath) -{ - Strings ss; - nixDB.queryStrings(txn, dbSubstitutes, srcPath, ss); - - Substitutes subs; - - for (Strings::iterator i = ss.begin(); i != ss.end(); ++i) { - if (i->size() < 4 || (*i)[3] != 0) { - /* Old-style substitute. !!! remove this code - eventually? */ - break; - } - Strings ss2 = unpackStrings(*i); - if (ss2.size() == 0) continue; - int version; - if (!string2Int(ss2.front(), version)) continue; - if (version != substituteVersion) continue; - if (ss2.size() != 4) throw Error("malformed substitute"); - Strings::iterator j = ss2.begin(); - j++; - Substitute sub; - sub.deriver = *j++; - sub.program = *j++; - sub.args = unpackStrings(*j++); - subs.push_back(sub); - } - - return subs; -} - - -static void writeSubstitutes(const Transaction & txn, - const Path & srcPath, const Substitutes & subs) -{ - Strings ss; - - for (Substitutes::const_iterator i = subs.begin(); - i != subs.end(); ++i) - { - Strings ss2; - ss2.push_back((format("%1%") % substituteVersion).str()); - ss2.push_back(i->deriver); - ss2.push_back(i->program); - ss2.push_back(packStrings(i->args)); - ss.push_back(packStrings(ss2)); - } - - nixDB.setStrings(txn, dbSubstitutes, srcPath, ss); -} - - -void registerSubstitute(const Transaction & txn, - const Path & srcPath, const Substitute & sub) -{ - assertStorePath(srcPath); - - Substitutes subs = readSubstitutes(txn, srcPath); - - if (find(subs.begin(), subs.end(), sub) != subs.end()) - return; - - /* New substitutes take precedence over old ones. If the - substitute is already present, it's moved to the front. */ - remove(subs.begin(), subs.end(), sub); - subs.push_front(sub); - - writeSubstitutes(txn, srcPath, subs); -} - - -Substitutes querySubstitutes(const Transaction & txn, const Path & srcPath) -{ - return readSubstitutes(txn, srcPath); -} - - -static void invalidatePath(Transaction & txn, const Path & path); - - -void clearSubstitutes() -{ - Transaction txn(nixDB); - - /* Iterate over all paths for which there are substitutes. */ - Paths subKeys; - nixDB.enumTable(txn, dbSubstitutes, subKeys); - for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) { - - /* Delete all substitutes for path *i. */ - nixDB.delPair(txn, dbSubstitutes, *i); - - /* Maintain the cleanup invariant. */ - if (!isValidPathTxn(txn, *i)) - invalidatePath(txn, *i); - } - - /* !!! there should be no referrers to any of the invalid - substitutable paths. This should be the case by construction - (the only referrers can be other invalid substitutable paths, - which have all been removed now). */ - - txn.commit(); -} - - -static void setHash(const Transaction & txn, const Path & storePath, - const Hash & hash) -{ - assert(hash.type == htSHA256); - nixDB.setString(txn, dbValidPaths, storePath, "sha256:" + printHash(hash)); -} - - -static Hash queryHash(const Transaction & txn, const Path & storePath) -{ - string s; - nixDB.queryString(txn, dbValidPaths, storePath, s); - string::size_type colon = s.find(':'); - if (colon == string::npos) - throw Error(format("corrupt hash `%1%' in valid-path entry for `%2%'") - % s % storePath); - HashType ht = parseHashType(string(s, 0, colon)); - if (ht == htUnknown) - throw Error(format("unknown hash type `%1%' in valid-path entry for `%2%'") - % string(s, 0, colon) % storePath); - return parseHash(ht, string(s, colon + 1)); -} - - -Hash queryPathHash(const Path & path) -{ - if (!isValidPath(path)) - throw Error(format("path `%1%' is not valid") % path); - return queryHash(noTxn, path); -} - - -void registerValidPath(const Transaction & txn, - const Path & path, const Hash & hash, const PathSet & references, - const Path & deriver) -{ - ValidPathInfo info; - info.path = path; - info.hash = hash; - info.references = references; - info.deriver = deriver; - ValidPathInfos infos; - infos.push_back(info); - registerValidPaths(txn, infos); -} - - -void registerValidPaths(const Transaction & txn, - const ValidPathInfos & infos) -{ - PathSet newPaths; - for (ValidPathInfos::const_iterator i = infos.begin(); - i != infos.end(); ++i) - newPaths.insert(i->path); - - for (ValidPathInfos::const_iterator i = infos.begin(); - i != infos.end(); ++i) - { - assertStorePath(i->path); - - debug(format("registering path `%1%'") % i->path); - setHash(txn, i->path, i->hash); - - setReferences(txn, i->path, i->references); - - /* Check that all referenced paths are also valid (or about to - become valid). */ - for (PathSet::iterator j = i->references.begin(); - j != i->references.end(); ++j) - if (!isValidPathTxn(txn, *j) && newPaths.find(*j) == newPaths.end()) - throw Error(format("cannot register path `%1%' as valid, since its reference `%2%' is invalid") - % i->path % *j); - - setDeriver(txn, i->path, i->deriver); - } -} - - -/* Invalidate a path. The caller is responsible for checking that - there are no referrers. */ -static void invalidatePath(Transaction & txn, const Path & path) -{ - debug(format("unregistering path `%1%'") % path); - - /* Clear the `references' entry for this path, as well as the - inverse `referrers' entries, and the `derivers' entry; but only - if there are no substitutes for this path. This maintains the - cleanup invariant. */ - if (querySubstitutes(txn, path).size() == 0) { - setReferences(txn, path, PathSet()); - nixDB.delPair(txn, dbDerivers, path); - } - - nixDB.delPair(txn, dbValidPaths, path); -} - - -Path makeStorePath(const string & type, - const Hash & hash, const string & suffix) -{ - /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ - string s = type + ":sha256:" + printHash(hash) + ":" - + nixStore + ":" + suffix; - - checkStoreName(suffix); - - return nixStore + "/" - + printHash32(compressHash(hashString(htSHA256, s), 20)) - + "-" + suffix; -} - - -Path makeFixedOutputPath(bool recursive, - string hashAlgo, Hash hash, string name) -{ - /* !!! copy/paste from primops.cc */ - Hash h = hashString(htSHA256, "fixed:out:" - + (recursive ? (string) "r:" : "") + hashAlgo + ":" - + printHash(hash) + ":" - + ""); - return makeStorePath("output:out", h, name); -} - - -static Path _addToStore(bool fixed, bool recursive, - string hashAlgo, const Path & _srcPath) -{ - Path srcPath(absPath(_srcPath)); - debug(format("adding `%1%' to the store") % srcPath); - - Hash h(htSHA256); - { - SwitchToOriginalUser sw; - h = hashPath(htSHA256, srcPath); - } - - string baseName = baseNameOf(srcPath); - - Path dstPath; - - if (fixed) { - - HashType ht(parseHashType(hashAlgo)); - Hash h2(ht); - { - SwitchToOriginalUser sw; - h2 = recursive ? hashPath(ht, srcPath) : hashFile(ht, srcPath); - } - - dstPath = makeFixedOutputPath(recursive, hashAlgo, h2, baseName); - } - - else dstPath = makeStorePath("source", h, baseName); - - if (!readOnlyMode) addTempRoot(dstPath); - - if (!readOnlyMode && !isValidPath(dstPath)) { - - /* The first check above is an optimisation to prevent - unnecessary lock acquisition. */ - - PathLocks outputLock(singleton(dstPath)); - - if (!isValidPath(dstPath)) { - - if (pathExists(dstPath)) deletePath(dstPath); - - copyPath(srcPath, dstPath); - - Hash h2 = hashPath(htSHA256, dstPath); - if (h != h2) - throw Error(format("contents of `%1%' changed while copying it to `%2%' (%3% -> %4%)") - % srcPath % dstPath % printHash(h) % printHash(h2)); - - canonicalisePathMetaData(dstPath); - - Transaction txn(nixDB); - registerValidPath(txn, dstPath, h, PathSet(), ""); - txn.commit(); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - - -Path addToStore(const Path & srcPath) -{ - return _addToStore(false, false, "", srcPath); -} - - -Path addToStoreFixed(bool recursive, string hashAlgo, const Path & srcPath) -{ - return _addToStore(true, recursive, hashAlgo, srcPath); -} - - -Path addTextToStore(const string & suffix, const string & s, - const PathSet & references) -{ - Hash hash = hashString(htSHA256, s); - - Path dstPath = makeStorePath("text", hash, suffix); - - if (!readOnlyMode) addTempRoot(dstPath); - - if (!readOnlyMode && !isValidPath(dstPath)) { - - PathLocks outputLock(singleton(dstPath)); - - if (!isValidPath(dstPath)) { - - if (pathExists(dstPath)) deletePath(dstPath); - - writeStringToFile(dstPath, s); - - canonicalisePathMetaData(dstPath); - - Transaction txn(nixDB); - registerValidPath(txn, dstPath, - hashPath(htSHA256, dstPath), references, ""); - txn.commit(); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - - -void deleteFromStore(const Path & _path, unsigned long long & bytesFreed) -{ - bytesFreed = 0; - Path path(canonPath(_path)); - - assertStorePath(path); - - Transaction txn(nixDB); - if (isValidPathTxn(txn, path)) { - PathSet referrers = getReferrers(txn, path); - for (PathSet::iterator i = referrers.begin(); - i != referrers.end(); ++i) - if (*i != path && isValidPathTxn(txn, *i)) - throw Error(format("cannot delete path `%1%' because it is in use by path `%2%'") % path % *i); - invalidatePath(txn, path); - } - txn.commit(); - - deletePath(path, bytesFreed); -} - - -void verifyStore(bool checkContents) -{ - Transaction txn(nixDB); - - Paths paths; - PathSet validPaths; - nixDB.enumTable(txn, dbValidPaths, paths); - - for (Paths::iterator i = paths.begin(); i != paths.end(); ++i) { - if (!pathExists(*i)) { - printMsg(lvlError, format("path `%1%' disappeared") % *i); - invalidatePath(txn, *i); - } else if (!isStorePath(*i)) { - printMsg(lvlError, format("path `%1%' is not in the Nix store") % *i); - invalidatePath(txn, *i); - } else { - if (checkContents) { - debug(format("checking contents of `%1%'") % *i); - Hash expected = queryHash(txn, *i); - Hash current = hashPath(expected.type, *i); - if (current != expected) { - printMsg(lvlError, format("path `%1%' was modified! " - "expected hash `%2%', got `%3%'") - % *i % printHash(expected) % printHash(current)); - } - } - validPaths.insert(*i); - } - } - - /* "Usable" paths are those that are valid or have a - substitute. */ - PathSet usablePaths(validPaths); - - /* Check that the values of the substitute mappings are valid - paths. */ - Paths subKeys; - nixDB.enumTable(txn, dbSubstitutes, subKeys); - for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) { - Substitutes subs = readSubstitutes(txn, *i); - if (!isStorePath(*i)) { - printMsg(lvlError, format("found substitutes for non-store path `%1%'") % *i); - nixDB.delPair(txn, dbSubstitutes, *i); - } - else if (subs.size() == 0) - nixDB.delPair(txn, dbSubstitutes, *i); - else - usablePaths.insert(*i); - } - - /* Check the cleanup invariant: only usable paths can have - `references', `referrers', or `derivers' entries. */ - - /* Check the `derivers' table. */ - Paths deriversKeys; - nixDB.enumTable(txn, dbDerivers, deriversKeys); - for (Paths::iterator i = deriversKeys.begin(); - i != deriversKeys.end(); ++i) - { - if (usablePaths.find(*i) == usablePaths.end()) { - printMsg(lvlError, format("found deriver entry for unusable path `%1%'") - % *i); - nixDB.delPair(txn, dbDerivers, *i); - } - else { - Path deriver = queryDeriver(txn, *i); - if (!isStorePath(deriver)) { - printMsg(lvlError, format("found corrupt deriver `%1%' for `%2%'") - % deriver % *i); - nixDB.delPair(txn, dbDerivers, *i); - } - } - } - - /* Check the `references' table. */ - Paths referencesKeys; - nixDB.enumTable(txn, dbReferences, referencesKeys); - for (Paths::iterator i = referencesKeys.begin(); - i != referencesKeys.end(); ++i) - { - if (usablePaths.find(*i) == usablePaths.end()) { - printMsg(lvlError, format("found references entry for unusable path `%1%'") - % *i); - setReferences(txn, *i, PathSet()); - } - else { - bool isValid = validPaths.find(*i) != validPaths.end(); - PathSet references; - queryReferences(txn, *i, references); - for (PathSet::iterator j = references.begin(); - j != references.end(); ++j) - { - string dummy; - if (!nixDB.queryString(txn, dbReferrers, addPrefix(*j, *i), dummy)) { - printMsg(lvlError, format("missing referrer mapping from `%1%' to `%2%'") - % *j % *i); - nixDB.setString(txn, dbReferrers, addPrefix(*j, *i), ""); - } - if (isValid && validPaths.find(*j) == validPaths.end()) { - printMsg(lvlError, format("incomplete closure: `%1%' needs missing `%2%'") - % *i % *j); - } - } - } - } - -#if 0 // !!! - /* Check the `referrers' table. */ - Paths referrersKeys; - nixDB.enumTable(txn, dbReferrers, referrersKeys); - for (Paths::iterator i = referrersKeys.begin(); - i != referrersKeys.end(); ++i) - { - if (usablePaths.find(*i) == usablePaths.end()) { - printMsg(lvlError, format("found referrers entry for unusable path `%1%'") - % *i); - nixDB.delPair(txn, dbReferrers, *i); - } - else { - PathSet referrers, newReferrers; - queryReferrers(txn, *i, referrers); - for (PathSet::iterator j = referrers.begin(); - j != referrers.end(); ++j) - { - Paths references; - if (usablePaths.find(*j) == usablePaths.end()) { - printMsg(lvlError, format("referrer mapping from `%1%' to unusable `%2%'") - % *i % *j); - } else { - nixDB.queryStrings(txn, dbReferences, *j, references); - if (find(references.begin(), references.end(), *i) == references.end()) { - printMsg(lvlError, format("missing reference mapping from `%1%' to `%2%'") - % *j % *i); - /* !!! repair by inserting *i into references */ - } - else newReferrers.insert(*j); - } - } - if (referrers != newReferrers) - nixDB.setStrings(txn, dbReferrers, *i, - Paths(newReferrers.begin(), newReferrers.end())); - } - } -#endif - - txn.commit(); -} - - -/* Upgrade from schema 1 (Nix <= 0.7) to schema 2 (Nix >= 0.8). */ -static void upgradeStore07() -{ - printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); - - Transaction txn(nixDB); - - Paths validPaths2; - nixDB.enumTable(txn, dbValidPaths, validPaths2); - PathSet validPaths(validPaths2.begin(), validPaths2.end()); - - std::cerr << "hashing paths..."; - int n = 0; - for (PathSet::iterator i = validPaths.begin(); i != validPaths.end(); ++i) { - checkInterrupt(); - string s; - nixDB.queryString(txn, dbValidPaths, *i, s); - if (s == "") { - Hash hash = hashPath(htSHA256, *i); - setHash(txn, *i, hash); - std::cerr << "."; - if (++n % 1000 == 0) { - txn.commit(); - txn.begin(nixDB); - } - } - } - std::cerr << std::endl; - - txn.commit(); - - txn.begin(nixDB); - - std::cerr << "processing closures..."; - for (PathSet::iterator i = validPaths.begin(); i != validPaths.end(); ++i) { - checkInterrupt(); - if (i->size() > 6 && string(*i, i->size() - 6) == ".store") { - ATerm t = ATreadFromNamedFile(i->c_str()); - if (!t) throw Error(format("cannot read aterm from `%1%'") % *i); - - ATermList roots, elems; - if (!matchOldClosure(t, roots, elems)) continue; - - for (ATermIterator j(elems); j; ++j) { - - ATerm path2; - ATermList references2; - if (!matchOldClosureElem(*j, path2, references2)) continue; - - Path path = aterm2String(path2); - if (validPaths.find(path) == validPaths.end()) - /* Skip this path; it's invalid. This is a normal - condition (Nix <= 0.7 did not enforce closure - on closure store expressions). */ - continue; - - PathSet references; - for (ATermIterator k(references2); k; ++k) { - Path reference = aterm2String(*k); - if (validPaths.find(reference) == validPaths.end()) - /* Bad reference. Set it anyway and let the - user fix it. */ - printMsg(lvlError, format("closure `%1%' contains reference from `%2%' " - "to invalid path `%3%' (run `nix-store --verify')") - % *i % path % reference); - references.insert(reference); - } - - PathSet prevReferences; - queryReferences(txn, path, prevReferences); - if (prevReferences.size() > 0 && references != prevReferences) - printMsg(lvlError, format("warning: conflicting references for `%1%'") % path); - - if (references != prevReferences) - setReferences(txn, path, references); - } - - std::cerr << "."; - } - } - std::cerr << std::endl; - - /* !!! maybe this transaction is way too big */ - txn.commit(); -} - - -/* Upgrade from schema 2 (0.8 <= Nix <= 0.9) to schema 3 (Nix >= - 0.10). The only thing to do here is to upgrade the old `referer' - table (which causes quadratic complexity in some cases) to the new - (and properly spelled) `referrer' table. */ -static void upgradeStore09() -{ - /* !!! we should disallow concurrent upgrades */ - - printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); - - if (!pathExists(nixDBPath + "/referers")) return; - - Transaction txn(nixDB); - - std::cerr << "converting referers to referrers..."; - - TableId dbReferers = nixDB.openTable("referers"); /* sic! */ - - Paths referersKeys; - nixDB.enumTable(txn, dbReferers, referersKeys); - - int n = 0; - for (Paths::iterator i = referersKeys.begin(); - i != referersKeys.end(); ++i) - { - Paths referers; - nixDB.queryStrings(txn, dbReferers, *i, referers); - for (Paths::iterator j = referers.begin(); - j != referers.end(); ++j) - nixDB.setString(txn, dbReferrers, addPrefix(*i, *j), ""); - if (++n % 1000 == 0) { - txn.commit(); - txn.begin(nixDB); - std::cerr << "|"; - } - std::cerr << "."; - } - - txn.commit(); - - std::cerr << std::endl; - - nixDB.closeTable(dbReferers); - - nixDB.deleteTable("referers"); -} - - -} diff --git a/src/libstore/store.hh b/src/libstore/store.hh deleted file mode 100644 index 7b18871e4f27..000000000000 --- a/src/libstore/store.hh +++ /dev/null @@ -1,178 +0,0 @@ -#ifndef __STORE_H -#define __STORE_H - -#include - -#include "hash.hh" - - -namespace nix { - - -class Transaction; - - -/* Nix store and database schema version. Version 1 (or 0) was Nix <= - 0.7. Version 2 was Nix 0.8 and 0.8. Version 3 is Nix 0.10 and - up. */ -const int nixSchemaVersion = 3; - - -/* A substitute is a program invocation that constructs some store - path (typically by fetching it from somewhere, e.g., from the - network). */ -struct Substitute -{ - /* The derivation that built this store path (empty if none). */ - Path deriver; - - /* Program to be executed to create the store path. Must be in - the output path of `storeExpr'. */ - Path program; - - /* Extra arguments to be passed to the program (the first argument - is the store path to be substituted). */ - Strings args; - - bool operator == (const Substitute & sub) const; -}; - -typedef list Substitutes; - - -/* Open the database environment. If `reserveSpace' is true, make - sure that a big empty file exists in /nix/var/nix/db/reserved. If - `reserveSpace' is false, delete this file if it exists. The idea - is that on normal operation, the file exists; but when we run the - garbage collector, it is deleted. This is to ensure that the - garbage collector has a small amount of disk space available, which - is required to open the Berkeley DB environment. */ -void openDB(bool reserveSpace = true); - -/* Create the required database tables. */ -void initDB(); - -/* Close the database. */ -void closeDB(); - -/* Get a transaction object. */ -void createStoreTransaction(Transaction & txn); - -/* Copy a path recursively. */ -void copyPath(const Path & src, const Path & dst); - -/* Register a substitute. */ -void registerSubstitute(const Transaction & txn, - const Path & srcPath, const Substitute & sub); - -/* Return the substitutes for the given path. */ -Substitutes querySubstitutes(const Transaction & txn, const Path & srcPath); - -/* Deregister all substitutes. */ -void clearSubstitutes(); - -/* Register the validity of a path, i.e., that `path' exists, that the - paths referenced by it exists, and in the case of an output path of - a derivation, that it has been produced by a succesful execution of - the derivation (or something equivalent). Also register the hash - of the file system contents of the path. The hash must be a - SHA-256 hash. */ -void registerValidPath(const Transaction & txn, - const Path & path, const Hash & hash, const PathSet & references, - const Path & deriver); - -struct ValidPathInfo -{ - Path path; - Path deriver; - Hash hash; - PathSet references; -}; - -typedef list ValidPathInfos; - -void registerValidPaths(const Transaction & txn, - const ValidPathInfos & infos); - -/* Throw an exception if `path' is not directly in the Nix store. */ -void assertStorePath(const Path & path); - -bool isInStore(const Path & path); -bool isStorePath(const Path & path); - -void checkStoreName(const string & name); - -/* Chop off the parts after the top-level store name, e.g., - /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */ -Path toStorePath(const Path & path); - -/* "Fix", or canonicalise, the meta-data of the files in a store path - after it has been built. In particular: - - the last modification date on each file is set to 0 (i.e., - 00:00:00 1/1/1970 UTC) - - the permissions are set of 444 or 555 (i.e., read-only with or - without execute permission; setuid bits etc. are cleared) - - the owner and group are set to the Nix user and group, if we're - in a setuid Nix installation. */ -void canonicalisePathMetaData(const Path & path); - -/* Checks whether a path is valid. */ -bool isValidPathTxn(const Transaction & txn, const Path & path); -bool isValidPath(const Path & path); - -/* Queries the hash of a valid path. */ -Hash queryPathHash(const Path & path); - -/* Sets the set of outgoing FS references for a store path. Use with - care! */ -void setReferences(const Transaction & txn, const Path & storePath, - const PathSet & references); - -/* Queries the set of outgoing FS references for a store path. The - result is not cleared. */ -void queryReferences(const Transaction & txn, - const Path & storePath, PathSet & references); - -/* Queries the set of incoming FS references for a store path. The - result is not cleared. */ -void queryReferrers(const Transaction & txn, - const Path & storePath, PathSet & referrers); - -/* Sets the deriver of a store path. Use with care! */ -void setDeriver(const Transaction & txn, const Path & storePath, - const Path & deriver); - -/* Query the deriver of a store path. Return the empty string if no - deriver has been set. */ -Path queryDeriver(const Transaction & txn, const Path & storePath); - -/* Constructs a unique store path name. */ -Path makeStorePath(const string & type, - const Hash & hash, const string & suffix); - -/* Copy the contents of a path to the store and register the validity - the resulting path. The resulting path is returned. */ -Path addToStore(const Path & srcPath); - -/* Like addToStore(), but for pre-adding the outputs of fixed-output - derivations. */ -Path addToStoreFixed(bool recursive, string hashAlgo, const Path & srcPath); - -Path makeFixedOutputPath(bool recursive, - string hashAlgo, Hash hash, string name); - -/* Like addToStore, but the contents written to the output path is a - regular file containing the given string. */ -Path addTextToStore(const string & suffix, const string & s, - const PathSet & references); - -/* Delete a value from the nixStore directory. */ -void deleteFromStore(const Path & path, unsigned long long & bytesFreed); - -void verifyStore(bool checkContents); - - -} - - -#endif /* !__STORE_H */ -- cgit 1.4.1