diff options
Diffstat (limited to 'src/libstore')
| -rw-r--r-- | src/libstore/build.cc | 304 | ||||
| -rw-r--r-- | src/libstore/crypto.cc | 88 | ||||
| -rw-r--r-- | src/libstore/crypto.hh | 40 | ||||
| -rw-r--r-- | src/libstore/derivations.cc | 83 | ||||
| -rw-r--r-- | src/libstore/derivations.hh | 40 | ||||
| -rw-r--r-- | src/libstore/download.cc | 2 | ||||
| -rw-r--r-- | src/libstore/download.hh | 5 | ||||
| -rw-r--r-- | src/libstore/gc.cc | 49 | ||||
| -rw-r--r-- | src/libstore/local-store.cc | 81 | ||||
| -rw-r--r-- | src/libstore/local-store.hh | 14 | ||||
| -rw-r--r-- | src/libstore/local.mk | 2 | ||||
| -rw-r--r-- | src/libstore/misc.cc | 108 | ||||
| -rw-r--r-- | src/libstore/misc.hh | 40 | ||||
| -rw-r--r-- | src/libstore/nar-info.cc | 134 | ||||
| -rw-r--r-- | src/libstore/nar-info.hh | 43 | ||||
| -rw-r--r-- | src/libstore/pathlocks.cc | 6 | ||||
| -rw-r--r-- | src/libstore/profiles.cc | 4 | ||||
| -rw-r--r-- | src/libstore/profiles.hh | 4 | ||||
| -rw-r--r-- | src/libstore/remote-store.cc | 12 | ||||
| -rw-r--r-- | src/libstore/remote-store.hh | 2 | ||||
| -rw-r--r-- | src/libstore/store-api.cc | 49 | ||||
| -rw-r--r-- | src/libstore/store-api.hh | 86 |
22 files changed, 791 insertions, 405 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 1c751ab98734..249ab2335bdf 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2,7 +2,6 @@ #include "references.hh" #include "pathlocks.hh" -#include "misc.hh" #include "globals.hh" #include "local-store.hh" #include "util.hh" @@ -34,47 +33,27 @@ #include <bzlib.h> -/* Includes required for chroot support. */ -#if HAVE_SYS_PARAM_H -#include <sys/param.h> -#endif -#if HAVE_SYS_MOUNT_H -#include <sys/mount.h> -#endif -#if HAVE_SYS_SYSCALL_H -#include <sys/syscall.h> -#endif -#if HAVE_SCHED_H -#include <sched.h> -#endif - -/* In GNU libc 2.11, <sys/mount.h> does not define `MS_PRIVATE', but - <linux/fs.h> does. */ -#if !defined MS_PRIVATE && defined HAVE_LINUX_FS_H -#include <linux/fs.h> -#endif - -#define CHROOT_ENABLED HAVE_CHROOT && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_PRIVATE) && defined(CLONE_NEWNS) && defined(SYS_pivot_root) - /* chroot-like behavior from Apple's sandbox */ #if __APPLE__ - #define SANDBOX_ENABLED 1 #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh" #else - #define SANDBOX_ENABLED 0 #define DEFAULT_ALLOWED_IMPURE_PREFIXES "" #endif -#if CHROOT_ENABLED +/* Includes required for chroot support. */ +#if __linux__ #include <sys/socket.h> #include <sys/ioctl.h> #include <net/if.h> #include <netinet/ip.h> -#endif - -#if __linux__ #include <sys/personality.h> #include <sys/mman.h> +#include <sched.h> +#include <sys/param.h> +#include <sys/mount.h> +#include <sys/syscall.h> +#include <linux/fs.h> +#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) #endif #if HAVE_STATVFS @@ -641,11 +620,15 @@ HookInstance::HookInstance() if (dup2(builderOut.writeSide, 4) == -1) throw SysError("dupping builder's stdout/stderr"); - execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(), - (format("%1%") % settings.maxSilentTime).str().c_str(), - (format("%1%") % settings.printBuildTrace).str().c_str(), - (format("%1%") % settings.buildTimeout).str().c_str(), - NULL); + Strings args = { + baseNameOf(buildHook), + settings.thisSystem, + (format("%1%") % settings.maxSilentTime).str(), + (format("%1%") % settings.printBuildTrace).str(), + (format("%1%") % settings.buildTimeout).str() + }; + + execv(buildHook.c_str(), stringsToCharPtrs(args).data()); throw SysError(format("executing ‘%1%’") % buildHook); }); @@ -781,10 +764,10 @@ private: DirsInChroot dirsInChroot; typedef map<string, string> Environment; Environment env; -#if SANDBOX_ENABLED + +#if __APPLE__ typedef string SandboxProfile; SandboxProfile additionalSandboxProfile; - AutoDelete autoDelSandbox; #endif @@ -922,7 +905,7 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const BasicDerivation & drv { this->drv = std::unique_ptr<BasicDerivation>(new BasicDerivation(drv)); state = &DerivationGoal::haveDerivation; - name = (format("building of %1%") % showPaths(outputPaths(drv))).str(); + name = (format("building of %1%") % showPaths(drv.outputPaths())).str(); trace("created"); /* Prevent the .chroot directory from being @@ -1034,7 +1017,7 @@ void DerivationGoal::loadDerivation() assert(worker.store.isValidPath(drvPath)); /* Get the derivation. */ - drv = std::unique_ptr<BasicDerivation>(new Derivation(derivationFromPath(worker.store, drvPath))); + drv = std::unique_ptr<BasicDerivation>(new Derivation(worker.store.derivationFromPath(drvPath))); haveDerivation(); } @@ -1061,10 +1044,19 @@ void DerivationGoal::haveDerivation() for (auto & i : invalidOutputs) if (pathFailed(i)) return; + /* Reject doing a hash build of anything other than a fixed-output + derivation. */ + if (buildMode == bmHash) { + if (drv->outputs.size() != 1 || + drv->outputs.find("out") == drv->outputs.end() || + drv->outputs["out"].hashAlgo == "") + throw Error(format("cannot do a hash build of non-fixed-output derivation ‘%1%’") % drvPath); + } + /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ - if (settings.useSubstitutes && substitutesAllowed(*drv)) + if (settings.useSubstitutes && drv->substitutesAllowed()) for (auto & i : invalidOutputs) addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair)); @@ -1143,8 +1135,10 @@ void DerivationGoal::repairClosure() /* Get the output closure. */ PathSet outputClosure; - for (auto & i : drv->outputs) - computeFSClosure(worker.store, i.second.path, outputClosure); + for (auto & i : drv->outputs) { + if (!wantOutput(i.first, wantedOutputs)) continue; + worker.store.computeFSClosure(i.second.path, outputClosure); + } /* Filter out our own outputs (which we have already checked). */ for (auto & i : drv->outputs) @@ -1154,11 +1148,11 @@ void DerivationGoal::repairClosure() derivation is responsible for which path in the output closure. */ PathSet inputClosure; - if (useDerivation) computeFSClosure(worker.store, drvPath, inputClosure); + if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); std::map<Path, Path> outputsToDrv; for (auto & i : inputClosure) if (isDerivation(i)) { - Derivation drv = derivationFromPath(worker.store, i); + Derivation drv = worker.store.derivationFromPath(i); for (auto & j : drv.outputs) outputsToDrv[j.second.path] = i; } @@ -1230,10 +1224,10 @@ void DerivationGoal::inputsRealised() `i' as input paths. Only add the closures of output paths that are specified as inputs. */ assert(worker.store.isValidPath(i.first)); - Derivation inDrv = derivationFromPath(worker.store, i.first); + Derivation inDrv = worker.store.derivationFromPath(i.first); for (auto & j : i.second) if (inDrv.outputs.find(j) != inDrv.outputs.end()) - computeFSClosure(worker.store, inDrv.outputs[j].path, inputPaths); + worker.store.computeFSClosure(inDrv.outputs[j].path, inputPaths); else throw Error( format("derivation ‘%1%’ requires non-existent output ‘%2%’ from input derivation ‘%3%’") @@ -1242,7 +1236,7 @@ void DerivationGoal::inputsRealised() /* Second, the input sources. */ for (auto & i : drv->inputSrcs) - computeFSClosure(worker.store, i, inputPaths); + worker.store.computeFSClosure(i, inputPaths); debug(format("added input paths %1%") % showPaths(inputPaths)); @@ -1265,46 +1259,6 @@ void DerivationGoal::inputsRealised() } -static bool isBuiltin(const BasicDerivation & drv) -{ - return string(drv.builder, 0, 8) == "builtin:"; -} - - -static bool canBuildLocally(const BasicDerivation & drv) -{ - return drv.platform == settings.thisSystem - || isBuiltin(drv) -#if __linux__ - || (drv.platform == "i686-linux" && settings.thisSystem == "x86_64-linux") - || (drv.platform == "armv6l-linux" && settings.thisSystem == "armv7l-linux") -#elif __FreeBSD__ - || (drv.platform == "i686-linux" && settings.thisSystem == "x86_64-freebsd") - || (drv.platform == "i686-linux" && settings.thisSystem == "i686-freebsd") -#endif - ; -} - - -static string get(const StringPairs & map, const string & key, const string & def = "") -{ - StringPairs::const_iterator i = map.find(key); - return i == map.end() ? def : i->second; -} - - -bool willBuildLocally(const BasicDerivation & drv) -{ - return get(drv.env, "preferLocalBuild") == "1" && canBuildLocally(drv); -} - - -bool substitutesAllowed(const BasicDerivation & drv) -{ - return get(drv.env, "allowSubstitutes", "1") == "1"; -} - - void DerivationGoal::tryToBuild() { trace("trying to build"); @@ -1327,7 +1281,7 @@ void DerivationGoal::tryToBuild() can't acquire the lock, then continue; hopefully some other goal can start a build, and if not, the main loop will sleep a few seconds and then retry this goal. */ - if (!outputLocks.lockPaths(outputPaths(*drv), "", false)) { + if (!outputLocks.lockPaths(drv->outputPaths(), "", false)) { worker.waitForAWhile(shared_from_this()); return; } @@ -1340,7 +1294,6 @@ void DerivationGoal::tryToBuild() now hold the locks on the output paths, no other process can build this derivation, so no further checks are necessary. */ validPaths = checkPathValidity(true, buildMode == bmRepair); - assert(buildMode != bmCheck || validPaths.size() == drv->outputs.size()); if (buildMode != bmCheck && validPaths.size() == drv->outputs.size()) { debug(format("skipping build of derivation ‘%1%’, someone beat us to it") % drvPath); outputLocks.setDeletion(true); @@ -1348,7 +1301,7 @@ void DerivationGoal::tryToBuild() return; } - missingPaths = outputPaths(*drv); + missingPaths = drv->outputPaths(); if (buildMode != bmCheck) for (auto & i : validPaths) missingPaths.erase(i); @@ -1371,7 +1324,7 @@ void DerivationGoal::tryToBuild() /* Don't do a remote build if the derivation has the attribute `preferLocalBuild' set. Also, check and repair modes are only supported for local builds. */ - bool buildLocally = buildMode != bmNormal || willBuildLocally(*drv); + bool buildLocally = buildMode != bmNormal || drv->willBuildLocally(); /* Is the build hook willing to accept this job? */ if (!buildLocally) { @@ -1667,7 +1620,7 @@ HookReply DerivationGoal::tryBuildHook() list it since the remote system *probably* already has it.) */ PathSet allInputs; allInputs.insert(inputPaths.begin(), inputPaths.end()); - computeFSClosure(worker.store, drvPath, allInputs); + worker.store.computeFSClosure(drvPath, allInputs); string s; for (auto & i : allInputs) { s += i; s += ' '; } @@ -1722,7 +1675,7 @@ void DerivationGoal::startBuilder() startNest(nest, lvlInfo, f % showPaths(missingPaths) % curRound % nrRounds); /* Right platform? */ - if (!canBuildLocally(*drv)) { + if (!drv->canBuildLocally()) { if (settings.printBuildTrace) printMsg(lvlError, format("@ unsupported-platform %1% %2%") % drvPath % drv->platform); throw Error( @@ -1730,6 +1683,10 @@ void DerivationGoal::startBuilder() % drv->platform % settings.thisSystem % drvPath); } +#if __APPLE__ + additionalSandboxProfile = get(drv->env, "__sandboxProfile"); +#endif + /* Are we doing a chroot build? Note that fixed-output derivations are never done in a chroot, mainly so that functions like fetchurl (which needs a proper /etc/resolv.conf) @@ -1743,7 +1700,13 @@ void DerivationGoal::startBuilder() throw Error("option ‘build-use-sandbox’ must be set to one of ‘true’, ‘false’ or ‘relaxed’"); if (x == "true") { if (get(drv->env, "__noChroot") == "1") - throw Error(format("derivation ‘%1%’ has ‘__noChroot’ set, but that's not allowed when ‘build-use-sandbox’ is ‘true’") % drvPath); + throw Error(format("derivation ‘%1%’ has ‘__noChroot’ set, " + "but that's not allowed when ‘build-use-sandbox’ is ‘true’") % drvPath); +#if __APPLE__ + if (additionalSandboxProfile != "") + throw Error(format("derivation ‘%1%’ specifies a sandbox profile, " + "but this is only allowed when ‘build-use-sandbox’ is ‘relaxed’") % drvPath); +#endif useChroot = true; } else if (x == "false") @@ -1785,7 +1748,7 @@ void DerivationGoal::startBuilder() /* In a sandbox, for determinism, always use the same temporary directory. */ - tmpDirInSandbox = useChroot ? "/tmp/nix-build-" + drvName + "-0" : tmpDir; + tmpDirInSandbox = useChroot ? canonPath("/tmp", true) + "/nix-build-" + drvName + "-0" : tmpDir; /* Add all bindings specified in the derivation via the environments, except those listed in the passAsFile @@ -1798,10 +1761,11 @@ void DerivationGoal::startBuilder() if (passAsFile.find(i.first) == passAsFile.end()) { env[i.first] = i.second; } else { - Path p = tmpDir + "/.attr-" + std::to_string(fileNr++); + string fn = ".attr-" + std::to_string(fileNr++); + Path p = tmpDir + "/" + fn; writeFile(p, i.second); filesToChown.insert(p); - env[i.first + "Path"] = p; + env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; } } @@ -1868,14 +1832,14 @@ void DerivationGoal::startBuilder() like passing all build-time dependencies of some path to a derivation that builds a NixOS DVD image. */ PathSet paths, paths2; - computeFSClosure(worker.store, storePath, paths); + worker.store.computeFSClosure(storePath, paths); paths2 = paths; for (auto & j : paths2) { if (isDerivation(j)) { - Derivation drv = derivationFromPath(worker.store, j); + Derivation drv = worker.store.derivationFromPath(j); for (auto & k : drv.outputs) - computeFSClosure(worker.store, k.second.path, paths); + worker.store.computeFSClosure(k.second.path, paths); } } @@ -1908,7 +1872,7 @@ void DerivationGoal::startBuilder() if (useChroot) { string defaultChrootDirs; -#if CHROOT_ENABLED +#if __linux__ if (isInStore(BASH_PATH)) defaultChrootDirs = "/bin/sh=" BASH_PATH; #endif @@ -1939,13 +1903,10 @@ void DerivationGoal::startBuilder() PathSet closure; for (auto & i : dirsInChroot) if (isInStore(i.second)) - computeFSClosure(worker.store, toStorePath(i.second), closure); + worker.store.computeFSClosure(toStorePath(i.second), closure); for (auto & i : closure) dirsInChroot[i] = i; -#if SANDBOX_ENABLED - additionalSandboxProfile = get(drv->env, "__sandboxProfile"); -#endif string allowed = settings.get("allowed-impure-host-deps", string(DEFAULT_ALLOWED_IMPURE_PREFIXES)); PathSet allowedPaths = tokenizeString<StringSet>(allowed); @@ -1967,12 +1928,12 @@ void DerivationGoal::startBuilder() } } if (!found) - throw Error(format("derivation '%1%' requested impure path ‘%2%’, but it was not in allowed-impure-host-deps (‘%3%’)") % drvPath % i % allowed); + throw Error(format("derivation ‘%1%’ requested impure path ‘%2%’, but it was not in allowed-impure-host-deps (‘%3%’)") % drvPath % i % allowed); dirsInChroot[i] = i; } -#if CHROOT_ENABLED +#if __linux__ /* Create a temporary directory in which we set up the chroot environment using bind-mounts. We put it in the Nix store to ensure that we can create hard-links to non-directory @@ -2065,7 +2026,7 @@ void DerivationGoal::startBuilder() for (auto & i : drv->outputs) dirsInChroot.erase(i.second.path); -#elif SANDBOX_ENABLED +#elif __APPLE__ /* We don't really have any parent prep work to do (yet?) All work happens in the child, instead. */ #else @@ -2148,7 +2109,7 @@ void DerivationGoal::startBuilder() builderOut.create(); /* Fork a child to build the package. */ -#if CHROOT_ENABLED +#if __linux__ if (useChroot) { /* Set up private namespaces for the build: @@ -2189,7 +2150,7 @@ void DerivationGoal::startBuilder() size_t stackSize = 1 * 1024 * 1024; char * stack = (char *) mmap(0, stackSize, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (!stack) throw SysError("allocating stack"); + if (stack == MAP_FAILED) throw SysError("allocating stack"); int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; if (!fixedOutput) flags |= CLONE_NEWNET; pid_t child = clone(childEntry, stack + stackSize, flags, this); @@ -2210,7 +2171,7 @@ void DerivationGoal::startBuilder() #endif { ProcessOptions options; - options.allowVfork = !buildUser.enabled() && !isBuiltin(*drv); + options.allowVfork = !buildUser.enabled() && !drv->isBuiltin(); pid = startProcess([&]() { runChild(); }, options); @@ -2250,7 +2211,7 @@ void DerivationGoal::runChild() commonChildInit(builderOut); -#if CHROOT_ENABLED +#if __linux__ if (useChroot) { /* Initialise the loopback interface. */ @@ -2383,10 +2344,8 @@ void DerivationGoal::runChild() if (mkdir("real-root", 0) == -1) throw SysError("cannot create real-root directory"); -#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) if (pivot_root(".", "real-root") == -1) throw SysError(format("cannot pivot old root directory onto ‘%1%’") % (chrootRootDir + "/real-root")); -#undef pivot_root if (chroot(".") == -1) throw SysError(format("cannot change root directory to ‘%1%’") % chrootRootDir); @@ -2466,9 +2425,9 @@ void DerivationGoal::runChild() const char *builder = "invalid"; string sandboxProfile; - if (isBuiltin(*drv)) { + if (drv->isBuiltin()) { ; -#if SANDBOX_ENABLED +#if __APPLE__ } else if (useChroot) { /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ PathSet ancestry; @@ -2527,7 +2486,9 @@ void DerivationGoal::runChild() sandboxProfile += "(allow file-read* file-write* process-exec\n"; for (auto & i : dirsInChroot) { if (i.first != i.second) - throw SysError(format("can't map '%1%' to '%2%': mismatched impure paths not supported on darwin")); + throw Error(format( + "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin") + % i.first % i.second); string path = i.first; struct stat st; @@ -2581,7 +2542,7 @@ void DerivationGoal::runChild() writeFull(STDERR_FILENO, string("\1\n")); /* Execute the program. This should not return. */ - if (isBuiltin(*drv)) { + if (drv->isBuiltin()) { try { logType = ltFlat; if (drv->builder == "builtin:fetchurl") @@ -2644,6 +2605,8 @@ void DerivationGoal::registerOutputs() outputs to allow hard links between outputs. */ InodesSeen inodesSeen; + Path checkSuffix = "-check"; + /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ @@ -2669,7 +2632,7 @@ void DerivationGoal::registerOutputs() && redirectedBadOutputs.find(path) != redirectedBadOutputs.end() && pathExists(redirected)) replaceValidPath(path, redirected); - if (buildMode == bmCheck) + if (buildMode == bmCheck && redirected != "") actualPath = redirected; } @@ -2732,12 +2695,29 @@ void DerivationGoal::registerOutputs() format("output path ‘%1%’ should be a non-executable regular file") % path); } - /* Check the hash. */ + /* Check the hash. In hash mode, move the path produced by + the derivation to its content-addressed location. */ Hash h2 = recursive ? hashPath(ht, actualPath).first : hashFile(ht, actualPath); - if (h != h2) - throw BuildError( - format("Nix expects output path ‘%1%’ to have %2% hash ‘%3%’, instead it has ‘%4%’") - % path % i.second.hashAlgo % printHash16or32(h) % printHash16or32(h2)); + if (buildMode == bmHash) { + Path dest = makeFixedOutputPath(recursive, ht, h2, drv->env["name"]); + printMsg(lvlError, format("build produced path ‘%1%’ with %2% hash ‘%3%’") + % dest % printHashType(ht) % printHash16or32(h2)); + if (worker.store.isValidPath(dest)) + return; + if (actualPath != dest) { + PathLocks outputLocks({dest}); + if (pathExists(dest)) + deletePath(dest); + if (rename(actualPath.c_str(), dest.c_str()) == -1) + throw SysError(format("moving ‘%1%’ to ‘%2%’") % actualPath % dest); + } + path = actualPath = dest; + } else { + if (h != h2) + throw BuildError( + format("output path ‘%1%’ has %2% hash ‘%3%’ when ‘%4%’ was expected") + % path % i.second.hashAlgo % printHash16or32(h2) % printHash16or32(h)); + } } /* Get rid of all weird permissions. This also checks that @@ -2753,9 +2733,20 @@ void DerivationGoal::registerOutputs() PathSet references = scanForReferences(actualPath, allPaths, hash); if (buildMode == bmCheck) { + if (!worker.store.isValidPath(path)) continue; ValidPathInfo info = worker.store.queryPathInfo(path); - if (hash.first != info.hash) - throw Error(format("derivation ‘%1%’ may not be deterministic: hash mismatch in output ‘%2%’") % drvPath % path); + if (hash.first != info.narHash) { + if (settings.keepFailed) { + Path dst = path + checkSuffix; + if (pathExists(dst)) deletePath(dst); + if (rename(actualPath.c_str(), dst.c_str())) + throw SysError(format("renaming ‘%1%’ to ‘%2%’") % actualPath % dst); + throw Error(format("derivation ‘%1%’ may not be deterministic: output ‘%2%’ differs from ‘%3%’") + % drvPath % path % dst); + } else + throw Error(format("derivation ‘%1%’ may not be deterministic: output ‘%2%’ differs") + % drvPath % path); + } continue; } @@ -2781,7 +2772,7 @@ void DerivationGoal::registerOutputs() for (auto & i : references) /* Don't call computeFSClosure on ourselves. */ if (actualPath != i) - computeFSClosure(worker.store, i, used); + worker.store.computeFSClosure(i, used); } else used = references; @@ -2800,13 +2791,15 @@ void DerivationGoal::registerOutputs() checkRefs("disallowedReferences", false, false); checkRefs("disallowedRequisites", false, true); - worker.store.optimisePath(path); // FIXME: combine with scanForReferences() + if (curRound == nrRounds) { + worker.store.optimisePath(path); // FIXME: combine with scanForReferences() - worker.store.markContentsGood(path); + worker.store.markContentsGood(path); + } ValidPathInfo info; info.path = path; - info.hash = hash.first; + info.narHash = hash.first; info.narSize = hash.second; info.references = references; info.deriver = drvPath; @@ -2815,10 +2808,37 @@ void DerivationGoal::registerOutputs() if (buildMode == bmCheck) return; - if (curRound > 1 && prevInfos != infos) - throw NotDeterministic( - format("result of ‘%1%’ differs from previous round; rejecting as non-deterministic") - % drvPath); + /* Compare the result with the previous round, and report which + path is different, if any.*/ + if (curRound > 1 && prevInfos != infos) { + assert(prevInfos.size() == infos.size()); + for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) + if (!(*i == *j)) { + Path prev = i->path + checkSuffix; + if (pathExists(prev)) + throw NotDeterministic( + format("output ‘%1%’ of ‘%2%’ differs from ‘%3%’ from previous round") + % i->path % drvPath % prev); + else + throw NotDeterministic( + format("output ‘%1%’ of ‘%2%’ differs from previous round") + % i->path % drvPath); + } + assert(false); // shouldn't happen + } + + if (settings.keepFailed) { + for (auto & i : drv->outputs) { + Path prev = i.second.path + checkSuffix; + if (pathExists(prev)) deletePath(prev); + if (curRound < nrRounds) { + Path dst = i.second.path + checkSuffix; + if (rename(i.second.path.c_str(), dst.c_str())) + throw SysError(format("renaming ‘%1%’ to ‘%2%’") % i.second.path % dst); + } + } + + } if (curRound < nrRounds) { prevInfos = infos; @@ -3141,6 +3161,7 @@ void SubstitutionGoal::tryNext() /* None left. Terminate this goal and let someone else deal with it. */ debug(format("path ‘%1%’ is required, but there is no substituter that can build it") % storePath); + /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ @@ -3348,7 +3369,7 @@ void SubstitutionGoal::finished() ValidPathInfo info2; info2.path = storePath; - info2.hash = hash.first; + info2.narHash = hash.first; info2.narSize = hash.second; info2.references = info.references; info2.deriver = info.deriver; @@ -3740,7 +3761,7 @@ void Worker::waitForInput() } } - if (!waitingForAWhile.empty() && lastWokenUp + settings.pollInterval <= after) { + if (!waitingForAWhile.empty() && lastWokenUp + (time_t) settings.pollInterval <= after) { lastWokenUp = after; for (auto & i : waitingForAWhile) { GoalPtr goal = i.lock(); @@ -3836,8 +3857,17 @@ void LocalStore::repairPath(const Path & path) worker.run(goals); - if (goal->getExitCode() != Goal::ecSuccess) - throw Error(format("cannot repair path ‘%1%’") % path, worker.exitStatus()); + if (goal->getExitCode() != Goal::ecSuccess) { + /* Since substituting the path didn't work, if we have a valid + deriver, then rebuild the deriver. */ + Path deriver = queryDeriver(path); + if (deriver != "" && isValidPath(deriver)) { + goals.clear(); + goals.insert(worker.makeDerivationGoal(deriver, StringSet(), bmRepair)); + worker.run(goals); + } else + throw Error(format("cannot repair path ‘%1%’") % path, worker.exitStatus()); + } } diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc new file mode 100644 index 000000000000..c1b57e51d9b4 --- /dev/null +++ b/src/libstore/crypto.cc @@ -0,0 +1,88 @@ +#include "crypto.hh" +#include "util.hh" + +#if HAVE_SODIUM +#include <sodium.h> +#endif + +namespace nix { + +static std::pair<std::string, std::string> split(const string & s) +{ + size_t colon = s.find(':'); + if (colon == std::string::npos || colon == 0) + return {"", ""}; + return {std::string(s, 0, colon), std::string(s, colon + 1)}; +} + +Key::Key(const string & s) +{ + auto ss = split(s); + + name = ss.first; + key = ss.second; + + if (name == "" || key == "") + throw Error("secret key is corrupt"); + + key = base64Decode(key); +} + +SecretKey::SecretKey(const string & s) + : Key(s) +{ +#if HAVE_SODIUM + if (key.size() != crypto_sign_SECRETKEYBYTES) + throw Error("secret key is not valid"); +#endif +} + +[[noreturn]] static void noSodium() +{ + throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); +} + +std::string SecretKey::signDetached(const std::string & data) const +{ +#if HAVE_SODIUM + unsigned char sig[crypto_sign_BYTES]; + unsigned long long sigLen; + crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(), + (unsigned char *) key.data()); + return name + ":" + base64Encode(std::string((char *) sig, sigLen)); +#else + noSodium(); +#endif +} + +PublicKey::PublicKey(const string & s) + : Key(s) +{ +#if HAVE_SODIUM + if (key.size() != crypto_sign_PUBLICKEYBYTES) + throw Error("public key is not valid"); +#endif +} + +bool verifyDetached(const std::string & data, const std::string & sig, + const PublicKeys & publicKeys) +{ +#if HAVE_SODIUM + auto ss = split(sig); + + auto key = publicKeys.find(ss.first); + if (key == publicKeys.end()) return false; + + auto sig2 = base64Decode(ss.second); + if (sig2.size() != crypto_sign_BYTES) + throw Error("signature is not valid"); + + return crypto_sign_verify_detached((unsigned char *) sig2.data(), + (unsigned char *) data.data(), data.size(), + (unsigned char *) key->second.key.data()) == 0; +#else + noSodium(); +#endif +} + +} diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh new file mode 100644 index 000000000000..a1489e753649 --- /dev/null +++ b/src/libstore/crypto.hh @@ -0,0 +1,40 @@ +#pragma once + +#include "types.hh" + +#include <map> + +namespace nix { + +struct Key +{ + std::string name; + std::string key; + + /* Construct Key from a string in the format + ‘<name>:<key-in-base64>’. */ + Key(const std::string & s); + +}; + +struct SecretKey : Key +{ + SecretKey(const std::string & s); + + /* Return a detached signature of the given string. */ + std::string signDetached(const std::string & s) const; +}; + +struct PublicKey : Key +{ + PublicKey(const std::string & data); +}; + +typedef std::map<std::string, PublicKey> PublicKeys; + +/* Return true iff ‘sig’ is a correct signature over ‘data’ using one + of the given public keys. */ +bool verifyDetached(const std::string & data, const std::string & sig, + const PublicKeys & publicKeys); + +} diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 7959d5bfc3d5..d9b009d40322 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -2,7 +2,6 @@ #include "store-api.hh" #include "globals.hh" #include "util.hh" -#include "misc.hh" #include "worker-protocol.hh" @@ -27,7 +26,49 @@ void DerivationOutput::parseHashInfo(bool & recursive, HashType & hashType, Hash } -Path writeDerivation(StoreAPI & store, +Path BasicDerivation::findOutput(const string & id) const +{ + auto i = outputs.find(id); + if (i == outputs.end()) + throw Error(format("derivation has no output ‘%1%’") % id); + return i->second.path; +} + + +bool BasicDerivation::willBuildLocally() const +{ + return get(env, "preferLocalBuild") == "1" && canBuildLocally(); +} + + +bool BasicDerivation::substitutesAllowed() const +{ + return get(env, "allowSubstitutes", "1") == "1"; +} + + +bool BasicDerivation::isBuiltin() const +{ + return string(builder, 0, 8) == "builtin:"; +} + + +bool BasicDerivation::canBuildLocally() const +{ + return platform == settings.thisSystem + || isBuiltin() +#if __linux__ + || (platform == "i686-linux" && settings.thisSystem == "x86_64-linux") + || (platform == "armv6l-linux" && settings.thisSystem == "armv7l-linux") +#elif __FreeBSD__ + || (platform == "i686-linux" && settings.thisSystem == "x86_64-freebsd") + || (platform == "i686-linux" && settings.thisSystem == "i686-freebsd") +#endif + ; +} + + +Path writeDerivation(ref<Store> store, const Derivation & drv, const string & name, bool repair) { PathSet references; @@ -38,10 +79,10 @@ Path writeDerivation(StoreAPI & store, (that can be missing (of course) and should not necessarily be held during a garbage collection). */ string suffix = name + drvExtension; - string contents = unparseDerivation(drv); + string contents = drv.unparse(); return settings.readOnlyMode ? computeStorePathForText(suffix, contents, references) - : store.addTextToStore(suffix, contents, references, repair); + : store->addTextToStore(suffix, contents, references, repair); } @@ -149,14 +190,14 @@ static void printStrings(string & res, ForwardIterator i, ForwardIterator j) } -string unparseDerivation(const Derivation & drv) +string Derivation::unparse() const { string s; s.reserve(65536); s += "Derive(["; bool first = true; - for (auto & i : drv.outputs) { + for (auto & i : outputs) { if (first) first = false; else s += ','; s += '('; printString(s, i.first); s += ','; printString(s, i.second.path); @@ -167,7 +208,7 @@ string unparseDerivation(const Derivation & drv) s += "],["; first = true; - for (auto & i : drv.inputDrvs) { + for (auto & i : inputDrvs) { if (first) first = false; else s += ','; s += '('; printString(s, i.first); s += ','; printStrings(s, i.second.begin(), i.second.end()); @@ -175,15 +216,15 @@ string unparseDerivation(const Derivation & drv) } s += "],"; - printStrings(s, drv.inputSrcs.begin(), drv.inputSrcs.end()); + printStrings(s, inputSrcs.begin(), inputSrcs.end()); - s += ','; printString(s, drv.platform); - s += ','; printString(s, drv.builder); - s += ','; printStrings(s, drv.args.begin(), drv.args.end()); + s += ','; printString(s, platform); + s += ','; printString(s, builder); + s += ','; printStrings(s, args.begin(), args.end()); s += ",["; first = true; - for (auto & i : drv.env) { + for (auto & i : env) { if (first) first = false; else s += ','; s += '('; printString(s, i.first); s += ','; printString(s, i.second); @@ -202,11 +243,11 @@ bool isDerivation(const string & fileName) } -bool isFixedOutputDrv(const Derivation & drv) +bool BasicDerivation::isFixedOutput() const { - return drv.outputs.size() == 1 && - drv.outputs.begin()->first == "out" && - drv.outputs.begin()->second.hash != ""; + return outputs.size() == 1 && + outputs.begin()->first == "out" && + outputs.begin()->second.hash != ""; } @@ -233,10 +274,10 @@ DrvHashes drvHashes; paths have been replaced by the result of a recursive call to this function, and that for fixed-output derivations we return a hash of its output path. */ -Hash hashDerivationModulo(StoreAPI & store, Derivation drv) +Hash hashDerivationModulo(Store & store, Derivation drv) { /* Return a fixed hash for fixed-output derivations. */ - if (isFixedOutputDrv(drv)) { + if (drv.isFixedOutput()) { DerivationOutputs::const_iterator i = drv.outputs.begin(); return hashString(htSHA256, "fixed:out:" + i->second.hashAlgo + ":" @@ -259,7 +300,7 @@ Hash hashDerivationModulo(StoreAPI & store, Derivation drv) } drv.inputDrvs = inputs2; - return hashString(htSHA256, unparseDerivation(drv)); + return hashString(htSHA256, drv.unparse()); } @@ -286,10 +327,10 @@ bool wantOutput(const string & output, const std::set<string> & wanted) } -PathSet outputPaths(const BasicDerivation & drv) +PathSet BasicDerivation::outputPaths() const { PathSet paths; - for (auto & i : drv.outputs) + for (auto & i : outputs) paths.insert(i.second.path); return paths; } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index f0842045f86b..6f98869b0fe0 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -50,40 +50,56 @@ struct BasicDerivation StringPairs env; virtual ~BasicDerivation() { }; + + /* Return the path corresponding to the output identifier `id' in + the given derivation. */ + Path findOutput(const string & id) const; + + bool willBuildLocally() const; + + bool substitutesAllowed() const; + + bool isBuiltin() const; + + bool canBuildLocally() const; + + /* Return true iff this is a fixed-output derivation. */ + bool isFixedOutput() const; + + /* Return the output paths of a derivation. */ + PathSet outputPaths() const; + }; struct Derivation : BasicDerivation { DerivationInputs inputDrvs; /* inputs that are sub-derivations */ + + /* Print a derivation. */ + std::string unparse() const; }; -class StoreAPI; +class Store; /* Write a derivation to the Nix store, and return its path. */ -Path writeDerivation(StoreAPI & store, +Path writeDerivation(ref<Store> store, const Derivation & drv, const string & name, bool repair = false); /* Read a derivation from a file. */ Derivation readDerivation(const Path & drvPath); -/* Print a derivation. */ -string unparseDerivation(const Derivation & drv); - -/* Check whether a file name ends with the extensions for +/* Check whether a file name ends with the extension for derivations. */ bool isDerivation(const string & fileName); -/* Return true iff this is a fixed-output derivation. */ -bool isFixedOutputDrv(const Derivation & drv); - -Hash hashDerivationModulo(StoreAPI & store, Derivation drv); +Hash hashDerivationModulo(Store & store, Derivation drv); /* Memoisation of hashDerivationModulo(). */ typedef std::map<Path, Hash> DrvHashes; -extern DrvHashes drvHashes; +extern DrvHashes drvHashes; // FIXME: global, not thread-safe /* Split a string specifying a derivation and a set of outputs (/nix/store/hash-foo!out1,out2,...) into the derivation path and @@ -95,8 +111,6 @@ Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outpu bool wantOutput(const string & output, const std::set<string> & wanted); -PathSet outputPaths(const BasicDerivation & drv); - struct Source; struct Sink; diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 822e9a8db867..01ce1ea2fd4c 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -188,7 +188,7 @@ DownloadResult downloadFile(string url, const DownloadOptions & options) } -Path downloadFileCached(const string & url, bool unpack) +Path downloadFileCached(ref<Store> store, const string & url, bool unpack) { Path cacheDir = getEnv("XDG_CACHE_HOME", getEnv("HOME", "") + "/.cache") + "/nix/tarballs"; createDirs(cacheDir); diff --git a/src/libstore/download.hh b/src/libstore/download.hh index c1cb25b90c32..7aec8de73e48 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -1,6 +1,7 @@ #pragma once #include "types.hh" + #include <string> namespace nix { @@ -18,9 +19,11 @@ struct DownloadResult string data, etag; }; +class Store; + DownloadResult downloadFile(string url, const DownloadOptions & options); -Path downloadFileCached(const string & url, bool unpack); +Path downloadFileCached(ref<Store> store, const string & url, bool unpack); MakeError(DownloadError, Error) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 998a7516a13d..d19af1cefaf2 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,5 +1,5 @@ +#include "derivations.hh" #include "globals.hh" -#include "misc.hh" #include "local-store.hh" #include <functional> @@ -83,7 +83,7 @@ void LocalStore::addIndirectRoot(const Path & path) } -Path addPermRoot(StoreAPI & store, const Path & _storePath, +Path Store::addPermRoot(const Path & _storePath, const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir) { Path storePath(canonPath(_storePath)); @@ -101,7 +101,7 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath, if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) throw Error(format("cannot create symlink ‘%1%’; already exists") % gcRoot); makeSymlink(gcRoot, storePath); - store.addIndirectRoot(gcRoot); + addIndirectRoot(gcRoot); } else { @@ -127,7 +127,7 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath, check if the root is in a directory in or linked from the gcroots directory. */ if (settings.checkRootReachability) { - Roots roots = store.findRoots(); + Roots roots = findRoots(); if (roots.find(gcRoot) == roots.end()) printMsg(lvlError, format( @@ -139,7 +139,7 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath, /* Grab the global GC root, causing us to block while a GC is in progress. This prevents the set of permanent roots from increasing while a GC is in progress. */ - store.syncWithGC(); + syncWithGC(); return gcRoot; } @@ -260,19 +260,16 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds) } -static void foundRoot(StoreAPI & store, - const Path & path, const Path & target, Roots & roots) +void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) { - Path storePath = toStorePath(target); - if (store.isValidPath(storePath)) - roots[path] = storePath; - else - printMsg(lvlInfo, format("skipping invalid root from ‘%1%’ to ‘%2%’") % path % storePath); -} - + auto foundRoot = [&](const Path & path, const Path & target) { + Path storePath = toStorePath(target); + if (isValidPath(storePath)) + roots[path] = storePath; + else + printMsg(lvlInfo, format("skipping invalid root from ‘%1%’ to ‘%2%’") % path % storePath); + }; -static void findRoots(StoreAPI & store, const Path & path, unsigned char type, Roots & roots) -{ try { if (type == DT_UNKNOWN) @@ -280,13 +277,13 @@ static void findRoots(StoreAPI & store, const Path & path, unsigned char type, R if (type == DT_DIR) { for (auto & i : readDirectory(path)) - findRoots(store, path + "/" + i.name, i.type, roots); + findRoots(path + "/" + i.name, i.type, roots); } else if (type == DT_LNK) { Path target = readLink(path); if (isInStore(target)) - foundRoot(store, path, target, roots); + foundRoot(path, target); /* Handle indirect roots. */ else { @@ -300,14 +297,14 @@ static void findRoots(StoreAPI & store, const Path & path, unsigned char type, R struct stat st2 = lstat(target); if (!S_ISLNK(st2.st_mode)) return; Path target2 = readLink(target); - if (isInStore(target2)) foundRoot(store, target, target2, roots); + if (isInStore(target2)) foundRoot(target, target2); } } } else if (type == DT_REG) { Path storePath = settings.nixStore + "/" + baseNameOf(path); - if (store.isValidPath(storePath)) + if (isValidPath(storePath)) roots[path] = storePath; } @@ -328,16 +325,16 @@ Roots LocalStore::findRoots() Roots roots; /* Process direct roots in {gcroots,manifests,profiles}. */ - nix::findRoots(*this, settings.nixStateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); + findRoots(settings.nixStateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); if (pathExists(settings.nixStateDir + "/manifests")) - nix::findRoots(*this, settings.nixStateDir + "/manifests", DT_UNKNOWN, roots); - nix::findRoots(*this, settings.nixStateDir + "/profiles", DT_UNKNOWN, roots); + findRoots(settings.nixStateDir + "/manifests", DT_UNKNOWN, roots); + findRoots(settings.nixStateDir + "/profiles", DT_UNKNOWN, roots); return roots; } -static void addAdditionalRoots(StoreAPI & store, PathSet & roots) +void LocalStore::findRuntimeRoots(PathSet & roots) { Path rootFinder = getEnv("NIX_ROOT_FINDER", settings.nixLibexecDir + "/nix/find-runtime-roots.pl"); @@ -353,7 +350,7 @@ static void addAdditionalRoots(StoreAPI & store, PathSet & roots) for (auto & i : paths) if (isInStore(i)) { Path path = toStorePath(i); - if (roots.find(path) == roots.end() && store.isValidPath(path)) { + if (roots.find(path) == roots.end() && isValidPath(path)) { debug(format("got additional root ‘%1%’") % path); roots.insert(path); } @@ -628,7 +625,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) to add running programs to the set of roots (to prevent them from being garbage collected). */ if (!options.ignoreLiveness) - addAdditionalRoots(*this, state.roots); + findRuntimeRoots(state.roots); /* Read the temporary roots. This acquires read locks on all per-process temporary root files. So after this point no paths diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 978bca28d7fa..308aebd73bb4 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -23,16 +23,11 @@ #include <time.h> #include <grp.h> -#if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H +#if __linux__ #include <sched.h> #include <sys/statvfs.h> #include <sys/mount.h> -#endif - -#if HAVE_LINUX_FS_H -#include <linux/fs.h> #include <sys/ioctl.h> -#include <errno.h> #endif #include <sqlite3.h> @@ -45,10 +40,7 @@ MakeError(SQLiteError, Error); MakeError(SQLiteBusy, SQLiteError); -static void throwSQLiteError(sqlite3 * db, const format & f) - __attribute__ ((noreturn)); - -static void throwSQLiteError(sqlite3 * db, const format & f) +[[noreturn]] static void throwSQLiteError(sqlite3 * db, const format & f) { int err = sqlite3_errcode(db); if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { @@ -402,9 +394,15 @@ int LocalStore::getSchema() } +bool LocalStore::haveWriteAccess() +{ + return access(settings.nixDBPath.c_str(), R_OK | W_OK) == 0; +} + + void LocalStore::openDB(bool create) { - if (access(settings.nixDBPath.c_str(), R_OK | W_OK)) + if (!haveWriteAccess()) throw SysError(format("Nix database directory ‘%1%’ is not writable") % settings.nixDBPath); /* Open the Nix database. */ @@ -502,7 +500,7 @@ void LocalStore::openDB(bool create) bind mount. So make the Nix store writable for this process. */ void LocalStore::makeStoreWritable() { -#if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_REMOUNT) +#if __linux__ if (getuid() != 0) return; /* Check if /nix/store is on a read-only mount. */ struct statvfs stat; @@ -607,10 +605,10 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe users group); we check for this case below. */ if (st.st_uid != geteuid()) { #if HAVE_LCHOWN - if (lchown(path.c_str(), geteuid(), (gid_t) -1) == -1) + if (lchown(path.c_str(), geteuid(), getegid()) == -1) #else if (!S_ISLNK(st.st_mode) && - chown(path.c_str(), geteuid(), (gid_t) -1) == -1) + chown(path.c_str(), geteuid(), getegid()) == -1) #endif throw SysError(format("changing owner of ‘%1%’ to %2%") % path % geteuid()); @@ -654,7 +652,7 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & assert(isDerivation(drvName)); drvName = string(drvName, 0, drvName.size() - drvExtension.size()); - if (isFixedOutputDrv(drv)) { + if (drv.isFixedOutput()) { DerivationOutputs::const_iterator out = drv.outputs.find("out"); if (out == drv.outputs.end()) throw Error(format("derivation ‘%1%’ does not have an output named ‘out’") % drvPath); @@ -693,7 +691,7 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che { SQLiteStmtUse use(stmtRegisterValidPath); stmtRegisterValidPath.bind(info.path); - stmtRegisterValidPath.bind("sha256:" + printHash(info.hash)); + stmtRegisterValidPath.bind("sha256:" + printHash(info.narHash)); stmtRegisterValidPath.bind(info.registrationTime == 0 ? time(0) : info.registrationTime); if (info.deriver != "") stmtRegisterValidPath.bind(info.deriver); @@ -844,7 +842,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1); assert(s); - info.hash = parseHashField(path, s); + info.narHash = parseHashField(path, s); info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); @@ -882,7 +880,7 @@ void LocalStore::updatePathInfo(const ValidPathInfo & info) stmtUpdatePathInfo.bind64(info.narSize); else stmtUpdatePathInfo.bind(); // null - stmtUpdatePathInfo.bind("sha256:" + printHash(info.hash)); + stmtUpdatePathInfo.bind("sha256:" + printHash(info.narHash)); stmtUpdatePathInfo.bind(info.path); if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE) throwSQLiteError(db, format("updating info of path ‘%1%’ in database") % info.path); @@ -952,14 +950,6 @@ PathSet LocalStore::queryAllValidPaths() } -void LocalStore::queryReferences(const Path & path, - PathSet & references) -{ - ValidPathInfo info = queryPathInfo(path); - references.insert(info.references.begin(), info.references.end()); -} - - void LocalStore::queryReferrers_(const Path & path, PathSet & referrers) { SQLiteStmtUse use(stmtQueryReferrers); @@ -1063,7 +1053,7 @@ StringSet LocalStore::queryDerivationOutputNames(const Path & path) Path LocalStore::queryPathFromHashPart(const string & hashPart) { - if (hashPart.size() != 32) throw Error("invalid hash part"); + if (hashPart.size() != storePathHashLen) throw Error("invalid hash part"); Path prefix = settings.nixStore + "/" + hashPart; @@ -1281,7 +1271,7 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths, Hash LocalStore::queryPathHash(const Path & path) { - return queryPathInfo(path).hash; + return queryPathInfo(path).narHash; } @@ -1305,7 +1295,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) PathSet paths; for (auto & i : infos) { - assert(i.hash.type == htSHA256); + assert(i.narHash.type == htSHA256); if (isValidPath_(i.path)) updatePathInfo(i); else @@ -1334,7 +1324,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) error if a cycle is detected and roll back the transaction. Cycles can only occur when a derivation has multiple outputs. */ - topoSortPaths(*this, paths); + topoSortPaths(paths); txn.commit(); } end_retry_sqlite; @@ -1404,7 +1394,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, ValidPathInfo info; info.path = dstPath; - info.hash = hash.first; + info.narHash = hash.first; info.narSize = hash.second; registerValidPath(info); } @@ -1460,7 +1450,7 @@ Path LocalStore::addTextToStore(const string & name, const string & s, ValidPathInfo info; info.path = dstPath; - info.hash = hash.first; + info.narHash = hash.first; info.narSize = hash.second; info.references = references; registerValidPath(info); @@ -1492,9 +1482,6 @@ struct HashAndWriteSink : Sink }; -#define EXPORT_MAGIC 0x4558494e - - static void checkSecrecy(const Path & path) { struct stat st; @@ -1531,7 +1518,7 @@ void LocalStore::exportPath(const Path & path, bool sign, PathSet references; queryReferences(path, references); - hashAndWriteSink << EXPORT_MAGIC << path << references << queryDeriver(path); + hashAndWriteSink << exportMagic << path << references << queryDeriver(path); if (sign) { Hash hash = hashAndWriteSink.currentHash(); @@ -1607,8 +1594,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source) restorePath(unpacked, hashAndReadSource); - unsigned int magic = readInt(hashAndReadSource); - if (magic != EXPORT_MAGIC) + uint32_t magic = readInt(hashAndReadSource); + if (magic != exportMagic) throw Error("Nix archive cannot be imported; wrong format"); Path dstPath = readStorePath(hashAndReadSource); @@ -1690,7 +1677,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) ValidPathInfo info; info.path = dstPath; - info.hash = hash.first; + info.narHash = hash.first; info.narSize = hash.second; info.references = references; info.deriver = deriver != "" && isValidPath(deriver) ? deriver : ""; @@ -1774,21 +1761,21 @@ bool LocalStore::verifyStore(bool checkContents, bool repair) /* Check the content hash (optionally - slow). */ printMsg(lvlTalkative, format("checking contents of ‘%1%’") % i); - HashResult current = hashPath(info.hash.type, i); + HashResult current = hashPath(info.narHash.type, i); - if (info.hash != nullHash && info.hash != current.first) { + if (info.narHash != nullHash && info.narHash != current.first) { printMsg(lvlError, format("path ‘%1%’ was modified! " "expected hash ‘%2%’, got ‘%3%’") - % i % printHash(info.hash) % printHash(current.first)); + % i % printHash(info.narHash) % printHash(current.first)); if (repair) repairPath(i); else errors = true; } else { bool update = false; /* Fill in missing hashes. */ - if (info.hash == nullHash) { + if (info.narHash == nullHash) { printMsg(lvlError, format("fixing missing hash on ‘%1%’") % i); - info.hash = current.first; + info.narHash = current.first; update = true; } @@ -1877,9 +1864,9 @@ bool LocalStore::pathContentsGood(const Path & path) if (!pathExists(path)) res = false; else { - HashResult current = hashPath(info.hash.type, path); + HashResult current = hashPath(info.narHash.type, path); Hash nullHash(htSHA256); - res = info.hash == nullHash || info.hash == current.first; + res = info.narHash == nullHash || info.narHash == current.first; } pathContentsGoodCache[path] = res; if (!res) printMsg(lvlError, format("path ‘%1%’ is corrupted or missing!") % path); @@ -1931,7 +1918,7 @@ ValidPathInfo LocalStore::queryPathInfoOld(const Path & path) } else if (name == "Deriver") { res.deriver = value; } else if (name == "Hash") { - res.hash = parseHashField(path, value); + res.narHash = parseHashField(path, value); } else if (name == "Registered-At") { int n = 0; string2Int(value, n); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ebdf19bf1359..a96000d9fbeb 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -80,7 +80,7 @@ struct SQLiteStmt }; -class LocalStore : public StoreAPI +class LocalStore : public Store { private: typedef std::map<Path, RunningSubstituter> RunningSubstituters; @@ -108,8 +108,6 @@ public: Hash queryPathHash(const Path & path) override; - void queryReferences(const Path & path, PathSet & references) override; - void queryReferrers(const Path & path, PathSet & referrers) override; Path queryDeriver(const Path & path) override; @@ -253,6 +251,12 @@ private: int getSchema(); +public: + + static bool haveWriteAccess(); + +private: + void openDB(bool create); void makeStoreWritable(); @@ -297,6 +301,10 @@ private: int openGCLock(LockType lockType); + void findRoots(const Path & path, unsigned char type, Roots & roots); + + void findRuntimeRoots(PathSet & roots); + void removeUnusedLinks(const GCState & state); void startSubstituter(const Path & substituter, diff --git a/src/libstore/local.mk b/src/libstore/local.mk index e78f47949ad3..9a01596c36be 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -8,7 +8,7 @@ libstore_SOURCES := $(wildcard $(d)/*.cc) libstore_LIBS = libutil libformat -libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) +libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) ifeq ($(OS), SunOS) libstore_LDFLAGS += -lsocket diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 61a976c02f5c..12472f017ce4 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,21 +1,21 @@ -#include "misc.hh" -#include "store-api.hh" -#include "local-store.hh" +#include "derivations.hh" #include "globals.hh" +#include "local-store.hh" +#include "store-api.hh" namespace nix { -Derivation derivationFromPath(StoreAPI & store, const Path & drvPath) +Derivation Store::derivationFromPath(const Path & drvPath) { assertStorePath(drvPath); - store.ensurePath(drvPath); + ensurePath(drvPath); return readDerivation(drvPath); } -void computeFSClosure(StoreAPI & store, const Path & path, +void Store::computeFSClosure(const Path & path, PathSet & paths, bool flipDirection, bool includeOutputs, bool includeDerivers) { if (paths.find(path) != paths.end()) return; @@ -24,50 +24,42 @@ void computeFSClosure(StoreAPI & store, const Path & path, PathSet edges; if (flipDirection) { - store.queryReferrers(path, edges); + queryReferrers(path, edges); if (includeOutputs) { - PathSet derivers = store.queryValidDerivers(path); + PathSet derivers = queryValidDerivers(path); for (auto & i : derivers) edges.insert(i); } if (includeDerivers && isDerivation(path)) { - PathSet outputs = store.queryDerivationOutputs(path); + PathSet outputs = queryDerivationOutputs(path); for (auto & i : outputs) - if (store.isValidPath(i) && store.queryDeriver(i) == path) + if (isValidPath(i) && queryDeriver(i) == path) edges.insert(i); } } else { - store.queryReferences(path, edges); + queryReferences(path, edges); if (includeOutputs && isDerivation(path)) { - PathSet outputs = store.queryDerivationOutputs(path); + PathSet outputs = queryDerivationOutputs(path); for (auto & i : outputs) - if (store.isValidPath(i)) edges.insert(i); + if (isValidPath(i)) edges.insert(i); } if (includeDerivers) { - Path deriver = store.queryDeriver(path); - if (store.isValidPath(deriver)) edges.insert(deriver); + Path deriver = queryDeriver(path); + if (isValidPath(deriver)) edges.insert(deriver); } } for (auto & i : edges) - computeFSClosure(store, i, paths, flipDirection, includeOutputs, includeDerivers); -} - - -Path findOutput(const Derivation & drv, string id) -{ - for (auto & i : drv.outputs) - if (i.first == id) return i.second.path; - throw Error(format("derivation has no output ‘%1%’") % id); + computeFSClosure(i, paths, flipDirection, includeOutputs, includeDerivers); } -void queryMissing(StoreAPI & store, const PathSet & targets, +void Store::queryMissing(const PathSet & targets, PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, unsigned long long & downloadSize, unsigned long long & narSize) { @@ -105,27 +97,27 @@ void queryMissing(StoreAPI & store, const PathSet & targets, DrvPathWithOutputs i2 = parseDrvPathWithOutputs(i); if (isDerivation(i2.first)) { - if (!store.isValidPath(i2.first)) { + if (!isValidPath(i2.first)) { // FIXME: we could try to substitute p. unknown.insert(i); continue; } - Derivation drv = derivationFromPath(store, i2.first); + Derivation drv = derivationFromPath(i2.first); PathSet invalid; for (auto & j : drv.outputs) if (wantOutput(j.first, i2.second) - && !store.isValidPath(j.second.path)) + && !isValidPath(j.second.path)) invalid.insert(j.second.path); if (invalid.empty()) continue; todoDrv.insert(i); - if (settings.useSubstitutes && substitutesAllowed(drv)) + if (settings.useSubstitutes && drv.substitutesAllowed()) query.insert(invalid.begin(), invalid.end()); } else { - if (store.isValidPath(i)) continue; + if (isValidPath(i)) continue; query.insert(i); todoNonDrv.insert(i); } @@ -134,20 +126,20 @@ void queryMissing(StoreAPI & store, const PathSet & targets, todo.clear(); SubstitutablePathInfos infos; - store.querySubstitutablePathInfos(query, infos); + querySubstitutablePathInfos(query, infos); for (auto & i : todoDrv) { DrvPathWithOutputs i2 = parseDrvPathWithOutputs(i); // FIXME: cache this - Derivation drv = derivationFromPath(store, i2.first); + Derivation drv = derivationFromPath(i2.first); PathSet outputs; bool mustBuild = false; - if (settings.useSubstitutes && substitutesAllowed(drv)) { + if (settings.useSubstitutes && drv.substitutesAllowed()) { for (auto & j : drv.outputs) { if (!wantOutput(j.first, i2.second)) continue; - if (!store.isValidPath(j.second.path)) { + if (!isValidPath(j.second.path)) { if (infos.find(j.second.path) == infos.end()) mustBuild = true; else @@ -181,38 +173,38 @@ void queryMissing(StoreAPI & store, const PathSet & targets, } -static void dfsVisit(StoreAPI & store, const PathSet & paths, - const Path & path, PathSet & visited, Paths & sorted, - PathSet & parents) +Paths Store::topoSortPaths(const PathSet & paths) { - if (parents.find(path) != parents.end()) - throw BuildError(format("cycle detected in the references of ‘%1%’") % path); + Paths sorted; + PathSet visited, parents; - if (visited.find(path) != visited.end()) return; - visited.insert(path); - parents.insert(path); + std::function<void(const Path & path)> dfsVisit; - PathSet references; - if (store.isValidPath(path)) - store.queryReferences(path, references); + dfsVisit = [&](const Path & path) { + if (parents.find(path) != parents.end()) + throw BuildError(format("cycle detected in the references of ‘%1%’") % path); - for (auto & i : references) - /* Don't traverse into paths that don't exist. That can - happen due to substitutes for non-existent paths. */ - if (i != path && paths.find(i) != paths.end()) - dfsVisit(store, paths, i, visited, sorted, parents); + if (visited.find(path) != visited.end()) return; + visited.insert(path); + parents.insert(path); - sorted.push_front(path); - parents.erase(path); -} + PathSet references; + if (isValidPath(path)) + queryReferences(path, references); + for (auto & i : references) + /* Don't traverse into paths that don't exist. That can + happen due to substitutes for non-existent paths. */ + if (i != path && paths.find(i) != paths.end()) + dfsVisit(i); + + sorted.push_front(path); + parents.erase(path); + }; -Paths topoSortPaths(StoreAPI & store, const PathSet & paths) -{ - Paths sorted; - PathSet visited, parents; for (auto & i : paths) - dfsVisit(store, paths, i, visited, sorted, parents); + dfsVisit(i); + return sorted; } diff --git a/src/libstore/misc.hh b/src/libstore/misc.hh deleted file mode 100644 index 495c528750fb..000000000000 --- a/src/libstore/misc.hh +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once - -#include "derivations.hh" - - -namespace nix { - - -/* Read a derivation, after ensuring its existence through - ensurePath(). */ -Derivation derivationFromPath(StoreAPI & store, const Path & drvPath); - -/* Place in `paths' the set of all store paths in the file system - closure of `storePath'; that is, all paths than can be directly or - indirectly reached from it. `paths' is not cleared. If - `flipDirection' is true, the set of paths that can reach - `storePath' is returned; that is, the closures under the - `referrers' relation instead of the `references' relation is - returned. */ -void computeFSClosure(StoreAPI & store, const Path & path, - PathSet & paths, bool flipDirection = false, - bool includeOutputs = false, bool includeDerivers = false); - -/* Return the path corresponding to the output identifier `id' in the - given derivation. */ -Path findOutput(const Derivation & drv, string id); - -/* Given a set of paths that are to be built, return the set of - derivations that will be built, and the set of output paths that - will be substituted. */ -void queryMissing(StoreAPI & store, const PathSet & targets, - PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize); - -bool willBuildLocally(const BasicDerivation & drv); - -bool substitutesAllowed(const BasicDerivation & drv); - - -} diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc new file mode 100644 index 000000000000..e9260a09bf5a --- /dev/null +++ b/src/libstore/nar-info.cc @@ -0,0 +1,134 @@ +#include "crypto.hh" +#include "globals.hh" +#include "nar-info.hh" + +namespace nix { + +NarInfo::NarInfo(const std::string & s, const std::string & whence) +{ + auto corrupt = [&]() { + throw Error("NAR info file ‘%1%’ is corrupt"); + }; + + auto parseHashField = [&](const string & s) { + string::size_type colon = s.find(':'); + if (colon == string::npos) corrupt(); + HashType ht = parseHashType(string(s, 0, colon)); + if (ht == htUnknown) corrupt(); + return parseHash16or32(ht, string(s, colon + 1)); + }; + + size_t pos = 0; + while (pos < s.size()) { + + size_t colon = s.find(':', pos); + if (colon == std::string::npos) corrupt(); + + std::string name(s, pos, colon - pos); + + size_t eol = s.find('\n', colon + 2); + if (eol == std::string::npos) corrupt(); + + std::string value(s, colon + 2, eol - colon - 2); + + if (name == "StorePath") { + if (!isStorePath(value)) corrupt(); + path = value; + } + else if (name == "URL") + url = value; + else if (name == "Compression") + compression = value; + else if (name == "FileHash") + fileHash = parseHashField(value); + else if (name == "FileSize") { + if (!string2Int(value, fileSize)) corrupt(); + } + else if (name == "NarHash") + narHash = parseHashField(value); + else if (name == "NarSize") { + if (!string2Int(value, narSize)) corrupt(); + } + else if (name == "References") { + auto refs = tokenizeString<Strings>(value, " "); + if (!references.empty()) corrupt(); + for (auto & r : refs) { + auto r2 = settings.nixStore + "/" + r; + if (!isStorePath(r2)) corrupt(); + references.insert(r2); + } + } + else if (name == "Deriver") { + auto p = settings.nixStore + "/" + value; + if (!isStorePath(p)) corrupt(); + deriver = p; + } + else if (name == "System") + system = value; + else if (name == "Sig") + sig = value; + + pos = eol + 1; + } + + if (compression == "") compression = "bzip2"; + + if (path.empty() || url.empty()) corrupt(); +} + +std::string NarInfo::to_string() const +{ + std::string res; + res += "StorePath: " + path + "\n"; + res += "URL: " + url + "\n"; + assert(compression != ""); + res += "Compression: " + compression + "\n"; + assert(fileHash.type == htSHA256); + res += "FileHash: sha256:" + printHash32(fileHash) + "\n"; + res += "FileSize: " + std::to_string(fileSize) + "\n"; + assert(narHash.type == htSHA256); + res += "NarHash: sha256:" + printHash32(narHash) + "\n"; + res += "NarSize: " + std::to_string(narSize) + "\n"; + + res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; + + if (!deriver.empty()) + res += "Deriver: " + baseNameOf(deriver) + "\n"; + + if (!system.empty()) + res += "System: " + system + "\n"; + + if (!sig.empty()) + res += "Sig: " + sig + "\n"; + + return res; +} + +std::string NarInfo::fingerprint() const +{ + return + "1;" + path + ";" + + printHashType(narHash.type) + ":" + printHash32(narHash) + ";" + + std::to_string(narSize) + ";" + + concatStringsSep(",", references); +} + +Strings NarInfo::shortRefs() const +{ + Strings refs; + for (auto & r : references) + refs.push_back(baseNameOf(r)); + return refs; +} + +void NarInfo::sign(const SecretKey & secretKey) +{ + sig = secretKey.signDetached(fingerprint()); +} + +bool NarInfo::checkSignature(const PublicKeys & publicKeys) const +{ + return sig != "" && verifyDetached(fingerprint(), sig, publicKeys); +} + +} diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh new file mode 100644 index 000000000000..22e27cb42ebf --- /dev/null +++ b/src/libstore/nar-info.hh @@ -0,0 +1,43 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" +#include "store-api.hh" + +namespace nix { + +struct NarInfo : ValidPathInfo +{ + std::string url; + std::string compression; + Hash fileHash; + uint64_t fileSize = 0; + std::string system; + std::string sig; // FIXME: support multiple signatures + + NarInfo() { } + NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } + NarInfo(const std::string & s, const std::string & whence); + + std::string to_string() const; + + /* Return a fingerprint of the store path to be used in binary + cache signatures. It contains the store path, the base-32 + SHA-256 hash of the NAR serialisation of the path, the size of + the NAR, and the sorted references. The size field is strictly + speaking superfluous, but might prevent endless/excessive data + attacks. */ + std::string fingerprint() const; + + void sign(const SecretKey & secretKey); + + /* Return true iff this .narinfo is signed by one of the specified + keys. */ + bool checkSignature(const PublicKeys & publicKeys) const; + +private: + + Strings shortRefs() const; +}; + +} diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 1c87034f8e1a..eddf5bcbda65 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -161,7 +161,11 @@ bool PathLocks::lockPaths(const PathSet & _paths, PathLocks::~PathLocks() { - unlock(); + try { + unlock(); + } catch (...) { + ignoreException(); + } } diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index da3f7da9d19d..cc83a838eddc 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -74,7 +74,7 @@ static void makeName(const Path & profile, unsigned int num, } -Path createGeneration(Path profile, Path outPath) +Path createGeneration(ref<Store> store, Path profile, Path outPath) { /* The new generation number should be higher than old the previous ones. */ @@ -108,7 +108,7 @@ Path createGeneration(Path profile, Path outPath) user environment etc. we've just built. */ Path generation; makeName(profile, num + 1, generation); - addPermRoot(*store, outPath, generation, false, true); + store->addPermRoot(outPath, generation, false, true); return generation; } diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index e99bbf398a86..d758d94b603c 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -31,7 +31,9 @@ typedef list<Generation> Generations; profile, sorted by generation number. */ Generations findGenerations(Path profile, int & curGen); -Path createGeneration(Path profile, Path outPath); +class Store; + +Path createGeneration(ref<Store> store, Path profile, Path outPath); void deleteGeneration(const Path & profile, unsigned int gen); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 262e4650bfb5..ab2ebb9aecc3 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -50,14 +50,8 @@ void RemoteStore::openConnection(bool reserveSpace) if (initialised) return; initialised = true; - string remoteMode = getEnv("NIX_REMOTE"); - - if (remoteMode == "daemon") - /* Connect to a daemon that does the privileged work for - us. */ - connectToDaemon(); - else - throw Error(format("invalid setting for NIX_REMOTE, ‘%1%’") % remoteMode); + /* Connect to a daemon that does the privileged work for us. */ + connectToDaemon(); from.fd = fdSocket; to.fd = fdSocket; @@ -279,7 +273,7 @@ ValidPathInfo RemoteStore::queryPathInfo(const Path & path) info.path = path; info.deriver = readString(from); if (info.deriver != "") assertStorePath(info.deriver); - info.hash = parseHash(htSHA256, readString(from)); + info.narHash = parseHash(htSHA256, readString(from)); info.references = readStorePaths<PathSet>(from); info.registrationTime = readInt(from); info.narSize = readLongLong(from); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index b0787c072904..f15182285e8c 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -14,7 +14,7 @@ struct FdSink; struct FdSource; -class RemoteStore : public StoreAPI +class RemoteStore : public Store { public: diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index a73ebd824264..82872cc335e3 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -61,7 +61,14 @@ Path followLinksToStorePath(const Path & path) string storePathToName(const Path & path) { assertStorePath(path); - return string(path, settings.nixStore.size() + 34); + return string(path, settings.nixStore.size() + storePathHashLen + 2); +} + + +string storePathToHash(const Path & path) +{ + assertStorePath(path); + return string(path, settings.nixStore.size() + 1, storePathHashLen); } @@ -217,10 +224,17 @@ Path computeStorePathForText(const string & name, const string & s, } +void Store::queryReferences(const Path & path, PathSet & references) +{ + ValidPathInfo info = queryPathInfo(path); + references.insert(info.references.begin(), info.references.end()); +} + + /* Return a string accepted by decodeValidPathInfo() that registers the specified paths as valid. Note: it's the responsibility of the caller to provide a closure. */ -string StoreAPI::makeValidityRegistration(const PathSet & paths, +string Store::makeValidityRegistration(const PathSet & paths, bool showDerivers, bool showHash) { string s = ""; @@ -231,7 +245,7 @@ string StoreAPI::makeValidityRegistration(const PathSet & paths, ValidPathInfo info = queryPathInfo(i); if (showHash) { - s += printHash(info.hash) + "\n"; + s += printHash(info.narHash) + "\n"; s += (format("%1%\n") % info.narSize).str(); } @@ -256,7 +270,7 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) if (hashGiven) { string s; getline(str, s); - info.hash = parseHash(htSHA256, s); + info.narHash = parseHash(htSHA256, s); getline(str, s); if (!string2Int(s, info.narSize)) throw Error("number expected"); } @@ -284,12 +298,12 @@ string showPaths(const PathSet & paths) } -void exportPaths(StoreAPI & store, const Paths & paths, +void Store::exportPaths(const Paths & paths, bool sign, Sink & sink) { for (auto & i : paths) { sink << 1; - store.exportPath(i, sign, sink); + exportPath(i, sign, sink); } sink << 0; } @@ -306,15 +320,24 @@ void exportPaths(StoreAPI & store, const Paths & paths, namespace nix { -std::shared_ptr<StoreAPI> store; +ref<Store> openStore(bool reserveSpace) +{ + enum { mDaemon, mLocal, mAuto } mode; + mode = getEnv("NIX_REMOTE") == "daemon" ? mDaemon : mAuto; -std::shared_ptr<StoreAPI> openStore(bool reserveSpace) -{ - if (getEnv("NIX_REMOTE") == "") - return std::shared_ptr<StoreAPI>(new LocalStore(reserveSpace)); - else - return std::shared_ptr<StoreAPI>(new RemoteStore()); + if (mode == mAuto) { + if (LocalStore::haveWriteAccess()) + mode = mLocal; + else if (pathExists(settings.nixDaemonSocketFile)) + mode = mDaemon; + else + mode = mLocal; + } + + return mode == mDaemon + ? (ref<Store>) make_ref<RemoteStore>() + : (ref<Store>) make_ref<LocalStore>(reserveSpace); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9cc5fd45b7c4..6f50e3c55aba 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -12,6 +12,13 @@ namespace nix { +/* Size of the hash part of store paths, in base-32 characters. */ +const size_t storePathHashLen = 32; // i.e. 160 bits + +/* Magic header of exportPath() output. */ +const uint32_t exportMagic = 0x4558494e; + + typedef std::map<Path, Path> Roots; @@ -85,7 +92,7 @@ struct ValidPathInfo { Path path; Path deriver; - Hash hash; + Hash narHash; PathSet references; time_t registrationTime = 0; unsigned long long narSize = 0; // 0 = unknown @@ -95,7 +102,7 @@ struct ValidPathInfo { return path == i.path - && hash == i.hash + && narHash == i.narHash && references == i.references; } }; @@ -103,7 +110,7 @@ struct ValidPathInfo typedef list<ValidPathInfo> ValidPathInfos; -enum BuildMode { bmNormal, bmRepair, bmCheck }; +enum BuildMode { bmNormal, bmRepair, bmCheck, bmHash }; struct BuildResult @@ -132,13 +139,14 @@ struct BuildResult struct BasicDerivation; +struct Derivation; -class StoreAPI +class Store { public: - virtual ~StoreAPI() { } + virtual ~Store() { } /* Check whether a path is valid. */ virtual bool isValidPath(const Path & path) = 0; @@ -155,10 +163,9 @@ public: /* Query the hash of a valid path. */ virtual Hash queryPathHash(const Path & path) = 0; - /* Query the set of outgoing FS references for a store path. The + /* Query the set of outgoing FS references for a store path. The result is not cleared. */ - virtual void queryReferences(const Path & path, - PathSet & references) = 0; + virtual void queryReferences(const Path & path, PathSet & references); /* Queries the set of incoming FS references for a store path. The result is not cleared. */ @@ -214,6 +221,10 @@ public: virtual void exportPath(const Path & path, bool sign, Sink & sink) = 0; + /* Export multiple paths in the format expected by ‘nix-store + --import’. */ + void exportPaths(const Paths & paths, bool sign, Sink & sink); + /* Import a sequence of NAR dumps created by exportPaths() into the Nix store. */ virtual Paths importPaths(bool requireSignature, Source & source) = 0; @@ -250,6 +261,10 @@ public: `path' has disappeared. */ virtual void addIndirectRoot(const Path & path) = 0; + /* Register a permanent GC root. */ + Path addPermRoot(const Path & storePath, + const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false); + /* Acquire the global GC lock, then immediately release it. This function must be called after registering a new permanent root, but before exiting. Otherwise, it is possible that a running @@ -298,6 +313,35 @@ public: /* Check the integrity of the Nix store. Returns true if errors remain. */ virtual bool verifyStore(bool checkContents, bool repair) = 0; + + /* Utility functions. */ + + /* Read a derivation, after ensuring its existence through + ensurePath(). */ + Derivation derivationFromPath(const Path & drvPath); + + /* Place in `paths' the set of all store paths in the file system + closure of `storePath'; that is, all paths than can be directly + or indirectly reached from it. `paths' is not cleared. If + `flipDirection' is true, the set of paths that can reach + `storePath' is returned; that is, the closures under the + `referrers' relation instead of the `references' relation is + returned. */ + void computeFSClosure(const Path & path, + PathSet & paths, bool flipDirection = false, + bool includeOutputs = false, bool includeDerivers = false); + + /* Given a set of paths that are to be built, return the set of + derivations that will be built, and the set of output paths + that will be substituted. */ + void queryMissing(const PathSet & targets, + PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, + unsigned long long & downloadSize, unsigned long long & narSize); + + /* Sort a set of paths topologically under the references + relation. If p refers to q, then p preceeds q in this list. */ + Paths topoSortPaths(const PathSet & paths); + }; @@ -312,6 +356,9 @@ bool isStorePath(const Path & path); /* Extract the name part of the given store path. */ string storePathToName(const Path & path); +/* Extract the hash part of the given store path. */ +string storePathToHash(const Path & path); + void checkStoreName(const string & name); @@ -372,24 +419,9 @@ Path computeStorePathForText(const string & name, const string & s, void removeTempRoots(); -/* Register a permanent GC root. */ -Path addPermRoot(StoreAPI & store, const Path & storePath, - const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false); - - -/* Sort a set of paths topologically under the references relation. - If p refers to q, then p preceeds q in this list. */ -Paths topoSortPaths(StoreAPI & store, const PathSet & paths); - - -/* For now, there is a single global store API object, but we'll - purify that in the future. */ -extern std::shared_ptr<StoreAPI> store; - - /* Factory method: open the Nix database, either through the local or remote implementation. */ -std::shared_ptr<StoreAPI> openStore(bool reserveSpace = true); +ref<Store> openStore(bool reserveSpace = true); /* Display a set of paths in human-readable form (i.e., between quotes @@ -401,12 +433,6 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven = false); -/* Export multiple paths in the format expected by ‘nix-store - --import’. */ -void exportPaths(StoreAPI & store, const Paths & paths, - bool sign, Sink & sink); - - MakeError(SubstError, Error) MakeError(BuildError, Error) /* denotes a permanent build failure */ |