LocalStore: Allow the physical and logical store directories to differ

This is primarily to subsume the functionality of the
copy-from-other-stores substituter. For example, in the NixOS
installer, we can now do (assuming we're in the target chroot, and the
Nix store of the installation CD is bind-mounted on /tmp/nix):

  $ nix-build ... --option substituters 'local?state=/tmp/nix/var&real=/tmp/nix/store'

However, unlike copy-from-other-stores, this also allows write access
to such a store. One application might be fetching substitutes for
/nix/store in a situation where the user doesn't have sufficient
privileges to create /nix, e.g.:

  $ NIX_REMOTE="local?state=/home/alice/nix/var&real=/home/alice/nix/store" nix-build ...

5 files changed, 68 insertions, 54 deletions

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index f232fe86a967..3233a8d5cd2a 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1269,6 +1269,9 @@ void DerivationGoal::tryToBuild()
 {
     trace("trying to build");
 
+    if (worker.store.storeDir != worker.store.realStoreDir)
+        throw Error("building with a diverted Nix store is not supported");
+
     /* Check for the possibility that some other goal in this process
        has locked the output since we checked in haveDerivation().
        (It can't happen between here and the lockPaths() call below
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index e5be048d8d81..77d13bbdcb8d 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -370,7 +370,6 @@ struct LocalStore::GCState
     bool gcKeepDerivations;
     unsigned long long bytesInvalidated;
     bool moveToTrash = true;
-    Path trashDir;
     bool shouldDelete;
     GCState(GCResults & results_) : results(results_), bytesInvalidated(0) { }
 };
@@ -407,10 +406,12 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
         invalidatePathChecked(path);
     }
 
+    Path realPath = realStoreDir + "/" + baseNameOf(path);
+
     struct stat st;
-    if (lstat(path.c_str(), &st)) {
+    if (lstat(realPath.c_str(), &st)) {
         if (errno == ENOENT) return;
-        throw SysError(format("getting status of %1%") % path);
+        throw SysError(format("getting status of %1%") % realPath);
     }
 
     printMsg(lvlInfo, format("deleting ‘%1%’") % path);
@@ -427,20 +428,20 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
         // if the path was not valid, need to determine the actual
         // size.
         try {
-            if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
-                throw SysError(format("making ‘%1%’ writable") % path);
-            Path tmp = state.trashDir + "/" + baseNameOf(path);
-            if (rename(path.c_str(), tmp.c_str()))
-                throw SysError(format("unable to rename ‘%1%’ to ‘%2%’") % path % tmp);
+            if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1)
+                throw SysError(format("making ‘%1%’ writable") % realPath);
+            Path tmp = trashDir + "/" + baseNameOf(path);
+            if (rename(realPath.c_str(), tmp.c_str()))
+                throw SysError(format("unable to rename ‘%1%’ to ‘%2%’") % realPath % tmp);
             state.bytesInvalidated += size;
         } catch (SysError & e) {
             if (e.errNo == ENOSPC) {
-                printMsg(lvlInfo, format("note: can't create move ‘%1%’: %2%") % path % e.msg());
-                deleteGarbage(state, path);
+                printMsg(lvlInfo, format("note: can't create move ‘%1%’: %2%") % realPath % e.msg());
+                deleteGarbage(state, realPath);
             }
         }
     } else
-        deleteGarbage(state, path);
+        deleteGarbage(state, realPath);
 
     if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) {
         printMsg(lvlInfo, format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed);
@@ -508,7 +509,8 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
 {
     checkInterrupt();
 
-    if (path == linksDir || path == state.trashDir) return;
+    auto realPath = realStoreDir + "/" + baseNameOf(path);
+    if (realPath == linksDir || realPath == trashDir) return;
 
     Activity act(*logger, lvlDebug, format("considering whether to delete ‘%1%’") % path);
 
@@ -590,7 +592,6 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 {
     GCState state(results);
     state.options = options;
-    state.trashDir = storeDir + "/trash";
     state.gcKeepOutputs = settings.gcKeepOutputs;
     state.gcKeepDerivations = settings.gcKeepDerivations;
 
@@ -639,9 +640,9 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
        that is not reachable from `roots' is garbage. */
 
     if (state.shouldDelete) {
-        if (pathExists(state.trashDir)) deleteGarbage(state, state.trashDir);
+        if (pathExists(trashDir)) deleteGarbage(state, trashDir);
         try {
-            createDirs(state.trashDir);
+            createDirs(trashDir);
         } catch (SysError & e) {
             if (e.errNo == ENOSPC) {
                 printMsg(lvlInfo, format("note: can't create trash directory: %1%") % e.msg());
@@ -671,8 +672,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
         try {
 
-            AutoCloseDir dir = opendir(storeDir.c_str());
-            if (!dir) throw SysError(format("opening directory ‘%1%’") % storeDir);
+            AutoCloseDir dir = opendir(realStoreDir.c_str());
+            if (!dir) throw SysError(format("opening directory ‘%1%’") % realStoreDir);
 
             /* Read the store and immediately delete all paths that
                aren't valid.  When using --max-freed etc., deleting
@@ -725,8 +726,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     fds.clear();
 
     /* Delete the trash directory. */
-    printMsg(lvlInfo, format("deleting ‘%1%’") % state.trashDir);
-    deleteGarbage(state, state.trashDir);
+    printMsg(lvlInfo, format("deleting ‘%1%’") % trashDir);
+    deleteGarbage(state, trashDir);
 
     /* Clean up the links directory. */
     if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) {
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 1ffacc5b035e..33df25c15833 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -38,10 +38,12 @@ namespace nix {
 
 LocalStore::LocalStore(const Params & params)
     : LocalFSStore(params)
+    , realStoreDir(get(params, "real", storeDir))
     , dbDir(get(params, "state", "") != "" ? get(params, "state", "") + "/db" : settings.nixDBPath)
-    , linksDir(storeDir + "/.links")
+    , linksDir(realStoreDir + "/.links")
     , reservedPath(dbDir + "/reserved")
     , schemaPath(dbDir + "/schema")
+    , trashDir(realStoreDir + "/trash")
     , requireSigs(settings.get("signed-binary-caches", std::string("")) != "") // FIXME: rename option
     , publicKeys(getDefaultPublicKeys())
 {
@@ -53,7 +55,7 @@ LocalStore::LocalStore(const Params & params)
     }
 
     /* Create missing state directories if they don't already exist. */
-    createDirs(storeDir);
+    createDirs(realStoreDir);
     makeStoreWritable();
     createDirs(linksDir);
     Path profilesDir = stateDir + "/profiles";
@@ -83,21 +85,21 @@ LocalStore::LocalStore(const Params & params)
                 % settings.buildUsersGroup);
         else {
             struct stat st;
-            if (stat(storeDir.c_str(), &st))
-                throw SysError(format("getting attributes of path ‘%1%’") % storeDir);
+            if (stat(realStoreDir.c_str(), &st))
+                throw SysError(format("getting attributes of path ‘%1%’") % realStoreDir);
 
             if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) {
-                if (chown(storeDir.c_str(), 0, gr->gr_gid) == -1)
-                    throw SysError(format("changing ownership of path ‘%1%’") % storeDir);
-                if (chmod(storeDir.c_str(), perm) == -1)
-                    throw SysError(format("changing permissions on path ‘%1%’") % storeDir);
+                if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1)
+                    throw SysError(format("changing ownership of path ‘%1%’") % realStoreDir);
+                if (chmod(realStoreDir.c_str(), perm) == -1)
+                    throw SysError(format("changing permissions on path ‘%1%’") % realStoreDir);
             }
         }
     }
 
     /* Ensure that the store and its parents are not symlinks. */
     if (getEnv("NIX_IGNORE_SYMLINK_STORE") != "1") {
-        Path path = storeDir;
+        Path path = realStoreDir;
         struct stat st;
         while (path != "/") {
             if (lstat(path.c_str(), &st))
@@ -343,15 +345,15 @@ void LocalStore::makeStoreWritable()
     if (getuid() != 0) return;
     /* Check if /nix/store is on a read-only mount. */
     struct statvfs stat;
-    if (statvfs(storeDir.c_str(), &stat) != 0)
+    if (statvfs(realStoreDir.c_str(), &stat) != 0)
         throw SysError("getting info about the Nix store mount point");
 
     if (stat.f_flag & ST_RDONLY) {
         if (unshare(CLONE_NEWNS) == -1)
             throw SysError("setting up a private mount namespace");
 
-        if (mount(0, storeDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
-            throw SysError(format("remounting %1% writable") % storeDir);
+        if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
+            throw SysError(format("remounting %1% writable") % realStoreDir);
     }
 #endif
 }
@@ -917,23 +919,25 @@ void LocalStore::addToStore(const ValidPathInfo & info, const std::string & nar,
 
         PathLocks outputLock;
 
+        Path realPath = realStoreDir + "/" + baseNameOf(info.path);
+
         /* Lock the output path.  But don't lock if we're being called
            from a build hook (whose parent process already acquired a
            lock on this path). */
         Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS"));
         if (find(locksHeld.begin(), locksHeld.end(), info.path) == locksHeld.end())
-            outputLock.lockPaths({info.path});
+            outputLock.lockPaths({realPath});
 
         if (repair || !isValidPath(info.path)) {
 
-            deletePath(info.path);
+            deletePath(realPath);
 
             StringSource source(nar);
-            restorePath(info.path, source);
+            restorePath(realPath, source);
 
-            canonicalisePathMetaData(info.path, -1);
+            canonicalisePathMetaData(realPath, -1);
 
-            optimisePath(info.path); // FIXME: combine with hashPath()
+            optimisePath(realPath); // FIXME: combine with hashPath()
 
             registerValidPath(info);
         }
@@ -957,19 +961,21 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
         /* The first check above is an optimisation to prevent
            unnecessary lock acquisition. */
 
-        PathLocks outputLock({dstPath});
+        Path realPath = realStoreDir + "/" + baseNameOf(dstPath);
+
+        PathLocks outputLock({realPath});
 
         if (repair || !isValidPath(dstPath)) {
 
-            deletePath(dstPath);
+            deletePath(realPath);
 
             if (recursive) {
                 StringSource source(dump);
-                restorePath(dstPath, source);
+                restorePath(realPath, source);
             } else
-                writeFile(dstPath, dump);
+                writeFile(realPath, dump);
 
-            canonicalisePathMetaData(dstPath, -1);
+            canonicalisePathMetaData(realPath, -1);
 
             /* Register the SHA-256 hash of the NAR serialisation of
                the path in the database.  We may just have computed it
@@ -980,9 +986,9 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
                 hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
                 hash.second = dump.size();
             } else
-                hash = hashPath(htSHA256, dstPath);
+                hash = hashPath(htSHA256, realPath);
 
-            optimisePath(dstPath); // FIXME: combine with hashPath()
+            optimisePath(realPath); // FIXME: combine with hashPath()
 
             ValidPathInfo info;
             info.path = dstPath;
@@ -1026,21 +1032,23 @@ Path LocalStore::addTextToStore(const string & name, const string & s,
 
     if (repair || !isValidPath(dstPath)) {
 
-        PathLocks outputLock({dstPath});
+        Path realPath = realStoreDir + "/" + baseNameOf(dstPath);
+
+        PathLocks outputLock({realPath});
 
         if (repair || !isValidPath(dstPath)) {
 
-            deletePath(dstPath);
+            deletePath(realPath);
 
-            writeFile(dstPath, s);
+            writeFile(realPath, s);
 
-            canonicalisePathMetaData(dstPath, -1);
+            canonicalisePathMetaData(realPath, -1);
 
             StringSink sink;
             dumpString(s, sink);
             auto hash = hashString(htSHA256, *sink.s);
 
-            optimisePath(dstPath);
+            optimisePath(realPath);
 
             ValidPathInfo info;
             info.path = dstPath;
@@ -1067,7 +1075,7 @@ Path LocalStore::createTempDirInStore()
         /* There is a slight possibility that `tmpDir' gets deleted by
            the GC between createTempDir() and addTempRoot(), so repeat
            until `tmpDir' exists. */
-        tmpDir = createTempDir(storeDir);
+        tmpDir = createTempDir(realStoreDir);
         addTempRoot(tmpDir);
     } while (!pathExists(tmpDir));
     return tmpDir;
@@ -1107,7 +1115,7 @@ bool LocalStore::verifyStore(bool checkContents, bool repair)
     AutoCloseFD fdGCLock = openGCLock(ltWrite);
 
     PathSet store;
-    for (auto & i : readDirectory(storeDir)) store.insert(i.name);
+    for (auto & i : readDirectory(realStoreDir)) store.insert(i.name);
 
     /* Check whether all valid paths actually exist. */
     printMsg(lvlInfo, "checking path existence...");
@@ -1271,7 +1279,7 @@ void LocalStore::upgradeStore7()
 {
     if (getuid() != 0) return;
     printMsg(lvlError, "removing immutable bits from the Nix store (this may take a while)...");
-    makeMutable(storeDir);
+    makeMutable(realStoreDir);
 }
 
 #else
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 5166c04e5319..3a2568ec9479 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -73,10 +73,12 @@ private:
 
     Sync<State, std::recursive_mutex> _state;
 
+    const Path realStoreDir;
     const Path dbDir;
     const Path linksDir;
     const Path reservedPath;
     const Path schemaPath;
+    const Path trashDir;
 
     bool requireSigs;
 
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 33f002f9dbb1..927478121244 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -176,7 +176,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
     /* Make the containing directory writable, but only if it's not
        the store itself (we don't want or need to mess with its
        permissions). */
-    bool mustToggle = !isStorePath(path);
+    bool mustToggle = dirOf(path) != realStoreDir;
     if (mustToggle) makeWritable(dirOf(path));
 
     /* When we're done, make the directory read-only again and reset
@@ -184,7 +184,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
     MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
 
     Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
-        % storeDir % getpid() % rand()).str();
+        % realStoreDir % getpid() % rand()).str();
 
     if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
         if (errno == EMLINK) {
@@ -229,7 +229,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
         addTempRoot(i);
         if (!isValidPath(i)) continue; /* path was GC'ed, probably */
         Activity act(*logger, lvlChatty, format("hashing files in ‘%1%’") % i);
-        optimisePath_(stats, i, inodeHash);
+        optimisePath_(stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
     }
 }