Merge branch 'master' into no-manifests

13 files changed, 199 insertions, 160 deletions

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 4bef6f02dc00..c57a63db69dd 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1533,7 +1533,7 @@ void DerivationGoal::startBuilder()
 
     /* Create a temporary directory where the build will take
        place. */
-    tmpDir = createTempDir("", "nix-build-" + baseNameOf(drvPath), false, false);
+    tmpDir = createTempDir("", "nix-build-" + baseNameOf(drvPath), false, false, 0700);
 
     /* For convenience, set an environment pointing to the top build
        directory. */
@@ -2099,6 +2099,8 @@ void DerivationGoal::computeClosure()
                 if (allowed.find(*i) == allowed.end())
                     throw BuildError(format("output is not allowed to refer to path `%1%'") % *i);
         }
+
+        worker.store.optimisePath(path); // FIXME: combine with scanForReferences()
     }
 
     /* Register each output path as valid, and register the sets of
@@ -2182,6 +2184,7 @@ void DerivationGoal::deleteTmpDir(bool force)
                 % drvPath % tmpDir);
             if (buildUser.enabled() && !amPrivileged())
                 getOwnership(tmpDir);
+            chmod(tmpDir.c_str(), 0755);
         }
         else
             deletePathWrapped(tmpDir);
@@ -2562,6 +2565,8 @@ void SubstitutionGoal::finished()
 
     HashResult hash = hashPath(htSHA256, storePath);
     
+    worker.store.optimisePath(storePath); // FIXME: combine with hashPath()
+    
     ValidPathInfo info2;
     info2.path = storePath;
     info2.hash = hash.first;
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 27e471d885cc..8f22b4afa6a1 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -1,5 +1,4 @@
-#ifndef __DERIVATIONS_H
-#define __DERIVATIONS_H
+#pragma once
 
 #include <map>
 
@@ -81,6 +80,3 @@ typedef std::map<Path, Hash> DrvHashes;
 extern DrvHashes drvHashes;
 
 }
-
-
-#endif /* !__DERIVATIONS_H */
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index f6ed7dd2264e..874efe4d32d9 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -436,6 +436,8 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path)
 {
     checkInterrupt();
 
+    if (path == linksDir) return true;
+
     struct stat st;
     if (lstat(path.c_str(), &st)) {
         if (errno == ENOENT) return true;
@@ -569,6 +571,37 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path)
 }
 
 
+/* Unlink all files in /nix/store/.links that have a link count of 1,
+   which indicates that there are no other links and so they can be
+   safely deleted.  FIXME: race condition with optimisePath(): we
+   might see a link count of 1 just before optimisePath() increases
+   the link count. */
+void LocalStore::removeUnusedLinks()
+{
+    AutoCloseDir dir = opendir(linksDir.c_str());
+    if (!dir) throw SysError(format("opening directory `%1%'") % linksDir);
+
+    struct dirent * dirent;
+    while (errno = 0, dirent = readdir(dir)) {
+        checkInterrupt();
+        string name = dirent->d_name;
+        if (name == "." || name == "..") continue;
+        Path path = linksDir + "/" + name;
+
+        struct stat st;
+        if (lstat(path.c_str(), &st) == -1)
+            throw SysError(format("statting `%1%'") % path);
+
+        if (st.st_nlink != 1) continue;
+
+        printMsg(lvlTalkative, format("deleting unused link `%1%'") % path);
+
+        if (unlink(path.c_str()) == -1)
+            throw SysError(format("deleting `%1%'") % path);
+    }
+}
+
+
 void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 {
     GCState state(results);
@@ -682,6 +715,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
        released. */
     foreach (PathSet::iterator, i, state.invalidated)
         deleteGarbage(state, *i);
+
+    /* Clean up the links directory. */
+    printMsg(lvlError, format("deleting unused links..."));
+    removeUnusedLinks();
 }
 
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 12a9b9ca15c0..1c0877a5e1e9 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -1,5 +1,4 @@
-#ifndef __GLOBALS_H
-#define __GLOBALS_H
+#pragma once
 
 #include "types.hh"
 
@@ -118,6 +117,3 @@ void setDefaultsFromEnvironment();
 
 
 }
-
-
-#endif /* !__GLOBALS_H */
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index aa21774df4a7..ebfcc946716a 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -12,6 +12,7 @@
 
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/time.h>
 #include <unistd.h>
 #include <utime.h>
 #include <fcntl.h>
@@ -208,6 +209,7 @@ LocalStore::LocalStore(bool reserveSpace)
 
     /* Create missing state directories if they don't already exist. */
     createDirs(nixStore);
+    createDirs(linksDir = nixStore + "/.links");
     Path profilesDir = nixStateDir + "/profiles";
     createDirs(nixStateDir + "/profiles");
     createDirs(nixStateDir + "/temproots");
@@ -444,7 +446,7 @@ void canonicalisePathMetaData(const Path & path, bool recurse)
             throw SysError(format("changing owner of `%1%' to %2%")
                 % path % geteuid());
     }
-    
+
     if (!S_ISLNK(st.st_mode)) {
 
         /* Mask out all type related bits. */
@@ -458,14 +460,20 @@ void canonicalisePathMetaData(const Path & path, bool recurse)
                 throw SysError(format("changing mode of `%1%' to %2$o") % path % mode);
         }
 
-        if (st.st_mtime != mtimeStore) {
-            struct utimbuf utimbuf;
-            utimbuf.actime = st.st_atime;
-            utimbuf.modtime = mtimeStore;
-            if (utime(path.c_str(), &utimbuf) == -1) 
-                throw SysError(format("changing modification time of `%1%'") % path);
-        }
-
+    }
+    
+    if (st.st_mtime != mtimeStore) {
+        struct timeval times[2];
+        times[0].tv_sec = st.st_atime;
+        times[0].tv_usec = 0;
+        times[1].tv_sec = mtimeStore;
+        times[1].tv_usec = 0;
+#if HAVE_LUTIMES
+        if (lutimes(path.c_str(), times) == -1)
+#else
+        if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)
+#endif                
+            throw SysError(format("changing modification time of `%1%'") % path);
     }
 
     if (recurse && S_ISDIR(st.st_mode)) {
@@ -1134,6 +1142,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
                 hash.second = dump.size();
             } else
                 hash = hashPath(htSHA256, dstPath);
+
+            optimisePath(dstPath); // FIXME: combine with hashPath()
             
             ValidPathInfo info;
             info.path = dstPath;
@@ -1188,6 +1198,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s,
             canonicalisePathMetaData(dstPath);
 
             HashResult hash = hashPath(htSHA256, dstPath);
+
+            optimisePath(dstPath);
             
             ValidPathInfo info;
             info.path = dstPath;
@@ -1423,6 +1435,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
             /* !!! if we were clever, we could prevent the hashPath()
                here. */
             HashResult hash = hashPath(htSHA256, dstPath);
+
+            optimisePath(dstPath); // FIXME: combine with hashPath()
             
             ValidPathInfo info;
             info.path = dstPath;
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 07d8198ecaa5..15dff1d02052 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -1,5 +1,4 @@
-#ifndef __LOCAL_STORE_H
-#define __LOCAL_STORE_H
+#pragma once
 
 #include <string>
 
@@ -86,6 +85,8 @@ private:
 
     typedef std::map<Path, RunningSubstituter> RunningSubstituters;
     RunningSubstituters runningSubstituters;
+
+    Path linksDir;
     
 public:
 
@@ -168,8 +169,11 @@ public:
 
     /* Optimise the disk space usage of the Nix store by hard-linking
        files with the same contents. */
-    void optimiseStore(bool dryRun, OptimiseStats & stats);
+    void optimiseStore(OptimiseStats & stats);
 
+    /* Optimise a single store path. */
+    void optimisePath(const Path & path);
+    
     /* Check the integrity of the Nix store. */
     void verifyStore(bool checkContents);
 
@@ -260,6 +264,8 @@ private:
         
     int openGCLock(LockType lockType);
     
+    void removeUnusedLinks();
+
     void startSubstituter(const Path & substituter,
         RunningSubstituter & runningSubstituter);
 
@@ -268,6 +274,8 @@ private:
     Path importPath(bool requireSignature, Source & source);
     
     void checkDerivationOutputs(const Path & drvPath, const Derivation & drv);
+
+    void optimisePath_(OptimiseStats & stats, const Path & path);
 };
 
 
@@ -302,6 +310,3 @@ void deletePathWrapped(const Path & path,
 void deletePathWrapped(const Path & path);
  
 }
-
-
-#endif /* !__LOCAL_STORE_H */
diff --git a/src/libstore/misc.hh b/src/libstore/misc.hh
index 850c12af4ebc..fe0bbdd799d7 100644
--- a/src/libstore/misc.hh
+++ b/src/libstore/misc.hh
@@ -1,5 +1,4 @@
-#ifndef __MISC_H
-#define __MISC_H
+#pragma once
 
 #include "derivations.hh"
 
@@ -35,6 +34,3 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
 
 
 }
-
-
-#endif /* !__MISC_H */
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index a486e66ef59e..84a72604bba2 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -1,6 +1,7 @@
 #include "util.hh"
 #include "local-store.hh"
 #include "immutable.hh"
+#include "globals.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -12,9 +13,6 @@
 namespace nix {
 
 
-typedef std::map<Hash, std::pair<Path, ino_t> > HashToPath;
-
-
 static void makeWritable(const Path & path)
 {
     struct stat st;
@@ -51,140 +49,152 @@ struct MakeImmutable
 };
 
 
-static void hashAndLink(bool dryRun, HashToPath & hashToPath,
-    OptimiseStats & stats, const Path & path)
+void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
 {
     struct stat st;
     if (lstat(path.c_str(), &st))
 	throw SysError(format("getting attributes of path `%1%'") % path);
 
+    if (S_ISDIR(st.st_mode)) {
+        Strings names = readDirectory(path);
+	foreach (Strings::iterator, i, names)
+	    optimisePath_(stats, path + "/" + *i);
+        return;
+    }
+    
+    /* We can hard link regular files and maybe symlinks. */
+    if (!S_ISREG(st.st_mode)
+#if CAN_LINK_SYMLINK
+        x
+        && !S_ISLNK(st.st_mode)
+#endif
+        ) return;
+        
     /* Sometimes SNAFUs can cause files in the Nix store to be
        modified, in particular when running programs as root under
        NixOS (example: $fontconfig/var/cache being modified).  Skip
-       those files. */
+       those files.  FIXME: check the modification time. */
     if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
         printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path);
         return;
     }
 
-    /* We can hard link regular files and symlinks. */
-    if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) {
+    /* Hash the file.  Note that hashPath() returns the hash over the
+       NAR serialisation, which includes the execute bit on the file.
+       Thus, executable and non-executable files with the same
+       contents *won't* be linked (which is good because otherwise the
+       permissions would be screwed up).
 
-        /* Hash the file.  Note that hashPath() returns the hash over
-           the NAR serialisation, which includes the execute bit on
-           the file.  Thus, executable and non-executable files with
-           the same contents *won't* be linked (which is good because
-           otherwise the permissions would be screwed up).
+       Also note that if `path' is a symlink, then we're hashing the
+       contents of the symlink (i.e. the result of readlink()), not
+       the contents of the target (which may not even exist). */
+    Hash hash = hashPath(htSHA256, path).first;
+    stats.totalFiles++;
+    printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));
 
-           Also note that if `path' is a symlink, then we're hashing
-           the contents of the symlink (i.e. the result of
-           readlink()), not the contents of the target (which may not
-           even exist). */
-        Hash hash = hashPath(htSHA256, path).first;
-        stats.totalFiles++;
-        printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));
+    /* Check if this is a known hash. */
+    Path linkPath = linksDir + "/" + printHash32(hash);
 
-        std::pair<Path, ino_t> prevPath = hashToPath[hash];
-        
-        if (prevPath.first == "") {
-            hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);
-            return;
-        }
-            
-        /* Yes!  We've seen a file with the same contents.  Replace
-           the current file with a hard link to that file. */
-        stats.sameContents++;
-        if (prevPath.second == st.st_ino) {
-            printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % prevPath.first);
-            return;
-        }
+    if (!pathExists(linkPath)) {
+        /* Nope, create a hard link in the links directory. */
+        makeMutable(path);
+        MakeImmutable mk1(path);
+
+        if (link(path.c_str(), linkPath.c_str()) == -1)
+            throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path);
+
+        return;
+    }
+
+    /* Yes!  We've seen a file with the same contents.  Replace the
+       current file with a hard link to that file. */
+    struct stat stLink;
+    if (lstat(linkPath.c_str(), &stLink))
+	throw SysError(format("getting attributes of path `%1%'") % linkPath);
+    
+    stats.sameContents++;
+    if (st.st_ino == stLink.st_ino) {
+        printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % linkPath);
+        return;
+    }
         
-        if (!dryRun) {
-            
-            printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % prevPath.first);
+    printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % linkPath);
 
-            Path tempLink = (format("%1%.tmp-%2%-%3%")
-                % path % getpid() % rand()).str();
+    Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
+        % nixStore % getpid() % rand()).str();
 
-            /* Make the containing directory writable, but only if
-               it's not the store itself (we don't want or need to
-               mess with its permissions). */
-            bool mustToggle = !isStorePath(path);
-            if (mustToggle) makeWritable(dirOf(path));
+    /* Make the containing directory writable, but only if it's not
+       the store itself (we don't want or need to mess with its
+       permissions). */
+    bool mustToggle = !isStorePath(path);
+    if (mustToggle) makeWritable(dirOf(path));
             
-            /* When we're done, make the directory read-only again and
-               reset its timestamp back to 0. */
-            MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
-
-            /* If ‘prevPath’ is immutable, we can't create hard links
-               to it, so make it mutable first (and make it immutable
-               again when we're done).  We also have to make ‘path’
-               mutable, otherwise rename() will fail to delete it. */
-            makeMutable(prevPath.first);
-            MakeImmutable mk1(prevPath.first);
-            
-            makeMutable(path);
-            MakeImmutable mk2(path);
-
-            if (link(prevPath.first.c_str(), tempLink.c_str()) == -1) {
-                if (errno == EMLINK) {
-                    /* Too many links to the same file (>= 32000 on
-                       most file systems).  This is likely to happen
-                       with empty files.  Just start over, creating
-                       links to the current file. */
-                    printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first);
-                    hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);
-                    return;
-                }
-                throw SysError(format("cannot link `%1%' to `%2%'")
-                    % tempLink % prevPath.first);
-            }
-
-            /* Atomically replace the old file with the new hard link. */
-            if (rename(tempLink.c_str(), path.c_str()) == -1) {
-                if (errno == EMLINK) {
-                    /* Some filesystems generate too many links on the
-                       rename, rather than on the original link.
-                       (Probably it temporarily increases the st_nlink
-                       field before decreasing it again.) */
-                    printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first);
-                    hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);
-
-                    /* Unlink the temp link. */
-                    if (unlink(tempLink.c_str()) == -1)
-                        printMsg(lvlError, format("unable to unlink `%1%'") % tempLink);
-                    return;
-                }
-                throw SysError(format("cannot rename `%1%' to `%2%'")
-                    % tempLink % path);
-            }
-        } else
-            printMsg(lvlTalkative, format("would link `%1%' to `%2%'") % path % prevPath.first);
-        
-        stats.filesLinked++;
-        stats.bytesFreed += st.st_size;
-        stats.blocksFreed += st.st_blocks;
+    /* When we're done, make the directory read-only again and reset
+       its timestamp back to 0. */
+    MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
+
+    /* If ‘linkPath’ is immutable, we can't create hard links to it,
+       so make it mutable first (and make it immutable again when
+       we're done).  We also have to make ‘path’ mutable, otherwise
+       rename() will fail to delete it. */
+    makeMutable(linkPath);
+    MakeImmutable mk1(linkPath);
+
+    makeMutable(path);
+    MakeImmutable mk2(path);
+
+    if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
+        if (errno == EMLINK) {
+            /* Too many links to the same file (>= 32000 on most file
+               systems).  This is likely to happen with empty files.
+               Just shrug and ignore. */
+            printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
+            return;
+        }
+        throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath);
     }
 
-    if (S_ISDIR(st.st_mode)) {
-        Strings names = readDirectory(path);
-	foreach (Strings::iterator, i, names)
-	    hashAndLink(dryRun, hashToPath, stats, path + "/" + *i);
+    /* Atomically replace the old file with the new hard link. */
+    if (rename(tempLink.c_str(), path.c_str()) == -1) {
+        if (errno == EMLINK) {
+            /* Some filesystems generate too many links on the rename,
+               rather than on the original link.  (Probably it
+               temporarily increases the st_nlink field before
+               decreasing it again.) */
+            printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
+
+            /* Unlink the temp link. */
+            if (unlink(tempLink.c_str()) == -1)
+                printMsg(lvlError, format("unable to unlink `%1%'") % tempLink);
+            return;
+        }
+        throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path);
     }
+
+    stats.filesLinked++;
+    stats.bytesFreed += st.st_size;
+    stats.blocksFreed += st.st_blocks;
 }
 
 
-void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats)
+void LocalStore::optimiseStore(OptimiseStats & stats)
 {
-    HashToPath hashToPath;
-
     PathSet paths = queryAllValidPaths();
 
     foreach (PathSet::iterator, i, paths) {
         addTempRoot(*i);
         if (!isValidPath(*i)) continue; /* path was GC'ed, probably */
         startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i);
-        hashAndLink(dryRun, hashToPath, stats, *i);
+        optimisePath_(stats, *i);
+    }
+}
+
+
+void LocalStore::optimisePath(const Path & path)
+{
+    if (queryBoolSetting("auto-optimise-store", true)) {
+        OptimiseStats stats;
+        optimisePath_(stats, path);
     }
 }
 
diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh
index 57ca1584a6d6..8a6b1450da2a 100644
--- a/src/libstore/pathlocks.hh
+++ b/src/libstore/pathlocks.hh
@@ -1,5 +1,4 @@
-#ifndef __PATHLOCKS_H
-#define __PATHLOCKS_H
+#pragma once
 
 #include "types.hh"
 
@@ -44,6 +43,3 @@ bool pathIsLockedByMe(const Path & path);
 
 
 }
-
-
-#endif /* !__PATHLOCKS_H */
diff --git a/src/libstore/references.hh b/src/libstore/references.hh
index 158c08a77646..013809d122f3 100644
--- a/src/libstore/references.hh
+++ b/src/libstore/references.hh
@@ -1,5 +1,4 @@
-#ifndef __REFERENCES_H
-#define __REFERENCES_H
+#pragma once
 
 #include "types.hh"
 #include "hash.hh"
@@ -10,5 +9,3 @@ PathSet scanForReferences(const Path & path, const PathSet & refs,
     HashResult & hash);
     
 }
-
-#endif /* !__REFERENCES_H */
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 68db0640ae70..ae4c48dad6ab 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -1,5 +1,4 @@
-#ifndef __REMOTE_STORE_H
-#define __REMOTE_STORE_H
+#pragma once
 
 #include <string>
 
@@ -103,6 +102,3 @@ private:
 
 
 }
-
-
-#endif /* !__REMOTE_STORE_H */
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 5d8c09f5a526..324d802dc450 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -1,5 +1,4 @@
-#ifndef __STOREAPI_H
-#define __STOREAPI_H
+#pragma once
 
 #include "hash.hh"
 #include "serialise.hh"
@@ -362,6 +361,3 @@ MakeError(BuildError, Error) /* denotes a permanent build failure */
 
 
 }
-
-
-#endif /* !__STOREAPI_H */
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index d4ac0ea16fed..9677a46c2896 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -1,6 +1,4 @@
-#ifndef __WORKER_PROTOCOL_H
-#define __WORKER_PROTOCOL_H
-
+#pragma once
 
 namespace nix {
 
@@ -67,6 +65,3 @@ template<class T> T readStorePaths(Source & from);
 
     
 }
-
-
-#endif /* !__WORKER_PROTOCOL_H */