Automatically optimise the Nix store when a new path is added

Auto-optimisation is enabled by default.  It can be turned off by
setting auto-optimise-store to false in nix.conf.

5 files changed, 45 insertions, 10 deletions

diff --git a/doc/manual/conf-file.xml b/doc/manual/conf-file.xml
index 1b19e56b5714..c095a001c169 100644
--- a/doc/manual/conf-file.xml
+++ b/doc/manual/conf-file.xml
@@ -337,7 +337,20 @@ build-use-chroot = /dev /proc /bin</programlisting>
     <literal>true</literal>.</para></listitem>
 
   </varlistentry>
-    
+
+  
+  <varlistentry><term><literal>auto-optimise-store</literal></term>
+
+    <listitem><para>If set to <literal>true</literal> (the default),
+    Nix automatically detects files in the store that have identical
+    contents, and replaces them with hard links to a single copy.
+    This saves disk space.  If set to <literal>false</literal>, you
+    can still run <command>nix-store --optimise</command> to get rid
+    of duplicate files.</para></listitem>
+
+  </varlistentry>
+
+
 </variablelist>
 
 </para>
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 26268f6ddb7d..a3bde3462364 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2093,6 +2093,8 @@ void DerivationGoal::computeClosure()
                 if (allowed.find(*i) == allowed.end())
                     throw BuildError(format("output is not allowed to refer to path `%1%'") % *i);
         }
+
+        worker.store.optimisePath(path); // FIXME: combine with scanForReferences()
     }
 
     /* Register each output path as valid, and register the sets of
@@ -2546,6 +2548,8 @@ void SubstitutionGoal::finished()
 
     HashResult hash = hashPath(htSHA256, storePath);
     
+    worker.store.optimisePath(storePath); // FIXME: combine with hashPath()
+    
     ValidPathInfo info2;
     info2.path = storePath;
     info2.hash = hash.first;
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index e009191b6c8b..05b2b9c6e542 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -209,6 +209,7 @@ LocalStore::LocalStore(bool reserveSpace)
 
     /* Create missing state directories if they don't already exist. */
     createDirs(nixStore);
+    createDirs(linksDir = nixStore + "/.links");
     Path profilesDir = nixStateDir + "/profiles";
     createDirs(nixStateDir + "/profiles");
     createDirs(nixStateDir + "/temproots");
@@ -1116,6 +1117,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
                 hash.second = dump.size();
             } else
                 hash = hashPath(htSHA256, dstPath);
+
+            optimisePath(dstPath); // FIXME: combine with hashPath()
             
             ValidPathInfo info;
             info.path = dstPath;
@@ -1170,6 +1173,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s,
             canonicalisePathMetaData(dstPath);
 
             HashResult hash = hashPath(htSHA256, dstPath);
+
+            optimisePath(dstPath);
             
             ValidPathInfo info;
             info.path = dstPath;
@@ -1405,6 +1410,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
             /* !!! if we were clever, we could prevent the hashPath()
                here. */
             HashResult hash = hashPath(htSHA256, dstPath);
+
+            optimisePath(dstPath); // FIXME: combine with hashPath()
             
             ValidPathInfo info;
             info.path = dstPath;
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 1bb47fb3bad5..7d30a2d408ae 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -85,6 +85,8 @@ private:
 
     typedef std::map<Path, RunningSubstituter> RunningSubstituters;
     RunningSubstituters runningSubstituters;
+
+    Path linksDir;
     
 public:
 
@@ -169,6 +171,9 @@ public:
        files with the same contents. */
     void optimiseStore(OptimiseStats & stats);
 
+    /* Optimise a single store path. */
+    void optimisePath(const Path & path);
+    
     /* Check the integrity of the Nix store. */
     void verifyStore(bool checkContents);
 
@@ -267,6 +272,8 @@ private:
     Path importPath(bool requireSignature, Source & source);
     
     void checkDerivationOutputs(const Path & drvPath, const Derivation & drv);
+
+    void optimisePath_(OptimiseStats & stats, const Path & path);
 };
 
 
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 0893db9d3130..a7aa14fb49ab 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -49,10 +49,7 @@ struct MakeImmutable
 };
 
 
-const string linksDir = ".links";
-
-
-static void hashAndLink(OptimiseStats & stats, const Path & path)
+void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
 {
     struct stat st;
     if (lstat(path.c_str(), &st))
@@ -61,7 +58,7 @@ static void hashAndLink(OptimiseStats & stats, const Path & path)
     if (S_ISDIR(st.st_mode)) {
         Strings names = readDirectory(path);
 	foreach (Strings::iterator, i, names)
-	    hashAndLink(stats, path + "/" + *i);
+	    optimisePath_(stats, path + "/" + *i);
         return;
     }
     
@@ -91,7 +88,7 @@ static void hashAndLink(OptimiseStats & stats, const Path & path)
     printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));
 
     /* Check if this is a known hash. */
-    Path linkPath = nixStore + "/" + linksDir + "/" + printHash32(hash);
+    Path linkPath = linksDir + "/" + printHash32(hash);
 
     if (!pathExists(linkPath)) {
         /* Nope, create a hard link in the links directory. */
@@ -177,15 +174,22 @@ static void hashAndLink(OptimiseStats & stats, const Path & path)
 
 void LocalStore::optimiseStore(OptimiseStats & stats)
 {
-    createDirs(nixStore + "/" + linksDir);
-
     PathSet paths = queryValidPaths();
 
     foreach (PathSet::iterator, i, paths) {
         addTempRoot(*i);
         if (!isValidPath(*i)) continue; /* path was GC'ed, probably */
         startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i);
-        hashAndLink(stats, *i);
+        optimisePath_(stats, *i);
+    }
+}
+
+
+void LocalStore::optimisePath(const Path & path)
+{
+    if (queryBoolSetting("auto-optimise-store", true)) {
+        OptimiseStats stats;
+        optimisePath_(stats, path);
     }
 }