* Caching of expression successors.

8 files changed, 145 insertions, 43 deletions

diff --git a/configure.ac b/configure.ac
index 9b36f90a9b44..e3b0f0c6e04c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -12,7 +12,7 @@ AC_PROG_CXX
 AC_PROG_RANLIB
 
 # Unix shell scripting should die a slow and painful death.
-AC_DEFINE_UNQUOTED(NIX_VALUES_DIR, "$(eval echo $prefix/values)", Nix values directory.)
+AC_DEFINE_UNQUOTED(NIX_STORE_DIR, "$(eval echo $prefix/store)", Nix store directory.)
 AC_DEFINE_UNQUOTED(NIX_STATE_DIR, "$(eval echo $localstatedir/nix)", Nix state directory.)
 AC_DEFINE_UNQUOTED(NIX_LOG_DIR, "$(eval echo $localstatedir/log/nix)", Nix log file directory.)
 
diff --git a/src/Makefile.am b/src/Makefile.am
index 4d8cd4229e06..573e84eb8a46 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -23,5 +23,5 @@ install-data-local:
 #	$(INSTALL) -d $(localstatedir)/nix/prebuilts/imports
 #	$(INSTALL) -d $(localstatedir)/nix/prebuilts/exports
 	$(INSTALL) -d $(localstatedir)/log/nix
-	$(INSTALL) -d $(prefix)/values
+	$(INSTALL) -d $(prefix)/store
 	$(bindir)/nix --init
diff --git a/src/eval.cc b/src/eval.cc
index a1b8db6e0732..4eb222197d38 100644
--- a/src/eval.cc
+++ b/src/eval.cc
@@ -25,7 +25,7 @@ bool pathExists(const string & path)
     res = stat(path.c_str(), &st);
     if (!res) return true;
     if (errno != ENOENT)
-        throw SysError("getting status of " + path);
+        throw SysError(format("getting status of %1%") % path);
     return false;
 }
 
@@ -105,7 +105,7 @@ static void runProgram(const string & program, Environment env)
             throw SysError(format("unable to execute %1%") % program);
             
         } catch (exception & e) {
-            cerr << "build error: " << e.what() << endl;
+            cerr << format("build error: %1%\n") % e.what();
         }
         _exit(1);
 
@@ -199,15 +199,62 @@ struct RStatus
 };
 
 
+static ATerm termFromHash(const Hash & hash)
+{
+    string path = queryFromStore(hash);
+    ATerm t = ATreadFromNamedFile(path.c_str());
+    if (!t) throw Error(format("cannot read aterm %1%") % path);
+    return t;
+}
+
+
+static Hash writeTerm(ATerm t)
+{
+    string path = nixStore + "/tmp.nix"; /* !!! */
+    if (!ATwriteToNamedTextFile(t, path.c_str()))
+        throw Error(format("cannot write aterm %1%") % path);
+    Hash hash = hashPath(path);
+    string path2 = nixStore + "/" + (string) hash + ".nix";
+    if (rename(path.c_str(), path2.c_str()) == -1)
+        throw SysError(format("renaming %1% to %2%") % path % path2);
+    setDB(nixDB, dbRefs, hash, path2);
+    return hash;
+}
+
+
 static FState realise(RStatus & status, FState fs)
 {
     char * s1, * s2, * s3;
     Content content;
-    ATermList refs, ins, outs, bnds;
+    ATermList refs, ins, bnds;
+
+    /* First repeatedly try to substitute $fs$ by any known successors
+       in order to speed up the rewrite process. */
+    {
+        string fsHash, scHash;
+        while (queryDB(nixDB, dbSuccessors, fsHash = hashTerm(fs), scHash)) {
+            debug(format("successor %1% -> %2%") % (string) fsHash % scHash);
+            FState fs2 = termFromHash(parseHash(scHash));
+            if (fs == fs2) {
+                debug(format("successor cycle detected in %1%") % printTerm(fs));
+                break;
+            }
+            fs = fs2;
+        }
+    }
+
+    /* Fall through. */
+
+    if (ATmatch(fs, "Include(<str>)", &s1)) {
+        return realise(status, termFromHash(parseHash(s1)));
+    }
     
-    if (ATmatch(fs, "File(<str>, <term>, [<list>])", &s1, &content, &refs)) {
+    else if (ATmatch(fs, "File(<str>, <term>, [<list>])", &s1, &content, &refs)) {
         string path(s1);
 
+        msg(format("realising atomic path %1%") % path);
+        Nest nest(true);
+
         if (path[0] != '/') throw Error("absolute path expected: " + path);
 
         /* Realise referenced paths. */
@@ -223,9 +270,15 @@ static FState realise(RStatus & status, FState fs)
         Hash hash = parseHash(s1);
 
         /* Normal form. */
-        ATerm nf = ATmake("File(<str>, <term>, <list>)",
+        ATerm nf = ATmake("File(<str>, <term>, <term>)",
             path.c_str(), content, refs2);
 
+        /* Register the normal form. */
+        if (fs != nf) {
+            Hash nfHash = writeTerm(nf);
+            setDB(nixDB, dbSuccessors, hashTerm(fs), nfHash);
+        }
+
         /* Perhaps the path already exists and has the right hash? */
         if (pathExists(path)) {
             if (hash == hashPath(path)) {
@@ -250,6 +303,9 @@ static FState realise(RStatus & status, FState fs)
     {
         string platform(s1), builder(s2), outPath(s3);
 
+        msg(format("realising derivate path %1%") % outPath);
+        Nest nest(true);
+
         checkPlatform(platform);
         
         /* Realise inputs. */
@@ -297,15 +353,13 @@ static FState realise(RStatus & status, FState fs)
            values.cc. */
         setDB(nixDB, dbRefs, outHash, outPath);
 
-#if 0
-        /* Register that targetHash was produced by evaluating
-           sourceHash; i.e., that targetHash is a normal form of
-           sourceHash. !!! this shouldn't be here */
-        setDB(nixDB, dbNFs, sourceHash, targetHash);
-#endif
-
-        return ATmake("File(<str>, Hash(<str>), <list>)",
+        /* Register the normal form of fs. */
+        FState nf = ATmake("File(<str>, Hash(<str>), <term>)",
             outPath.c_str(), ((string) outHash).c_str(), ins2);
+        Hash nfHash = writeTerm(nf);
+        setDB(nixDB, dbSuccessors, hashTerm(fs), nfHash);
+
+        return nf;
     }
 
     throw badTerm("bad file system state expression", fs);
diff --git a/src/globals.cc b/src/globals.cc
index 640e960b1f75..a81c90caa41d 100644
--- a/src/globals.cc
+++ b/src/globals.cc
@@ -3,7 +3,7 @@
 
 
 string dbRefs = "refs";
-string dbNFs = "nfs";
+string dbSuccessors = "successors";
 string dbNetSources = "netsources";
 
 string nixStore = "/UNINIT";
@@ -14,6 +14,6 @@ string nixDB = "/UNINIT";
 void initDB()
 {
     createDB(nixDB, dbRefs);
-    createDB(nixDB, dbNFs);
+    createDB(nixDB, dbSuccessors);
     createDB(nixDB, dbNetSources);
 }
diff --git a/src/globals.hh b/src/globals.hh
index 3cb231ee20cb..8597ae2f89d7 100644
--- a/src/globals.hh
+++ b/src/globals.hh
@@ -14,17 +14,15 @@ using namespace std;
    resolve CHash(hash) content descriptors. */
 extern string dbRefs;
 
-/* dbNFs :: Hash -> Hash
+/* dbSuccessors :: Hash -> Hash
 
-   Each pair (h1, h2) in this mapping records the fact that the normal
-   form of an expression with hash h1 is Hash(h2).
+   Each pair (h1, h2) in this mapping records the fact that a
+   successor of an fstate expression with hash h1 is stored in a file
+   with hash h2.
 
-   TODO: maybe this should be that the normal form of an expression
-   with hash h1 is an expression with hash h2; this would be more
-   general, but would require us to store lots of small expressions in
-   the file system just to support the caching mechanism.
-*/
-extern string dbNFs;
+   Note that a term $y$ is successor of $x$ iff there exists a
+   sequence of rewrite steps that rewrites $x$ into $y$. */
+extern string dbSuccessors;
 
 /* dbNetSources :: Hash -> URL
 
diff --git a/src/nix.cc b/src/nix.cc
index fe9ab453b239..9c9936f4b44a 100644
--- a/src/nix.cc
+++ b/src/nix.cc
@@ -22,7 +22,7 @@ static ArgType argType = atpUnknown;
 
    Operations:
 
-     --evaluate / -e: evaluate values
+     --realise / -r: realise values
      --delete / -d: delete values
      --query / -q: query stored values
      --add: add values
@@ -87,8 +87,8 @@ static void getArgType(Strings & flags)
 }
 
 
-/* Evaluate values. */
-static void opEvaluate(Strings opFlags, Strings opArgs)
+/* Realise values. */
+static void opRealise(Strings opFlags, Strings opArgs)
 {
     getArgType(opFlags);
     if (!opFlags.empty()) throw UsageError("unknown flag");
@@ -101,16 +101,19 @@ static void opEvaluate(Strings opFlags, Strings opArgs)
             hash = parseHash(*it);
         else if (argType == atpName)
             throw Error("not implemented");
-        else if (argType == atpPath)
-            hash = addValue(*it);
-        Expr e = ATmake("Deref(Hash(<str>))", ((string) hash).c_str());
-        cerr << printExpr(evalValue(e)) << endl;
+        else if (argType == atpPath) {
+            string path;
+            addToStore(*it, path, hash);
+        }
+        FState fs = ATmake("Include(<str>)", ((string) hash).c_str());
+        realiseFState(fs);
     }
 }
 
 
 static void opDelete(Strings opFlags, Strings opArgs)
 {
+#if 0
     getArgType(opFlags);
     if (!opFlags.empty()) throw UsageError("unknown flag");
 
@@ -126,6 +129,7 @@ static void opDelete(Strings opFlags, Strings opArgs)
             throw Error("invalid argument type");
         deleteValue(hash);
     }
+#endif
 }
 
 
@@ -138,7 +142,12 @@ static void opAdd(Strings opFlags, Strings opArgs)
 
     for (Strings::iterator it = opArgs.begin();
          it != opArgs.end(); it++)
-        cout << (string) addValue(*it) << endl;
+    {
+        string path;
+        Hash hash;
+        addToStore(*it, path, hash);
+        cout << format("%1% %2%\n") % (string) hash % path;
+    }
 }
 
 
@@ -158,6 +167,7 @@ struct StdoutSink : DumpSink
    output. */
 static void opDump(Strings opFlags, Strings opArgs)
 {
+#if 0
     getArgType(opFlags);
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (opArgs.size() != 1) throw UsageError("only one argument allowed");
@@ -174,6 +184,7 @@ static void opDump(Strings opFlags, Strings opArgs)
         path = arg;
 
     dumpPath(path, sink);
+#endif
 }
 
 
@@ -218,7 +229,7 @@ static void opInit(Strings opFlags, Strings opArgs)
 static void run(int argc, char * * argv)
 {
     /* Setup Nix paths. */
-    nixValues = NIX_VALUES_DIR;
+    nixStore = NIX_STORE_DIR;
     nixLogDir = NIX_LOG_DIR;
     nixDB = (string) NIX_STATE_DIR + "/nixstate.db";
 
@@ -253,8 +264,8 @@ static void run(int argc, char * * argv)
 
         Operation oldOp = op;
 
-        if (arg == "--evaluate" || arg == "-e")
-            op = opEvaluate;
+        if (arg == "--realise" || arg == "-r")
+            op = opRealise;
         else if (arg == "--delete" || arg == "-d")
             op = opDelete;
         else if (arg == "--add")
diff --git a/src/util.cc b/src/util.cc
index a042a65b075c..65ceea938352 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -36,7 +36,7 @@ string absPath(string path, string dir)
         /* !!! canonicalise */
         char resolved[PATH_MAX];
         if (!realpath(path.c_str(), resolved))
-            throw SysError("cannot canonicalise path " + path);
+            throw SysError(format("cannot canonicalise path %1%") % path);
         path = resolved;
     }
     return path;
@@ -46,7 +46,8 @@ string absPath(string path, string dir)
 string dirOf(string path)
 {
     unsigned int pos = path.rfind('/');
-    if (pos == string::npos) throw Error("invalid file name: " + path);
+    if (pos == string::npos)
+        throw Error(format("invalid file name: %1%") % path);
     return string(path, 0, pos);
 }
 
@@ -54,7 +55,8 @@ string dirOf(string path)
 string baseNameOf(string path)
 {
     unsigned int pos = path.rfind('/');
-    if (pos == string::npos) throw Error("invalid file name: " + path);
+    if (pos == string::npos)
+        throw Error(format("invalid file name %1% ") % path);
     return string(path, pos + 1);
 }
 
@@ -63,7 +65,7 @@ void deletePath(string path)
 {
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError("getting attributes of path " + path);
+        throw SysError(format("getting attributes of path %1%") % path);
 
     if (S_ISDIR(st.st_mode)) {
         DIR * dir = opendir(path.c_str());
@@ -79,11 +81,36 @@ void deletePath(string path)
     }
 
     if (remove(path.c_str()) == -1)
-        throw SysError("cannot unlink " + path);
+        throw SysError(format("cannot unlink %1%") % path);
+}
+
+
+static int nestingLevel = 0;
+
+
+Nest::Nest(bool nest)
+{
+    this->nest = nest;
+    if (nest) nestingLevel++;
+}
+
+
+Nest::~Nest()
+{
+    if (nest) nestingLevel--;
+}
+
+
+void msg(const format & f)
+{
+    string spaces;
+    for (int i = 0; i < nestingLevel; i++)
+        spaces += "  ";
+    cerr << format("%1%%2%\n") % spaces % f.str();
 }
 
 
 void debug(const format & f)
 {
-    cerr << format("debug: %1%\n") % f.str();
+    msg(format("debug: %1%") % f.str());
 }
diff --git a/src/util.hh b/src/util.hh
index cf6f7d0c1c65..6242fcb112ae 100644
--- a/src/util.hh
+++ b/src/util.hh
@@ -61,6 +61,18 @@ string baseNameOf(string path);
 void deletePath(string path);
 
 
+/* Messages. */
+
+class Nest
+{
+private:
+    bool nest;
+public:
+    Nest(bool nest);
+    ~Nest();
+};
+
+void msg(const format & f);
 void debug(const format & f);