diff options
| -rw-r--r-- | src/Makefile.am | 3 | ||||
| -rw-r--r-- | src/exec.cc | 123 | ||||
| -rw-r--r-- | src/exec.hh | 18 | ||||
| -rw-r--r-- | src/fix.cc | 74 | ||||
| -rw-r--r-- | src/fstate.cc | 555 | ||||
| -rw-r--r-- | src/fstate.hh | 89 | ||||
| -rw-r--r-- | src/nix.cc | 3 | ||||
| -rw-r--r-- | src/normalise.cc | 265 | ||||
| -rw-r--r-- | src/normalise.hh | 25 | ||||
| -rw-r--r-- | src/normalise.o | bin | 0 -> 888596 bytes | |||
| -rw-r--r-- | src/store.cc | 2 | ||||
| -rw-r--r-- | src/test.cc | 37 | ||||
| -rw-r--r-- | src/util.cc | 2 | ||||
| -rw-r--r-- | substitute.mk | 8 |
14 files changed, 617 insertions, 587 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index 3c590f4c09f8..92aee2f55910 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -20,7 +20,8 @@ test_LDADD = libshared.a libnix.a -ldb_cxx-4 -lATerm noinst_LIBRARIES = libnix.a libshared.a libnix_a_SOURCES = util.cc hash.cc archive.cc md5.c \ - fstate.cc store.cc globals.cc db.cc references.cc + store.cc fstate.cc normalise.cc exec.cc \ + globals.cc db.cc references.cc libshared_a_SOURCES = shared.cc diff --git a/src/exec.cc b/src/exec.cc new file mode 100644 index 000000000000..016dbaeddf73 --- /dev/null +++ b/src/exec.cc @@ -0,0 +1,123 @@ +#include <iostream> + +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/wait.h> +#include <unistd.h> +#include <fcntl.h> + +#include "exec.hh" +#include "util.hh" +#include "globals.hh" + + +class AutoDelete +{ + string path; + bool del; +public: + + AutoDelete(const string & p) : path(p) + { + del = true; + } + + ~AutoDelete() + { + if (del) deletePath(path); + } + + void cancel() + { + del = false; + } +}; + + +/* Run a program. */ +void runProgram(const string & program, Environment env) +{ + /* Create a log file. */ + string logFileName = nixLogDir + "/run.log"; + /* !!! auto-pclose on exit */ + FILE * logFile = popen(("tee -a " + logFileName + " >&2").c_str(), "w"); /* !!! escaping */ + if (!logFile) + throw SysError(format("creating log file `%1%'") % logFileName); + + /* Create a temporary directory where the build will take + place. */ + static int counter = 0; + string tmpDir = (format("/tmp/nix-%1%-%2%") % getpid() % counter++).str(); + + if (mkdir(tmpDir.c_str(), 0777) == -1) + throw SysError(format("creating directory `%1%'") % tmpDir); + + AutoDelete delTmpDir(tmpDir); + + /* Fork a child to build the package. */ + pid_t pid; + switch (pid = fork()) { + + case -1: + throw SysError("unable to fork"); + + case 0: + + try { /* child */ + + if (chdir(tmpDir.c_str()) == -1) + throw SysError(format("changing into to `%1%'") % tmpDir); + + /* Fill in the environment. We don't bother freeing + the strings, since we'll exec or die soon + anyway. */ + const char * env2[env.size() + 1]; + int i = 0; + for (Environment::iterator it = env.begin(); + it != env.end(); it++, i++) + env2[i] = (new string(it->first + "=" + it->second))->c_str(); + env2[i] = 0; + + /* Dup the log handle into stderr. */ + if (dup2(fileno(logFile), STDERR_FILENO) == -1) + throw SysError("cannot pipe standard error into log file"); + + /* Dup stderr to stdin. */ + if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) + throw SysError("cannot dup stderr into stdout"); + + /* Make the program executable. !!! hack. */ + if (chmod(program.c_str(), 0755)) + throw SysError("cannot make program executable"); + + /* Execute the program. This should not return. */ + execle(program.c_str(), baseNameOf(program).c_str(), 0, env2); + + throw SysError(format("unable to execute %1%") % program); + + } catch (exception & e) { + cerr << format("build error: %1%\n") % e.what(); + } + _exit(1); + + } + + /* parent */ + + /* Close the logging pipe. Note that this should not cause + the logger to exit until builder exits (because the latter + has an open file handle to the former). */ + pclose(logFile); + + /* Wait for the child to finish. */ + int status; + if (waitpid(pid, &status, 0) != pid) + throw Error("unable to wait for child"); + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + delTmpDir.cancel(); + throw Error("unable to build package"); + } +} + + diff --git a/src/exec.hh b/src/exec.hh new file mode 100644 index 000000000000..9dc8e0cd029c --- /dev/null +++ b/src/exec.hh @@ -0,0 +1,18 @@ +#ifndef __EXEC_H +#define __EXEC_H + +#include <string> +#include <map> + +using namespace std; + + +/* A Unix environment is a mapping from strings to strings. */ +typedef map<string, string> Environment; + + +/* Run a program. */ +void runProgram(const string & program, Environment env); + + +#endif /* !__EXEC_H */ diff --git a/src/fix.cc b/src/fix.cc index f55074907fbb..cf6d5617aaa7 100644 --- a/src/fix.cc +++ b/src/fix.cc @@ -2,8 +2,7 @@ #include <iostream> #include "globals.hh" -#include "fstate.hh" -#include "store.hh" +#include "normalise.hh" #include "shared.hh" @@ -111,12 +110,11 @@ static Expr evalExpr(Expr e) ATmatch(e, "FSId(<str>)", &s1)) return e; - if (ATgetType(e) == AT_APPL && - ((string) ATgetName(ATgetAFun(e)) == "Slice" || - (string) ATgetName(ATgetAFun(e)) == "Derive")) - { + try { + parseFState(e); return ATmake("FSId(<str>)", - ((string) writeTerm(e, "", 0)).c_str()); + ((string) writeTerm(e, "")).c_str()); + } catch (...) { /* !!! catch parse errors only */ } /* Application. */ @@ -139,10 +137,17 @@ static Expr evalExpr(Expr e) string dstPath; FSId id; addToStore(srcPath, dstPath, id, true); - FState fs = ATmake("Slice([<str>], [(<str>, <str>, [])])", - ((string) id).c_str(), dstPath.c_str(), ((string) id).c_str()); + + SliceElem elem; + elem.path = dstPath; + elem.id = id; + FState fs; + fs.type = FState::fsSlice; + fs.slice.roots.push_back(id); + fs.slice.elems.push_back(elem); + return ATmake("FSId(<str>)", - ((string) writeTerm(fs, "", 0)).c_str()); + ((string) writeTerm(unparseFState(fs), "")).c_str()); } /* Packages are transformed into Derive fstate expressions. */ @@ -160,8 +165,10 @@ static Expr evalExpr(Expr e) } /* Gather information for building the Derive expression. */ - ATermList ins = ATempty, env = ATempty; - string builder, name; + FState fs; + fs.type = FState::fsDerive; + fs.derive.platform = SYSTEM; + string name; bnds = ATempty; for (map<string, ATerm>::iterator it = bndMap.begin(); @@ -169,21 +176,19 @@ static Expr evalExpr(Expr e) { string key = it->first; ATerm value = it->second; - char * id; - if (ATmatch(value, "FSId(<str>)", &id)) { - Strings paths = fstatePaths(parseHash(id), false); + if (ATmatch(value, "FSId(<str>)", &s1)) { + FSId id = parseHash(s1); + Strings paths = fstatePaths(id, false); if (paths.size() != 1) abort(); string path = *(paths.begin()); - ins = ATinsert(ins, ATmake("<str>", id)); - env = ATinsert(env, ATmake("(<str>, <str>)", - key.c_str(), path.c_str())); - if (key == "build") builder = path; + fs.derive.inputs.push_back(id); + fs.derive.env.push_back(StringPair(key, path)); + if (key == "build") fs.derive.builder = path; } else if (ATmatch(value, "<str>", &s1)) { if (key == "name") name = s1; - env = ATinsert(env, - ATmake("(<str>, <str>)", key.c_str(), s1)); + fs.derive.env.push_back(StringPair(key, s1)); } else throw badTerm("invalid package argument", value); @@ -191,31 +196,24 @@ static Expr evalExpr(Expr e) ATmake("(<str>, <term>)", key.c_str(), value)); } - /* Hash the normal form to produce a unique but deterministic - path name for this package. */ - ATerm nf = ATmake("Package(<term>)", ATreverse(bnds)); - FSId outId = hashTerm(nf); - - if (builder == "") - throw badTerm("no builder specified", nf); + if (fs.derive.builder == "") + throw badTerm("no builder specified", e); if (name == "") - throw badTerm("no package name specified", nf); + throw badTerm("no package name specified", e); + /* Hash the fstate-expression with no outputs to produce a + unique but deterministic path name for this package. */ + Hash outId = hashTerm(unparseFState(fs)); string outPath = canonPath(nixStore + "/" + ((string) outId).c_str() + "-" + name); - - env = ATinsert(env, ATmake("(<str>, <str>)", "out", outPath.c_str())); - - /* Construct the result. */ - FState fs = - ATmake("Derive([(<str>, <str>)], <term>, <str>, <str>, <term>)", - outPath.c_str(), ((string) outId).c_str(), - ins, builder.c_str(), SYSTEM, env); + fs.derive.env.push_back(StringPair("out", outPath)); + fs.derive.outputs.push_back(DeriveOutput(outPath, outId)); + debug(format("%1%: %2%") % (string) outId % name); /* Write the resulting term into the Nix store directory. */ return ATmake("FSId(<str>)", - ((string) writeTerm(fs, "-d-" + name, 0)).c_str()); + ((string) writeTerm(unparseFState(fs), "-d-" + name)).c_str()); } /* BaseName primitive function. */ diff --git a/src/fstate.cc b/src/fstate.cc index 31dd17582584..85f8c75cc5bc 100644 --- a/src/fstate.cc +++ b/src/fstate.cc @@ -1,141 +1,6 @@ -#include <map> -#include <iostream> - -#include <sys/types.h> -#include <sys/stat.h> -#include <sys/wait.h> -#include <unistd.h> -#include <fcntl.h> - #include "fstate.hh" #include "globals.hh" #include "store.hh" -#include "db.hh" -#include "references.hh" - - -/* A Unix environment is a mapping from strings to strings. */ -typedef map<string, string> Environment; - - -class AutoDelete -{ - string path; - bool del; -public: - - AutoDelete(const string & p) : path(p) - { - del = true; - } - - ~AutoDelete() - { - if (del) deletePath(path); - } - - void cancel() - { - del = false; - } -}; - - -/* Run a program. */ -static void runProgram(const string & program, Environment env) -{ - /* Create a log file. */ - string logFileName = nixLogDir + "/run.log"; - /* !!! auto-pclose on exit */ - FILE * logFile = popen(("tee -a " + logFileName + " >&2").c_str(), "w"); /* !!! escaping */ - if (!logFile) - throw SysError(format("creating log file `%1%'") % logFileName); - - /* Create a temporary directory where the build will take - place. */ - static int counter = 0; - string tmpDir = (format("/tmp/nix-%1%-%2%") % getpid() % counter++).str(); - - if (mkdir(tmpDir.c_str(), 0777) == -1) - throw SysError(format("creating directory `%1%'") % tmpDir); - - AutoDelete delTmpDir(tmpDir); - - /* Fork a child to build the package. */ - pid_t pid; - switch (pid = fork()) { - - case -1: - throw SysError("unable to fork"); - - case 0: - - try { /* child */ - - if (chdir(tmpDir.c_str()) == -1) - throw SysError(format("changing into to `%1%'") % tmpDir); - - /* Fill in the environment. We don't bother freeing - the strings, since we'll exec or die soon - anyway. */ - const char * env2[env.size() + 1]; - int i = 0; - for (Environment::iterator it = env.begin(); - it != env.end(); it++, i++) - env2[i] = (new string(it->first + "=" + it->second))->c_str(); - env2[i] = 0; - - /* Dup the log handle into stderr. */ - if (dup2(fileno(logFile), STDERR_FILENO) == -1) - throw SysError("cannot pipe standard error into log file"); - - /* Dup stderr to stdin. */ - if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) - throw SysError("cannot dup stderr into stdout"); - - /* Make the program executable. !!! hack. */ - if (chmod(program.c_str(), 0755)) - throw SysError("cannot make program executable"); - - /* Execute the program. This should not return. */ - execle(program.c_str(), baseNameOf(program).c_str(), 0, env2); - - throw SysError(format("unable to execute %1%") % program); - - } catch (exception & e) { - cerr << format("build error: %1%\n") % e.what(); - } - _exit(1); - - } - - /* parent */ - - /* Close the logging pipe. Note that this should not cause - the logger to exit until builder exits (because the latter - has an open file handle to the former). */ - pclose(logFile); - - /* Wait for the child to finish. */ - int status; - if (waitpid(pid, &status, 0) != pid) - throw Error("unable to wait for child"); - - if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { - delTmpDir.cancel(); - throw Error("unable to build package"); - } -} - - -/* Throw an exception if the given platform string is not supported by - the platform we are executing on. */ -static void checkPlatform(const string & platform) -{ - if (platform != thisSystem) - throw Error(format("a `%1%' is required, but I am a `%2%'") - % platform % thisSystem); -} string printTerm(ATerm t) @@ -157,23 +22,16 @@ Hash hashTerm(ATerm t) } -FState hash2fstate(Hash hash) -{ - return ATmake("Include(<str>)", ((string) hash).c_str()); -} - - -ATerm termFromId(const FSId & id, string * p) +ATerm termFromId(const FSId & id) { string path = expandId(id); - if (p) *p = path; ATerm t = ATreadFromNamedFile(path.c_str()); if (!t) throw Error(format("cannot read aterm from `%1%'") % path); return t; } -FSId writeTerm(ATerm t, const string & suffix, string * p) +FSId writeTerm(ATerm t, const string & suffix) { FSId id = hashTerm(t); @@ -182,28 +40,14 @@ FSId writeTerm(ATerm t, const string & suffix, string * p) if (!ATwriteToNamedTextFile(t, path.c_str())) throw Error(format("cannot write aterm %1%") % path); - registerPath(path, id); - if (p) *p = path; +// debug(format("written term %1% = %2%") % (string) id % +// printTerm(t)); + registerPath(path, id); return id; } -void registerSuccessor(const FSId & id1, const FSId & id2) -{ - setDB(nixDB, dbSuccessors, id1, id2); -} - - -static FSId storeSuccessor(const FSId & id1, FState sc, - string * p) -{ - FSId id2 = writeTerm(sc, "-s-" + (string) id1, p); - registerSuccessor(id1, id2); - return id2; -} - - static void parseIds(ATermList ids, FSIds & out) { while (!ATisEmpty(ids)) { @@ -217,17 +61,40 @@ static void parseIds(ATermList ids, FSIds & out) } -static void checkSlice(const Slice & slice); +typedef set<FSId> FSIdSet; + + +static void checkSlice(const Slice & slice) +{ + if (slice.elems.size() == 0) + throw Error("empty slice"); + + FSIdSet decl; + for (SliceElems::const_iterator i = slice.elems.begin(); + i != slice.elems.end(); i++) + decl.insert(i->id); + + for (FSIds::const_iterator i = slice.roots.begin(); + i != slice.roots.end(); i++) + if (decl.find(*i) == decl.end()) + throw Error(format("undefined id: %1%") % (string) *i); + + for (SliceElems::const_iterator i = slice.elems.begin(); + i != slice.elems.end(); i++) + for (FSIds::const_iterator j = i->refs.begin(); + j != i->refs.end(); j++) + if (decl.find(*j) == decl.end()) + throw Error(format("undefined id: %1%") % (string) *j); +} /* Parse a slice. */ -static Slice parseSlice(FState fs) +static bool parseSlice(ATerm t, Slice & slice) { - Slice slice; ATermList roots, elems; - if (!ATmatch(fs, "Slice([<list>], [<list>])", &roots, &elems)) - throw badTerm("not a slice", fs); + if (!ATmatch(t, "Slice([<list>], [<list>])", &roots, &elems)) + return false; parseIds(roots, slice.roots); @@ -246,337 +113,117 @@ static Slice parseSlice(FState fs) } checkSlice(slice); - - return slice; + return true; } -static ATermList unparseIds(const FSIds & ids) +static bool parseDerive(ATerm t, Derive & derive) { - ATermList l = ATempty; - for (FSIds::const_iterator i = ids.begin(); - i != ids.end(); i++) - l = ATinsert(l, - ATmake("<str>", ((string) *i).c_str())); - return ATreverse(l); -} - - -static FState unparseSlice(const Slice & slice) -{ - ATermList roots = unparseIds(slice.roots); - - ATermList elems = ATempty; - for (SliceElems::const_iterator i = slice.elems.begin(); - i != slice.elems.end(); i++) - elems = ATinsert(elems, - ATmake("(<str>, <str>, <term>)", - i->path.c_str(), - ((string) i->id).c_str(), - unparseIds(i->refs))); - - return ATmake("Slice(<term>, <term>)", roots, elems); -} - - -typedef set<FSId> FSIdSet; - - -static Slice normaliseFState2(FSId id, StringSet & usedPaths) -{ - debug(format("normalising fstate")); - Nest nest(true); - - /* Try to substitute $id$ by any known successors in order to - speed up the rewrite process. */ - string idSucc; - while (queryDB(nixDB, dbSuccessors, id, idSucc)) { - debug(format("successor %1% -> %2%") % (string) id % idSucc); - id = parseHash(idSucc); - } - - /* Get the fstate expression. */ - string fsPath; - FState fs = termFromId(id, &fsPath); - - /* Already in normal form (i.e., a slice)? */ - if (ATgetType(fs) == AT_APPL && - (string) ATgetName(ATgetAFun(fs)) == "Slice") - { - usedPaths.insert(fsPath); - return parseSlice(fs); - } - - /* Then we it's a Derive node. */ ATermList outs, ins, bnds; char * builder; char * platform; - if (!ATmatch(fs, "Derive([<list>], [<list>], <str>, <str>, [<list>])", - &outs, &ins, &builder, &platform, &bnds)) - throw badTerm("not a derive", fs); - - /* Right platform? */ - checkPlatform(platform); - - /* Realise inputs (and remember all input paths). */ - FSIds inIds; - parseIds(ins, inIds); - - typedef map<string, SliceElem> ElemMap; - ElemMap inMap; - - for (FSIds::iterator i = inIds.begin(); i != inIds.end(); i++) { - Slice slice = normaliseFState(*i); - realiseSlice(slice); + if (!ATmatch(t, "Derive([<list>], [<list>], <str>, <str>, [<list>])", + &outs, &ins, &builder, &platform, &bnds)) + return false; - for (SliceElems::iterator j = slice.elems.begin(); - j != slice.elems.end(); j++) - inMap[j->path] = *j; + while (!ATisEmpty(outs)) { + char * s1, * s2; + ATerm t = ATgetFirst(outs); + if (!ATmatch(t, "(<str>, <str>)", &s1, &s2)) + throw badTerm("not a derive output", t); + derive.outputs.push_back(DeriveOutput(s1, parseHash(s2))); + outs = ATgetNext(outs); } - Strings inPaths; - for (ElemMap::iterator i = inMap.begin(); i != inMap.end(); i++) - inPaths.push_back(i->second.path); + parseIds(ins, derive.inputs); - /* Build the environment. */ - Environment env; + derive.builder = builder; + derive.platform = platform; + while (!ATisEmpty(bnds)) { char * s1, * s2; ATerm bnd = ATgetFirst(bnds); if (!ATmatch(bnd, "(<str>, <str>)", &s1, &s2)) throw badTerm("tuple of strings expected", bnd); - env[s1] = s2; + derive.env.push_back(StringPair(s1, s2)); bnds = ATgetNext(bnds); } - /* Parse the outputs. */ - typedef map<string, FSId> OutPaths; - OutPaths outPaths; - while (!ATisEmpty(outs)) { - ATerm t = ATgetFirst(outs); - char * s1, * s2; - if (!ATmatch(t, "(<str>, <str>)", &s1, &s2)) - throw badTerm("string expected", t); - outPaths[s1] = parseHash(s2); - inPaths.push_back(s1); - outs = ATgetNext(outs); - } - - /* We can skip running the builder if we can expand all output - paths from their ids. */ - bool fastBuild = false; -#if 0 - for (OutPaths::iterator i = outPaths.begin(); - i != outPaths.end(); i++) - { - try { - expandId(i->second, i->first); - } catch (...) { - fastBuild = false; - break; - } - } -#endif - - if (!fastBuild) { - - /* Check that none of the outputs exist. */ - for (OutPaths::iterator i = outPaths.begin(); - i != outPaths.end(); i++) - if (pathExists(i->first)) - throw Error(format("path `%1%' exists") % i->first); - - /* Run the builder. */ - runProgram(builder, env); - - } else - debug(format("skipping build")); - - Slice slice; - - /* Check whether the output paths were created, and register each - one. */ - FSIdSet used; - for (OutPaths::iterator i = outPaths.begin(); - i != outPaths.end(); i++) - { - string path = i->first; - if (!pathExists(path)) - throw Error(format("path `%1%' does not exist") % path); - registerPath(path, i->second); - slice.roots.push_back(i->second); - - Strings refs = filterReferences(path, inPaths); - - SliceElem elem; - elem.path = path; - elem.id = i->second; - - for (Strings::iterator j = refs.begin(); j != refs.end(); j++) { - ElemMap::iterator k; - OutPaths::iterator l; - if ((k = inMap.find(*j)) != inMap.end()) { - elem.refs.push_back(k->second.id); - used.insert(k->second.id); - for (FSIds::iterator m = k->second.refs.begin(); - m != k->second.refs.end(); m++) - used.insert(*m); - } else if ((l = outPaths.find(*j)) != outPaths.end()) { - elem.refs.push_back(l->second); - used.insert(l->second); - } else - throw Error(format("unknown referenced path `%1%'") % *j); - } - - slice.elems.push_back(elem); - } - - for (ElemMap::iterator i = inMap.begin(); - i != inMap.end(); i++) - { - FSIdSet::iterator j = used.find(i->second.id); - if (j == used.end()) - debug(format("NOT referenced: `%1%'") % i->second.path); - else { - debug(format("referenced: `%1%'") % i->second.path); - slice.elems.push_back(i->second); - } - } - - FState nf = unparseSlice(slice); - debug(printTerm(nf)); - storeSuccessor(id, nf, &fsPath); - usedPaths.insert(fsPath); - - parseSlice(nf); /* check */ - - return slice; + return true; } -Slice normaliseFState(FSId id) +FState parseFState(ATerm t) { - StringSet dummy; - return normaliseFState2(id, dummy); + FState fs; + if (parseSlice(t, fs.slice)) + fs.type = FState::fsSlice; + else if (parseDerive(t, fs.derive)) + fs.type = FState::fsDerive; + else throw badTerm("not an fstate-expression", t); + return fs; } -static void checkSlice(const Slice & slice) +static ATermList unparseIds(const FSIds & ids) { - if (slice.elems.size() == 0) - throw Error("empty slice"); - - FSIdSet decl; - for (SliceElems::const_iterator i = slice.elems.begin(); - i != slice.elems.end(); i++) - decl.insert(i->id); - - for (FSIds::const_iterator i = slice.roots.begin(); - i != slice.roots.end(); i++) - if (decl.find(*i) == decl.end()) - throw Error(format("undefined id: %1%") % (string) *i); - - for (SliceElems::const_iterator i = slice.elems.begin(); - i != slice.elems.end(); i++) - for (FSIds::const_iterator j = i->refs.begin(); - j != i->refs.end(); j++) - if (decl.find(*j) == decl.end()) - throw Error(format("undefined id: %1%") % (string) *j); + ATermList l = ATempty; + for (FSIds::const_iterator i = ids.begin(); + i != ids.end(); i++) + l = ATinsert(l, + ATmake("<str>", ((string) *i).c_str())); + return ATreverse(l); } -void realiseSlice(const Slice & slice) +static ATerm unparseSlice(const Slice & slice) { - debug(format("realising slice")); - Nest nest(true); - - /* Perhaps all paths already contain the right id? */ - - bool missing = false; + ATermList roots = unparseIds(slice.roots); + + ATermList elems = ATempty; for (SliceElems::const_iterator i = slice.elems.begin(); i != slice.elems.end(); i++) - { - SliceElem elem = *i; - string id; - if (!queryDB(nixDB, dbPath2Id, elem.path, id)) { - if (pathExists(elem.path)) - throw Error(format("path `%1%' obstructed") % elem.path); - missing = true; - break; - } - if (parseHash(id) != elem.id) - throw Error(format("path `%1%' obstructed") % elem.path); - } - - if (!missing) { - debug(format("already installed")); - return; - } + elems = ATinsert(elems, + ATmake("(<str>, <str>, <term>)", + i->path.c_str(), + ((string) i->id).c_str(), + unparseIds(i->refs))); - /* For each element, expand its id at its path. */ - for (SliceElems::const_iterator i = slice.elems.begin(); - i != slice.elems.end(); i++) - { - SliceElem elem = *i; - expandId(elem.id, elem.path); - } + return ATmake("Slice(<term>, <term>)", roots, elems); } -Strings fstatePaths(const FSId & id, bool normalise) +static ATerm unparseDerive(const Derive & derive) { - Strings paths; - - FState fs = termFromId(id); - - ATermList outs, ins, bnds; - char * builder; - char * platform; - - if (normalise || - (ATgetType(fs) == AT_APPL && - (string) ATgetName(ATgetAFun(fs)) == "Slice")) - { - Slice slice; - if (normalise) - slice = normaliseFState(id); - else - slice = parseSlice(fs); - - /* !!! fix complexity */ - for (FSIds::const_iterator i = slice.roots.begin(); - i != slice.roots.end(); i++) - for (SliceElems::const_iterator j = slice.elems.begin(); - j != slice.elems.end(); j++) - if (*i == j->id) paths.push_back(j->path); - } - - else if (ATmatch(fs, "Derive([<list>], [<list>], <str>, <str>, [<list>])", - &outs, &ins, &builder, &platform, &bnds)) - { - while (!ATisEmpty(outs)) { - ATerm t = ATgetFirst(outs); - char * s1, * s2; - if (!ATmatch(t, "(<str>, <str>)", &s1, &s2)) - throw badTerm("string expected", t); - paths.push_back(s1); - outs = ATgetNext(outs); - } - } + ATermList outs = ATempty; + for (DeriveOutputs::const_iterator i = derive.outputs.begin(); + i != derive.outputs.end(); i++) + outs = ATinsert(outs, + ATmake("(<str>, <str>)", + i->first.c_str(), ((string) i->second).c_str())); - else throw badTerm("in fstatePaths", fs); + ATermList env = ATempty; + for (StringPairs::const_iterator i = derive.env.begin(); + i != derive.env.end(); i++) + env = ATinsert(env, + ATmake("(<str>, <str>)", + i->first.c_str(), i->second.c_str())); - return paths; + return ATmake("Derive(<term>, <term>, <str>, <str>, <term>)", + ATreverse(outs), + unparseIds(derive.inputs), + derive.builder.c_str(), + derive.platform.c_str(), + ATreverse(env)); } -StringSet fstateRefs(const FSId & id) +ATerm unparseFState(const FState & fs) { - StringSet paths; - Slice slice = normaliseFState2(id, paths); - for (SliceElems::const_iterator i = slice.elems.begin(); - i != slice.elems.end(); i++) - paths.insert(i->path); - return paths; + if (fs.type == FState::fsSlice) + return unparseSlice(fs.slice); + else if (fs.type == FState::fsDerive) + return unparseDerive(fs.derive); + else abort(); } diff --git a/src/fstate.hh b/src/fstate.hh index 0d89e7e360c7..681a8d094180 100644 --- a/src/fstate.hh +++ b/src/fstate.hh @@ -5,55 +5,13 @@ extern "C" { #include <aterm2.h> } -#include "hash.hh" #include "store.hh" -using namespace std; - -/* \section{Abstract syntax of Nix file system state expressions} - - A Nix file system state expression, or FState, describes a - (partial) state of the file system. - - Slice : [Id] * [(Path, Id, [Id])] -> FState - - (update) - Path(path, content, refs) specifies a file object (its full path - and contents), along with all file objects referenced by it (that - is, that it has pointers to). We assume that all files are - self-referential. This prevents us from having to deal with - cycles. - - Derive : [(Path, Id)] * [FStateId] * Path * [(String, String)] -> FState - - (update) - Derive(platform, builder, ins, outs, env) specifies the creation of - new file objects (in paths declared by `outs') by the execution of - a program `builder' on a platform `platform'. This execution takes - place in a file system state given by `ins'. `env' specifies a - mapping of strings to strings. - - A FState expression is in {\em $f$-normal form} if all Derive nodes - have been reduced to File nodes. - - DISCUSSION: the idea is that a Regular/Directory is interchangeable - with its CHash. This would appear to break referential - transparency, e.g., Derive(..., ..., [...CHash(h)...], ...) can - only be reduced in a context were the Regular/Directory equivalent - of Hash(h) is known. However, CHash should be viewed strictly as a - shorthand; that is, when we export an expression containing a - CHash, we should also export the file object referenced by that - CHash. - -*/ - -typedef ATerm FState; -typedef ATerm Content; +/* Abstract syntax of fstate-expressions. */ typedef list<FSId> FSIds; - struct SliceElem { string path; @@ -69,6 +27,27 @@ struct Slice SliceElems elems; }; +typedef pair<string, FSId> DeriveOutput; +typedef pair<string, string> StringPair; +typedef list<DeriveOutput> DeriveOutputs; +typedef list<StringPair> StringPairs; + +struct Derive +{ + DeriveOutputs outputs; + FSIds inputs; + string builder; + string platform; + StringPairs env; +}; + +struct FState +{ + enum { fsSlice, fsDerive } type; + Slice slice; + Derive derive; +}; + /* Return a canonical textual representation of an expression. */ string printTerm(ATerm t); @@ -81,28 +60,16 @@ Error badTerm(const format & f, ATerm t); Hash hashTerm(ATerm t); /* Read an aterm from disk, given its id. */ -ATerm termFromId(const FSId & id, string * p = 0); +ATerm termFromId(const FSId & id); /* Write an aterm to the Nix store directory, and return its hash. */ -FSId writeTerm(ATerm t, const string & suffix, string * p = 0); - -/* Register a successor. */ -void registerSuccessor(const FSId & id1, const FSId & id2); - - -/* Normalise an fstate-expression, that is, return an equivalent - Slice. */ -Slice normaliseFState(FSId id); - -/* Realise a Slice in the file system. */ -void realiseSlice(const Slice & slice); +FSId writeTerm(ATerm t, const string & suffix); -/* Get the list of root (output) paths of the given - fstate-expression. */ -Strings fstatePaths(const FSId & id, bool normalise); +/* Parse an fstate-expression. */ +FState parseFState(ATerm t); -/* Get the list of paths referenced by the given fstate-expression. */ -StringSet fstateRefs(const FSId & id); +/* Parse an fstate-expression. */ +ATerm unparseFState(const FState & fs); #endif /* !__FSTATE_H */ diff --git a/src/nix.cc b/src/nix.cc index 5785cd6b4cc2..f5ca0b4d8793 100644 --- a/src/nix.cc +++ b/src/nix.cc @@ -1,8 +1,7 @@ #include <iostream> #include "globals.hh" -#include "store.hh" -#include "fstate.hh" +#include "normalise.hh" #include "archive.hh" #include "shared.hh" diff --git a/src/normalise.cc b/src/normalise.cc new file mode 100644 index 000000000000..bdeddf08dc1b --- /dev/null +++ b/src/normalise.cc @@ -0,0 +1,265 @@ +#include <map> + +#include "normalise.hh" +#include "references.hh" +#include "db.hh" +#include "exec.hh" +#include "globals.hh" + + +void registerSuccessor(const FSId & id1, const FSId & id2) +{ + setDB(nixDB, dbSuccessors, id1, id2); +} + + +static FSId storeSuccessor(const FSId & id1, ATerm sc) +{ + FSId id2 = writeTerm(sc, "-s-" + (string) id1); + registerSuccessor(id1, id2); + return id2; +} + + +typedef set<FSId> FSIdSet; + + +Slice normaliseFState(FSId id) +{ + debug(format("normalising fstate %1%") % (string) id); + Nest nest(true); + + /* Try to substitute $id$ by any known successors in order to + speed up the rewrite process. */ + string idSucc; + while (queryDB(nixDB, dbSuccessors, id, idSucc)) { + debug(format("successor %1% -> %2%") % (string) id % idSucc); + id = parseHash(idSucc); + } + + /* Get the fstate expression. */ + FState fs = parseFState(termFromId(id)); + + /* It this is a normal form (i.e., a slice) we are done. */ + if (fs.type == FState::fsSlice) return fs.slice; + + /* Otherwise, it's a derivation. */ + + /* Right platform? */ + if (fs.derive.platform != thisSystem) + throw Error(format("a `%1%' is required, but I am a `%2%'") + % fs.derive.platform % thisSystem); + + /* Realise inputs (and remember all input paths). */ + typedef map<string, SliceElem> ElemMap; + + ElemMap inMap; + + for (FSIds::iterator i = fs.derive.inputs.begin(); + i != fs.derive.inputs.end(); i++) { + Slice slice = normaliseFState(*i); + realiseSlice(slice); + + for (SliceElems::iterator j = slice.elems.begin(); + j != slice.elems.end(); j++) + inMap[j->path] = *j; + } + + Strings inPaths; + for (ElemMap::iterator i = inMap.begin(); i != inMap.end(); i++) + inPaths.push_back(i->second.path); + + /* Build the environment. */ + Environment env; + for (StringPairs::iterator i = fs.derive.env.begin(); + i != fs.derive.env.end(); i++) + env[i->first] = i->second; + + /* Parse the outputs. */ + typedef map<string, FSId> OutPaths; + OutPaths outPaths; + for (DeriveOutputs::iterator i = fs.derive.outputs.begin(); + i != fs.derive.outputs.end(); i++) + { + debug(format("building %1% in %2%") % (string) i->second % i->first); + outPaths[i->first] = i->second; + inPaths.push_back(i->first); + } + + /* We can skip running the builder if we can expand all output + paths from their ids. */ + bool fastBuild = false; +#if 0 + for (OutPaths::iterator i = outPaths.begin(); + i != outPaths.end(); i++) + { + try { + expandId(i->second, i->first); + } catch (...) { + fastBuild = false; + break; + } + } +#endif + + if (!fastBuild) { + + /* Check that none of the outputs exist. */ + for (OutPaths::iterator i = outPaths.begin(); + i != outPaths.end(); i++) + if (pathExists(i->first)) + throw Error(format("path `%1%' exists") % i->first); + + /* Run the builder. */ + debug(format("building...")); + runProgram(fs.derive.builder, env); + debug(format("build completed")); + + } else + debug(format("skipping build")); + + /* Check whether the output paths were created, and register each + one. */ + FSIdSet used; + for (OutPaths::iterator i = outPaths.begin(); + i != outPaths.end(); i++) + { + string path = i->first; + if (!pathExists(path)) + throw Error(format("path `%1%' does not exist") % path); + registerPath(path, i->second); + fs.slice.roots.push_back(i->second); + + Strings refs = filterReferences(path, inPaths); + + SliceElem elem; + elem.path = path; + elem.id = i->second; + + for (Strings::iterator j = refs.begin(); j != refs.end(); j++) { + ElemMap::iterator k; + OutPaths::iterator l; + if ((k = inMap.find(*j)) != inMap.end()) { + elem.refs.push_back(k->second.id); + used.insert(k->second.id); + for (FSIds::iterator m = k->second.refs.begin(); + m != k->second.refs.end(); m++) + used.insert(*m); + } else if ((l = outPaths.find(*j)) != outPaths.end()) { + elem.refs.push_back(l->second); + used.insert(l->second); + } else + throw Error(format("unknown referenced path `%1%'") % *j); + } + + fs.slice.elems.push_back(elem); + } + + for (ElemMap::iterator i = inMap.begin(); + i != inMap.end(); i++) + { + FSIdSet::iterator j = used.find(i->second.id); + if (j == used.end()) + debug(format("NOT referenced: `%1%'") % i->second.path); + else { + debug(format("referenced: `%1%'") % i->second.path); + fs.slice.elems.push_back(i->second); + } + } + + fs.type = FState::fsSlice; + ATerm nf = unparseFState(fs); + debug(format("normal form: %1%") % printTerm(nf)); + storeSuccessor(id, nf); + + return fs.slice; +} + + +void realiseSlice(const Slice & slice) +{ + debug(format("realising slice")); + Nest nest(true); + + /* Perhaps all paths already contain the right id? */ + + bool missing = false; + for (SliceElems::const_iterator i = slice.elems.begin(); + i != slice.elems.end(); i++) + { + SliceElem elem = *i; + string id; + if (!queryDB(nixDB, dbPath2Id, elem.path, id)) { + if (pathExists(elem.path)) + throw Error(format("path `%1%' obstructed") % elem.path); + missing = true; + break; + } + if (parseHash(id) != elem.id) + throw Error(format("path `%1%' obstructed") % elem.path); + } + + if (!missing) { + debug(format("already installed")); + return; + } + + /* For each element, expand its id at its path. */ + for (SliceElems::const_iterator i = slice.elems.begin(); + i != slice.elems.end(); i++) + { + SliceElem elem = *i; + debug(format("expanding %1% in %2%") % (string) elem.id % elem.path); + expandId(elem.id, elem.path); + } +} + + +Strings fstatePaths(const FSId & id, bool normalise) +{ + Strings paths; + + FState fs; + + if (normalise) { + fs.slice = normaliseFState(id); + fs.type = FState::fsSlice; + } else + fs = parseFState(termFromId(id)); + + if (fs.type == FState::fsSlice) { + /* !!! fix complexity */ + for (FSIds::const_iterator i = fs.slice.roots.begin(); + i != fs.slice.roots.end(); i++) + for (SliceElems::const_iterator j = fs.slice.elems.begin(); + j != fs.slice.elems.end(); j++) + if (*i == j->id) paths.push_back(j->path); + } + + else if (fs.type == FState::fsDerive) { + for (DeriveOutputs::iterator i = fs.derive.outputs.begin(); + i != fs.derive.outputs.end(); i++) + paths.push_back(i->first); + } + + else abort(); + + return paths; +} + + +StringSet fstateRefs(const FSId & id) +{ + StringSet paths; + Slice slice = normaliseFState(id); + for (SliceElems::const_iterator i = slice.elems.begin(); + i != slice.elems.end(); i++) + paths.insert(i->path); + return paths; +} + + +void findGenerators(const FSIds & ids) +{ + +} diff --git a/src/normalise.hh b/src/normalise.hh new file mode 100644 index 000000000000..85dbca5ef5e0 --- /dev/null +++ b/src/normalise.hh @@ -0,0 +1,25 @@ +#ifndef __NORMALISE_H +#define __NORMALISE_H + +#include "fstate.hh" + + +/* Normalise an fstate-expression, that is, return an equivalent + Slice. */ +Slice normaliseFState(FSId id); + +/* Realise a Slice in the file system. */ +void realiseSlice(const Slice & slice); + +/* Get the list of root (output) paths of the given + fstate-expression. */ +Strings fstatePaths(const FSId & id, bool normalise); + +/* Get the list of paths referenced by the given fstate-expression. */ +StringSet fstateRefs(const FSId & id); + +/* Register a successor. */ +void registerSuccessor(const FSId & id1, const FSId & id2); + + +#endif /* !__NORMALISE_H */ diff --git a/src/normalise.o b/src/normalise.o new file mode 100644 index 000000000000..c23482c54d1c --- /dev/null +++ b/src/normalise.o Binary files differdiff --git a/src/store.cc b/src/store.cc index 864213692ed8..0ce5f26044fb 100644 --- a/src/store.cc +++ b/src/store.cc @@ -7,7 +7,7 @@ #include "globals.hh" #include "db.hh" #include "archive.hh" -#include "fstate.hh" +#include "normalise.hh" struct CopySink : DumpSink diff --git a/src/test.cc b/src/test.cc index 219281c8befc..fa7c93820558 100644 --- a/src/test.cc +++ b/src/test.cc @@ -6,14 +6,14 @@ #include "hash.hh" #include "archive.hh" #include "util.hh" -#include "fstate.hh" -#include "store.hh" +#include "normalise.hh" #include "globals.hh" void realise(FSId id) { - cout << format("realising %1%\n") % (string) id; + debug(format("TEST: realising %1%") % (string) id); + Nest nest(true); Slice slice = normaliseFState(id); realiseSlice(slice); } @@ -117,7 +117,7 @@ void runTests() string builder1fn; addToStore("./test-builder-1.sh", builder1fn, builder1id); - FState fs1 = ATmake( + ATerm fs1 = ATmake( "Slice([<str>], [(<str>, <str>, [])])", ((string) builder1id).c_str(), builder1fn.c_str(), @@ -127,7 +127,7 @@ void runTests() realise(fs1id); realise(fs1id); - FState fs2 = ATmake( + ATerm fs2 = ATmake( "Slice([<str>], [(<str>, <str>, [])])", ((string) builder1id).c_str(), (builder1fn + "_bla").c_str(), @@ -139,7 +139,7 @@ void runTests() string out1id = hashString("foo"); /* !!! bad */ string out1fn = nixStore + "/" + (string) out1id + "-hello.txt"; - FState fs3 = ATmake( + ATerm fs3 = ATmake( "Derive([(<str>, <str>)], [<str>], <str>, <str>, [(\"out\", <str>)])", out1fn.c_str(), ((string) out1id).c_str(), @@ -158,7 +158,7 @@ void runTests() string builder4fn; addToStore("./test-builder-2.sh", builder4fn, builder4id); - FState fs4 = ATmake( + ATerm fs4 = ATmake( "Slice([<str>], [(<str>, <str>, [])])", ((string) builder4id).c_str(), builder4fn.c_str(), @@ -169,7 +169,7 @@ void runTests() string out5id = hashString("bar"); /* !!! bad */ string out5fn = nixStore + "/" + (string) out5id + "-hello2"; - FState fs5 = ATmake( + ATerm fs5 = ATmake( "Derive([(<str>, <str>)], [<str>], <str>, <str>, [(\"out\", <str>), (\"builder\", <str>)])", out5fn.c_str(), ((string) out5id).c_str(), @@ -183,27 +183,6 @@ void runTests() realise(fs5id); realise(fs5id); - - -#if 0 - FState fs2 = ATmake( - "Path(<str>, Hash(<str>), [])", - (builder1fn + "_bla").c_str(), - ((string) builder1h).c_str()); - realise(fs2); - realise(fs2); - - string out1fn = nixStore + "/hello.txt"; - FState fs3 = ATmake( - "Derive(<str>, <str>, [<term>], <str>, [(\"out\", <str>)])", - thisSystem.c_str(), - builder1fn.c_str(), - fs1, - out1fn.c_str(), - out1fn.c_str()); - realise(fs3); -#endif - } diff --git a/src/util.cc b/src/util.cc index 00a3063d6d13..d7c1fe60e15a 100644 --- a/src/util.cc +++ b/src/util.cc @@ -150,7 +150,7 @@ void msg(const format & f) { string spaces; for (int i = 0; i < nestingLevel; i++) - spaces += " "; + spaces += "| "; cerr << format("%1%%2%\n") % spaces % f.str(); } diff --git a/substitute.mk b/substitute.mk new file mode 100644 index 000000000000..af3549253ca9 --- /dev/null +++ b/substitute.mk @@ -0,0 +1,8 @@ +%: %.in Makefile + sed \ + -e s^@prefix\@^$(prefix)^g \ + -e s^@bindir\@^$(bindir)^g \ + -e s^@sysconfdir\@^$(sysconfdir)^g \ + -e s^@localstatedir\@^$(localstatedir)^g \ + < $< > $@ || rm $@ + chmod +x $@ |