From 9f0f020929c9e093405cc6193d2f227cab763912 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Nov 2003 10:55:27 +0000 Subject: * libnix -> libstore. --- src/Makefile.am | 2 +- src/fix-ng/Makefile.am | 4 +- src/libmain/Makefile.am | 2 +- src/libnix/Makefile.am | 17 -- src/libnix/db.cc | 425 ----------------------------------------- src/libnix/db.hh | 89 --------- src/libnix/exec.cc | 127 ------------ src/libnix/exec.hh | 22 --- src/libnix/expr.cc | 210 -------------------- src/libnix/expr.hh | 60 ------ src/libnix/globals.cc | 8 - src/libnix/globals.hh | 29 --- src/libnix/normalise.cc | 369 ----------------------------------- src/libnix/normalise.hh | 46 ----- src/libnix/pathlocks.cc | 90 --------- src/libnix/pathlocks.hh | 24 --- src/libnix/references.cc | 106 ---------- src/libnix/references.hh | 10 - src/libnix/store.cc | 407 --------------------------------------- src/libnix/store.hh | 72 ------- src/libnix/test-builder-1.sh | 3 - src/libnix/test-builder-2.sh | 9 - src/libnix/test.cc | 162 ---------------- src/libstore/Makefile.am | 17 ++ src/libstore/db.cc | 425 +++++++++++++++++++++++++++++++++++++++++ src/libstore/db.hh | 89 +++++++++ src/libstore/exec.cc | 127 ++++++++++++ src/libstore/exec.hh | 22 +++ src/libstore/expr.cc | 210 ++++++++++++++++++++ src/libstore/expr.hh | 60 ++++++ src/libstore/globals.cc | 8 + src/libstore/globals.hh | 29 +++ src/libstore/normalise.cc | 369 +++++++++++++++++++++++++++++++++++ src/libstore/normalise.hh | 46 +++++ src/libstore/pathlocks.cc | 90 +++++++++ src/libstore/pathlocks.hh | 24 +++ src/libstore/references.cc | 106 ++++++++++ src/libstore/references.hh | 10 + src/libstore/store.cc | 407 +++++++++++++++++++++++++++++++++++++++ src/libstore/store.hh | 72 +++++++ src/libstore/test-builder-1.sh | 3 + src/libstore/test-builder-2.sh | 9 + src/libstore/test.cc | 162 ++++++++++++++++ src/nix-hash/Makefile.am | 4 +- src/nix/Makefile.am | 4 +- 45 files changed, 2293 insertions(+), 2293 deletions(-) delete mode 100644 src/libnix/Makefile.am delete mode 100644 src/libnix/db.cc delete mode 100644 src/libnix/db.hh delete mode 100644 src/libnix/exec.cc delete mode 100644 src/libnix/exec.hh delete mode 100644 src/libnix/expr.cc delete mode 100644 src/libnix/expr.hh delete mode 100644 src/libnix/globals.cc delete mode 100644 src/libnix/globals.hh delete mode 100644 src/libnix/normalise.cc delete mode 100644 src/libnix/normalise.hh delete mode 100644 src/libnix/pathlocks.cc delete mode 100644 src/libnix/pathlocks.hh delete mode 100644 src/libnix/references.cc delete mode 100644 src/libnix/references.hh delete mode 100644 src/libnix/store.cc delete mode 100644 src/libnix/store.hh delete mode 100755 src/libnix/test-builder-1.sh delete mode 100755 src/libnix/test-builder-2.sh delete mode 100644 src/libnix/test.cc create mode 100644 src/libstore/Makefile.am create mode 100644 src/libstore/db.cc create mode 100644 src/libstore/db.hh create mode 100644 src/libstore/exec.cc create mode 100644 src/libstore/exec.hh create mode 100644 src/libstore/expr.cc create mode 100644 src/libstore/expr.hh create mode 100644 src/libstore/globals.cc create mode 100644 src/libstore/globals.hh create mode 100644 src/libstore/normalise.cc create mode 100644 src/libstore/normalise.hh create mode 100644 src/libstore/pathlocks.cc create mode 100644 src/libstore/pathlocks.hh create mode 100644 src/libstore/references.cc create mode 100644 src/libstore/references.hh create mode 100644 src/libstore/store.cc create mode 100644 src/libstore/store.hh create mode 100755 src/libstore/test-builder-1.sh create mode 100755 src/libstore/test-builder-2.sh create mode 100644 src/libstore/test.cc (limited to 'src') diff --git a/src/Makefile.am b/src/Makefile.am index e62c29d7ce29..ed989fed20b2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1 +1 @@ -SUBDIRS = boost libutil libnix libmain nix nix-hash fix-ng +SUBDIRS = boost libutil libstore libmain nix nix-hash fix-ng diff --git a/src/fix-ng/Makefile.am b/src/fix-ng/Makefile.am index cf854932975b..df3d1730892d 100644 --- a/src/fix-ng/Makefile.am +++ b/src/fix-ng/Makefile.am @@ -1,12 +1,12 @@ bin_PROGRAMS = fix-ng fix_ng_SOURCES = fix-expr.cc parser.cc eval.cc primops.cc fix.cc -fix_ng_LDADD = ../libmain/libmain.a ../libnix/libnix.a ../libutil/libutil.a \ +fix_ng_LDADD = ../libmain/libmain.a ../libstore/libstore.a ../libutil/libutil.a \ ../boost/format/libformat.a -L../../externals/inst/lib -ldb_cxx \ -lsglr -lATB -lconversion -lasfix2 -lmept -lATerm AM_CXXFLAGS = \ - -I.. -I../../externals/inst/include -I../libutil -I../libnix -I../libmain + -I.. -I../../externals/inst/include -I../libutil -I../libstore -I../libmain # Parse table generation. diff --git a/src/libmain/Makefile.am b/src/libmain/Makefile.am index edd6685450f0..5d3144ccc7a2 100644 --- a/src/libmain/Makefile.am +++ b/src/libmain/Makefile.am @@ -7,6 +7,6 @@ AM_CXXFLAGS = \ -DNIX_DATA_DIR=\"$(datadir)\" \ -DNIX_STATE_DIR=\"$(localstatedir)/nix\" \ -DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \ - -I.. -I../../externals/inst/include -I../libutil -I../libnix + -I.. -I../../externals/inst/include -I../libutil -I../libstore EXTRA_DIST = *.hh diff --git a/src/libnix/Makefile.am b/src/libnix/Makefile.am deleted file mode 100644 index 055ef1af7b96..000000000000 --- a/src/libnix/Makefile.am +++ /dev/null @@ -1,17 +0,0 @@ -noinst_LIBRARIES = libnix.a - -libnix_a_SOURCES = \ - store.cc expr.cc normalise.cc exec.cc \ - globals.cc db.cc references.cc pathlocks.cc - -AM_CXXFLAGS = -DSYSTEM=\"@host@\" -Wall \ - -I.. -I../../externals/inst/include -I../libutil - -EXTRA_DIST = *.hh *.h test-builder-*.sh - -check_PROGRAMS = test-aterm - -test_aterm_SOURCES = test-aterm.cc -test_aterm_LDADD = libnix.a $(LDADD) ../boost/format/libformat.a \ - -L../../externals/inst/lib -ldb_cxx -lATerm - diff --git a/src/libnix/db.cc b/src/libnix/db.cc deleted file mode 100644 index 63ec2724fcb3..000000000000 --- a/src/libnix/db.cc +++ /dev/null @@ -1,425 +0,0 @@ -#include -#include -#include - -#include - -#include "db.hh" -#include "util.hh" -#include "pathlocks.hh" - - -/* Wrapper class to ensure proper destruction. */ -class DestroyDbc -{ - Dbc * dbc; -public: - DestroyDbc(Dbc * _dbc) : dbc(_dbc) { } - ~DestroyDbc() { dbc->close(); /* close() frees dbc */ } -}; - - -static void rethrow(DbException & e) -{ - throw Error(e.what()); -} - - -Transaction::Transaction() - : txn(0) -{ -} - - -Transaction::Transaction(Database & db) -{ - db.requireEnv(); - try { - db.env->txn_begin(0, &txn, 0); - } catch (DbException e) { rethrow(e); } -} - - -Transaction::~Transaction() -{ - if (txn) abort(); -} - - -void Transaction::commit() -{ - if (!txn) throw Error("commit called on null transaction"); - debug(format("committing transaction %1%") % (void *) txn); - DbTxn * txn2 = txn; - txn = 0; - try { - txn2->commit(0); - } catch (DbException e) { rethrow(e); } -} - - -void Transaction::abort() -{ - if (!txn) throw Error("abort called on null transaction"); - debug(format("aborting transaction %1%") % (void *) txn); - DbTxn * txn2 = txn; - txn = 0; - try { - txn2->abort(); - } catch (DbException e) { rethrow(e); } -} - - -void Transaction::moveTo(Transaction & t) -{ - if (t.txn) throw Error("target txn already exists"); - t.txn = txn; - txn = 0; -} - - -void Database::requireEnv() -{ - if (!env) throw Error("database environment not open"); -} - - -Db * Database::getDb(TableId table) -{ - map::iterator i = tables.find(table); - if (i == tables.end()) - throw Error("unknown table id"); - return i->second; -} - - -Database::Database() - : env(0) - , nextId(1) -{ -} - - -Database::~Database() -{ - close(); -} - - -int getAccessorCount(int fd) -{ - if (lseek(fd, 0, SEEK_SET) == -1) - throw SysError("seeking accessor count"); - char buf[128]; - int len; - if ((len = read(fd, buf, sizeof(buf) - 1)) == -1) - throw SysError("reading accessor count"); - buf[len] = 0; - int count; - if (sscanf(buf, "%d", &count) != 1) { - debug(format("accessor count is invalid: `%1%'") % buf); - return -1; - } - return count; -} - - -void setAccessorCount(int fd, int n) -{ - if (lseek(fd, 0, SEEK_SET) == -1) - throw SysError("seeking accessor count"); - string s = (format("%1%") % n).str(); - const char * s2 = s.c_str(); - if (write(fd, s2, strlen(s2)) != (ssize_t) strlen(s2) || - ftruncate(fd, strlen(s2)) != 0) - throw SysError("writing accessor count"); -} - - -void openEnv(DbEnv * env, const string & path, u_int32_t flags) -{ - env->open(path.c_str(), - DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | - DB_CREATE | flags, - 0666); -} - - -void Database::open(const string & path) -{ - if (env) throw Error(format("environment already open")); - - try { - - debug(format("opening database environment")); - - - /* Create the database environment object. */ - env = new DbEnv(0); - - env->set_lg_bsize(32 * 1024); /* default */ - env->set_lg_max(256 * 1024); /* must be > 4 * lg_bsize */ - env->set_lk_detect(DB_LOCK_DEFAULT); - env->set_flags(DB_TXN_WRITE_NOSYNC, 1); - - - /* The following code provides automatic recovery of the - database environment. Recovery is necessary when a process - dies while it has the database open. To detect this, - processes atomically increment a counter when the open the - database, and decrement it when they close it. If we see - that counter is > 0 but no processes are accessing the - database---determined by attempting to obtain a write lock - on a lock file on which all accessors have a read lock---we - must run recovery. Note that this also ensures that we - only run recovery when there are no other accessors (which - could cause database corruption). */ - - /* !!! close fdAccessors / fdLock on exception */ - - /* Open the accessor count file. */ - string accessorsPath = path + "/accessor_count"; - fdAccessors = ::open(accessorsPath.c_str(), O_RDWR | O_CREAT, 0666); - if (fdAccessors == -1) - throw SysError(format("opening file `%1%'") % accessorsPath); - - /* Open the lock file. */ - string lockPath = path + "/access_lock"; - fdLock = ::open(lockPath.c_str(), O_RDWR | O_CREAT, 0666); - if (fdLock == -1) - throw SysError(format("opening lock file `%1%'") % lockPath); - - /* Try to acquire a write lock. */ - debug(format("attempting write lock on `%1%'") % lockPath); - if (lockFile(fdLock, ltWrite, false)) { /* don't wait */ - - debug(format("write lock granted")); - - /* We have a write lock, which means that there are no - other readers or writers. */ - - int n = getAccessorCount(fdAccessors); - setAccessorCount(fdAccessors, 1); - - if (n != 0) { - printMsg(lvlTalkative, - format("accessor count is %1%, running recovery") % n); - - /* Open the environment after running recovery. */ - openEnv(env, path, DB_RECOVER); - } - - else - /* Open the environment normally. */ - openEnv(env, path, 0); - - /* Downgrade to a read lock. */ - debug(format("downgrading to read lock on `%1%'") % lockPath); - lockFile(fdLock, ltRead, true); - - } else { - /* There are other accessors. */ - debug(format("write lock refused")); - - /* Acquire a read lock. */ - debug(format("acquiring read lock on `%1%'") % lockPath); - lockFile(fdLock, ltRead, true); /* wait indefinitely */ - - /* Increment the accessor count. */ - lockFile(fdAccessors, ltWrite, true); - int n = getAccessorCount(fdAccessors) + 1; - setAccessorCount(fdAccessors, n); - debug(format("incremented accessor count to %1%") % n); - lockFile(fdAccessors, ltNone, true); - - /* Open the environment normally. */ - openEnv(env, path, 0); - } - - } catch (DbException e) { rethrow(e); } -} - - -void Database::close() -{ - if (!env) return; - - /* Close the database environment. */ - debug(format("closing database environment")); - - try { - - for (map::iterator i = tables.begin(); - i != tables.end(); i++) - { - Db * db = i->second; - db->close(DB_NOSYNC); - delete db; - } - -// env->txn_checkpoint(0, 0, 0); - env->close(0); - - } catch (DbException e) { rethrow(e); } - - delete env; - - /* Decrement the accessor count. */ - lockFile(fdAccessors, ltWrite, true); - int n = getAccessorCount(fdAccessors) - 1; - setAccessorCount(fdAccessors, n); - debug(format("decremented accessor count to %1%") % n); - lockFile(fdAccessors, ltNone, true); - - ::close(fdAccessors); - ::close(fdLock); -} - - -TableId Database::openTable(const string & tableName) -{ - requireEnv(); - TableId table = nextId++; - - try { - - Db * db = new Db(env, 0); - - try { - db->open(0, tableName.c_str(), 0, - DB_HASH, DB_CREATE | DB_AUTO_COMMIT, 0666); - } catch (...) { - delete db; - throw; - } - - tables[table] = db; - - } catch (DbException e) { rethrow(e); } - - return table; -} - - -bool Database::queryString(const Transaction & txn, TableId table, - const string & key, string & data) -{ - try { - Db * db = getDb(table); - - Dbt kt((void *) key.c_str(), key.length()); - Dbt dt; - - int err = db->get(txn.txn, &kt, &dt, 0); - if (err) return false; - - if (!dt.get_data()) - data = ""; - else - data = string((char *) dt.get_data(), dt.get_size()); - - } catch (DbException e) { rethrow(e); } - - return true; -} - - -bool Database::queryStrings(const Transaction & txn, TableId table, - const string & key, Strings & data) -{ - string d; - - if (!queryString(txn, table, key, d)) - return false; - - string::iterator it = d.begin(); - - while (it != d.end()) { - - if (it + 4 > d.end()) - throw Error(format("short db entry: `%1%'") % d); - - unsigned int len; - len = (unsigned char) *it++; - len |= ((unsigned char) *it++) << 8; - len |= ((unsigned char) *it++) << 16; - len |= ((unsigned char) *it++) << 24; - - if (it + len > d.end()) - throw Error(format("short db entry: `%1%'") % d); - - string s; - while (len--) s += *it++; - - data.push_back(s); - } - - return true; -} - - -void Database::setString(const Transaction & txn, TableId table, - const string & key, const string & data) -{ - try { - Db * db = getDb(table); - Dbt kt((void *) key.c_str(), key.length()); - Dbt dt((void *) data.c_str(), data.length()); - db->put(txn.txn, &kt, &dt, 0); - } catch (DbException e) { rethrow(e); } -} - - -void Database::setStrings(const Transaction & txn, TableId table, - const string & key, const Strings & data) -{ - string d; - - for (Strings::const_iterator it = data.begin(); - it != data.end(); it++) - { - string s = *it; - unsigned int len = s.size(); - - d += len & 0xff; - d += (len >> 8) & 0xff; - d += (len >> 16) & 0xff; - d += (len >> 24) & 0xff; - - d += s; - } - - setString(txn, table, key, d); -} - - -void Database::delPair(const Transaction & txn, TableId table, - const string & key) -{ - try { - Db * db = getDb(table); - Dbt kt((void *) key.c_str(), key.length()); - db->del(txn.txn, &kt, 0); - /* Non-existence of a pair with the given key is not an - error. */ - } catch (DbException e) { rethrow(e); } -} - - -void Database::enumTable(const Transaction & txn, TableId table, - Strings & keys) -{ - try { - Db * db = getDb(table); - - Dbc * dbc; - db->cursor(txn.txn, &dbc, 0); - DestroyDbc destroyDbc(dbc); - - Dbt kt, dt; - while (dbc->get(&kt, &dt, DB_NEXT) != DB_NOTFOUND) - keys.push_back( - string((char *) kt.get_data(), kt.get_size())); - - } catch (DbException e) { rethrow(e); } -} diff --git a/src/libnix/db.hh b/src/libnix/db.hh deleted file mode 100644 index 1c681b9b5419..000000000000 --- a/src/libnix/db.hh +++ /dev/null @@ -1,89 +0,0 @@ -#ifndef __DB_H -#define __DB_H - -#include -#include -#include - -#include - -#include "util.hh" - -using namespace std; - - -class Database; - - -class Transaction -{ - friend class Database; - -private: - DbTxn * txn; - -public: - Transaction(); - Transaction(Database & _db); - ~Transaction(); - - void abort(); - void commit(); - - void moveTo(Transaction & t); -}; - - -#define noTxn Transaction() - - -typedef unsigned int TableId; /* table handles */ - - -class Database -{ - friend class Transaction; - -private: - DbEnv * env; - - int fdLock; - int fdAccessors; - - TableId nextId; - map tables; - - void requireEnv(); - - Db * getDb(TableId table); - -public: - Database(); - ~Database(); - - void open(const string & path); - void close(); - - TableId openTable(const string & table); - - bool queryString(const Transaction & txn, TableId table, - const string & key, string & data); - - bool queryStrings(const Transaction & txn, TableId table, - const string & key, Strings & data); - - void setString(const Transaction & txn, TableId table, - const string & key, const string & data); - - void setStrings(const Transaction & txn, TableId table, - const string & key, const Strings & data); - - void delPair(const Transaction & txn, TableId table, - const string & key); - - void enumTable(const Transaction & txn, TableId table, - Strings & keys); -}; - - -#endif /* !__DB_H */ diff --git a/src/libnix/exec.cc b/src/libnix/exec.cc deleted file mode 100644 index 47a385f147d5..000000000000 --- a/src/libnix/exec.cc +++ /dev/null @@ -1,127 +0,0 @@ -#include - -#include -#include -#include -#include -#include - -#include "exec.hh" -#include "util.hh" -#include "globals.hh" - - -static string pathNullDevice = "/dev/null"; - - -/* Run a program. */ -void runProgram(const string & program, - const Strings & args, const Environment & env, - const string & logFileName) -{ - /* Create a log file. */ - string logCommand = - verbosity >= lvlDebug - ? "tee " + logFileName + " >&2" - : "cat > " + logFileName; - /* !!! auto-pclose on exit */ - FILE * logFile = popen(logCommand.c_str(), "w"); /* !!! escaping */ - if (!logFile) - throw SysError(format("creating log file `%1%'") % logFileName); - - /* Create a temporary directory where the build will take - place. */ - string tmpDir = createTempDir(); - - AutoDelete delTmpDir(tmpDir); - - /* Fork a child to build the package. */ - pid_t pid; - switch (pid = fork()) { - - case -1: - throw SysError("unable to fork"); - - case 0: - - try { /* child */ - - if (chdir(tmpDir.c_str()) == -1) - throw SysError(format("changing into to `%1%'") % tmpDir); - - /* Fill in the arguments. */ - const char * argArr[args.size() + 2]; - const char * * p = argArr; - string progName = baseNameOf(program); - *p++ = progName.c_str(); - for (Strings::const_iterator i = args.begin(); - i != args.end(); i++) - *p++ = i->c_str(); - *p = 0; - - /* Fill in the environment. */ - Strings envStrs; - const char * envArr[env.size() + 1]; - p = envArr; - for (Environment::const_iterator i = env.begin(); - i != env.end(); i++) - *p++ = envStrs.insert(envStrs.end(), - i->first + "=" + i->second)->c_str(); - *p = 0; - - /* Dup the log handle into stderr. */ - if (dup2(fileno(logFile), STDERR_FILENO) == -1) - throw SysError("cannot pipe standard error into log file"); - - /* Dup stderr to stdin. */ - if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) - throw SysError("cannot dup stderr into stdout"); - - /* Reroute stdin to /dev/null. */ - int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); - if (fdDevNull == -1) - throw SysError(format("cannot open `%1%'") % pathNullDevice); - if (dup2(fdDevNull, STDIN_FILENO) == -1) - throw SysError("cannot dup null device into stdin"); - - /* Execute the program. This should not return. */ - execve(program.c_str(), (char * *) argArr, (char * *) envArr); - - throw SysError(format("unable to execute %1%") % program); - - } catch (exception & e) { - cerr << format("build error: %1%\n") % e.what(); - } - _exit(1); - - } - - /* parent */ - - /* Close the logging pipe. Note that this should not cause - the logger to exit until builder exits (because the latter - has an open file handle to the former). */ - pclose(logFile); - - /* Wait for the child to finish. */ - int status; - if (waitpid(pid, &status, 0) != pid) - throw Error("unable to wait for child"); - - if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { - if (keepFailed) { - printMsg(lvlTalkative, - format("program `%1%' failed; keeping build directory `%2%'") - % program % tmpDir); - delTmpDir.cancel(); - } - if (WIFEXITED(status)) - throw Error(format("program `%1%' failed with exit code %2%") - % program % WEXITSTATUS(status)); - else if (WIFSIGNALED(status)) - throw Error(format("program `%1%' failed due to signal %2%") - % program % WTERMSIG(status)); - else - throw Error(format("program `%1%' died abnormally") % program); - } -} diff --git a/src/libnix/exec.hh b/src/libnix/exec.hh deleted file mode 100644 index fc5bd6ac8d0b..000000000000 --- a/src/libnix/exec.hh +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef __EXEC_H -#define __EXEC_H - -#include -#include - -#include "util.hh" - -using namespace std; - - -/* A Unix environment is a mapping from strings to strings. */ -typedef map Environment; - - -/* Run a program. */ -void runProgram(const string & program, - const Strings & args, const Environment & env, - const string & logFileName); - - -#endif /* !__EXEC_H */ diff --git a/src/libnix/expr.cc b/src/libnix/expr.cc deleted file mode 100644 index 7bb1f5306129..000000000000 --- a/src/libnix/expr.cc +++ /dev/null @@ -1,210 +0,0 @@ -#include "expr.hh" -#include "globals.hh" -#include "store.hh" - - -Error badTerm(const format & f, ATerm t) -{ - char * s = ATwriteToString(t); - if (!s) throw Error("cannot print term"); - if (strlen(s) > 1000) { - int len; - s = ATwriteToSharedString(t, &len); - if (!s) throw Error("cannot print term"); - } - return Error(format("%1%, in `%2%'") % f.str() % (string) s); -} - - -Hash hashTerm(ATerm t) -{ - return hashString(atPrint(t)); -} - - -Path writeTerm(ATerm t, const string & suffix) -{ - /* The id of a term is its hash. */ - Hash h = hashTerm(t); - - Path path = canonPath(nixStore + "/" + - (string) h + suffix + ".nix"); - - if (!isValidPath(path)) { - char * s = ATwriteToString(t); - if (!s) throw Error(format("cannot write aterm to `%1%'") % path); - addTextToStore(path, string(s)); - } - - return path; -} - - -static void parsePaths(ATermList paths, PathSet & out) -{ - ATMatcher m; - for (ATermIterator i(paths); i; ++i) { - string s; - if (!(atMatch(m, *i) >> s)) - throw badTerm("not a path", *i); - out.insert(s); - } -} - - -static void checkClosure(const Closure & closure) -{ - if (closure.elems.size() == 0) - throw Error("empty closure"); - - PathSet decl; - for (ClosureElems::const_iterator i = closure.elems.begin(); - i != closure.elems.end(); i++) - decl.insert(i->first); - - for (PathSet::const_iterator i = closure.roots.begin(); - i != closure.roots.end(); i++) - if (decl.find(*i) == decl.end()) - throw Error(format("undefined root path `%1%'") % *i); - - for (ClosureElems::const_iterator i = closure.elems.begin(); - i != closure.elems.end(); i++) - for (PathSet::const_iterator j = i->second.refs.begin(); - j != i->second.refs.end(); j++) - if (decl.find(*j) == decl.end()) - throw Error( - format("undefined path `%1%' referenced by `%2%'") - % *j % i->first); -} - - -/* Parse a closure. */ -static bool parseClosure(ATerm t, Closure & closure) -{ - ATermList roots, elems; - ATMatcher m; - - if (!(atMatch(m, t) >> "Closure" >> roots >> elems)) - return false; - - parsePaths(roots, closure.roots); - - for (ATermIterator i(elems); i; ++i) { - string path; - ATermList refs; - if (!(atMatch(m, *i) >> "" >> path >> refs)) - throw badTerm("not a closure element", *i); - ClosureElem elem; - parsePaths(refs, elem.refs); - closure.elems[path] = elem; - } - - checkClosure(closure); - return true; -} - - -static bool parseDerivation(ATerm t, Derivation & derivation) -{ - ATMatcher m; - ATermList outs, ins, args, bnds; - string builder, platform; - - if (!(atMatch(m, t) >> "Derive" >> outs >> ins >> platform - >> builder >> args >> bnds)) - return false; - - parsePaths(outs, derivation.outputs); - parsePaths(ins, derivation.inputs); - - derivation.builder = builder; - derivation.platform = platform; - - for (ATermIterator i(args); i; ++i) { - string s; - if (!(atMatch(m, *i) >> s)) - throw badTerm("string expected", *i); - derivation.args.push_back(s); - } - - for (ATermIterator i(bnds); i; ++i) { - string s1, s2; - if (!(atMatch(m, *i) >> "" >> s1 >> s2)) - throw badTerm("tuple of strings expected", *i); - derivation.env[s1] = s2; - } - - return true; -} - - -NixExpr parseNixExpr(ATerm t) -{ - NixExpr ne; - if (parseClosure(t, ne.closure)) - ne.type = NixExpr::neClosure; - else if (parseDerivation(t, ne.derivation)) - ne.type = NixExpr::neDerivation; - else throw badTerm("not a Nix expression", t); - return ne; -} - - -static ATermList unparsePaths(const PathSet & paths) -{ - ATermList l = ATempty; - for (PathSet::const_iterator i = paths.begin(); - i != paths.end(); i++) - l = ATinsert(l, ATmake("", i->c_str())); - return ATreverse(l); -} - - -static ATerm unparseClosure(const Closure & closure) -{ - ATermList roots = unparsePaths(closure.roots); - - ATermList elems = ATempty; - for (ClosureElems::const_iterator i = closure.elems.begin(); - i != closure.elems.end(); i++) - elems = ATinsert(elems, - ATmake("(, )", - i->first.c_str(), - unparsePaths(i->second.refs))); - - return ATmake("Closure(, )", roots, elems); -} - - -static ATerm unparseDerivation(const Derivation & derivation) -{ - ATermList args = ATempty; - for (Strings::const_iterator i = derivation.args.begin(); - i != derivation.args.end(); i++) - args = ATinsert(args, ATmake("", i->c_str())); - - ATermList env = ATempty; - for (StringPairs::const_iterator i = derivation.env.begin(); - i != derivation.env.end(); i++) - env = ATinsert(env, - ATmake("(, )", - i->first.c_str(), i->second.c_str())); - - return ATmake("Derive(, , , , , )", - unparsePaths(derivation.outputs), - unparsePaths(derivation.inputs), - derivation.platform.c_str(), - derivation.builder.c_str(), - ATreverse(args), - ATreverse(env)); -} - - -ATerm unparseNixExpr(const NixExpr & ne) -{ - if (ne.type == NixExpr::neClosure) - return unparseClosure(ne.closure); - else if (ne.type == NixExpr::neDerivation) - return unparseDerivation(ne.derivation); - else abort(); -} diff --git a/src/libnix/expr.hh b/src/libnix/expr.hh deleted file mode 100644 index f5abf9af0d3a..000000000000 --- a/src/libnix/expr.hh +++ /dev/null @@ -1,60 +0,0 @@ -#ifndef __FSTATE_H -#define __FSTATE_H - -#include "aterm.hh" -#include "store.hh" - - -/* Abstract syntax of Nix expressions. */ - -struct ClosureElem -{ - PathSet refs; -}; - -typedef map ClosureElems; - -struct Closure -{ - PathSet roots; - ClosureElems elems; -}; - -typedef map StringPairs; - -struct Derivation -{ - PathSet outputs; - PathSet inputs; /* Nix expressions, not actual inputs */ - string platform; - Path builder; - Strings args; - StringPairs env; -}; - -struct NixExpr -{ - enum { neClosure, neDerivation } type; - Closure closure; - Derivation derivation; -}; - - -/* Throw an exception with an error message containing the given - aterm. */ -Error badTerm(const format & f, ATerm t); - -/* Hash an aterm. */ -Hash hashTerm(ATerm t); - -/* Write an aterm to the Nix store directory, and return its path. */ -Path writeTerm(ATerm t, const string & suffix); - -/* Parse a Nix expression. */ -NixExpr parseNixExpr(ATerm t); - -/* Parse a Nix expression. */ -ATerm unparseNixExpr(const NixExpr & ne); - - -#endif /* !__FSTATE_H */ diff --git a/src/libnix/globals.cc b/src/libnix/globals.cc deleted file mode 100644 index a292b49aeae0..000000000000 --- a/src/libnix/globals.cc +++ /dev/null @@ -1,8 +0,0 @@ -#include "globals.hh" - -string nixStore = "/UNINIT"; -string nixDataDir = "/UNINIT"; -string nixLogDir = "/UNINIT"; -string nixDBPath = "/UNINIT"; - -bool keepFailed = false; diff --git a/src/libnix/globals.hh b/src/libnix/globals.hh deleted file mode 100644 index 1b4d0bde3ffe..000000000000 --- a/src/libnix/globals.hh +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef __GLOBALS_H -#define __GLOBALS_H - -#include - -using namespace std; - -/* Path names. */ - -/* nixStore is the directory where we generally store atomic and - derived files. */ -extern string nixStore; - -extern string nixDataDir; /* !!! fix */ - -/* nixLogDir is the directory where we log various operations. */ -extern string nixLogDir; - -/* nixDBPath is the path name of our Berkeley DB environment. */ -extern string nixDBPath; - - -/* Misc. global flags. */ - -/* Whether to keep temporary directories of failed builds. */ -extern bool keepFailed; - - -#endif /* !__GLOBALS_H */ diff --git a/src/libnix/normalise.cc b/src/libnix/normalise.cc deleted file mode 100644 index cb2bf4f5b183..000000000000 --- a/src/libnix/normalise.cc +++ /dev/null @@ -1,369 +0,0 @@ -#include - -#include "normalise.hh" -#include "references.hh" -#include "exec.hh" -#include "pathlocks.hh" -#include "globals.hh" - - -static Path useSuccessor(const Path & path) -{ - string pathSucc; - if (querySuccessor(path, pathSucc)) { - debug(format("successor %1% -> %2%") % (string) path % pathSucc); - return pathSucc; - } else - return path; -} - - -Path normaliseNixExpr(const Path & _nePath, PathSet pending) -{ - startNest(nest, lvlTalkative, - format("normalising expression in `%1%'") % (string) _nePath); - - /* Try to substitute the expression by any known successors in - order to speed up the rewrite process. */ - Path nePath = useSuccessor(_nePath); - - /* Get the Nix expression. */ - NixExpr ne = exprFromPath(nePath, pending); - - /* If this is a normal form (i.e., a closure) we are done. */ - if (ne.type == NixExpr::neClosure) return nePath; - if (ne.type != NixExpr::neDerivation) abort(); - - - /* Otherwise, it's a derivation expression, and we have to build it to - determine its normal form. */ - - - /* Some variables. */ - - /* Input paths, with their closure elements. */ - ClosureElems inClosures; - - /* Referenceable paths (i.e., input and output paths). */ - PathSet allPaths; - - /* The environment to be passed to the builder. */ - Environment env; - - /* The result. */ - NixExpr nf; - nf.type = NixExpr::neClosure; - - - /* The outputs are referenceable paths. */ - for (PathSet::iterator i = ne.derivation.outputs.begin(); - i != ne.derivation.outputs.end(); i++) - { - debug(format("building path `%1%'") % *i); - allPaths.insert(*i); - } - - /* Obtain locks on all output paths. The locks are automatically - released when we exit this function or Nix crashes. */ - PathLocks outputLocks(ne.derivation.outputs); - - /* Now check again whether there is a successor. This is because - another process may have started building in parallel. After - it has finished and released the locks, we can (and should) - reuse its results. (Strictly speaking the first successor - check above can be omitted, but that would be less efficient.) - Note that since we now hold the locks on the output paths, no - other process can build this expression, so no further checks - are necessary. */ - { - Path nePath2 = useSuccessor(nePath); - if (nePath != nePath2) { - NixExpr ne = exprFromPath(nePath2, pending); - debug(format("skipping build of expression `%1%', someone beat us to it") - % (string) nePath); - if (ne.type != NixExpr::neClosure) abort(); - return nePath2; - } - } - - /* Right platform? */ - if (ne.derivation.platform != thisSystem) - throw Error(format("a `%1%' is required, but I am a `%2%'") - % ne.derivation.platform % thisSystem); - - /* Realise inputs (and remember all input paths). */ - for (PathSet::iterator i = ne.derivation.inputs.begin(); - i != ne.derivation.inputs.end(); i++) - { - Path nfPath = normaliseNixExpr(*i, pending); - realiseClosure(nfPath, pending); - /* !!! nfPath should be a root of the garbage collector while - we are building */ - NixExpr ne = exprFromPath(nfPath, pending); - if (ne.type != NixExpr::neClosure) abort(); - for (ClosureElems::iterator j = ne.closure.elems.begin(); - j != ne.closure.elems.end(); j++) - { - inClosures[j->first] = j->second; - allPaths.insert(j->first); - } - } - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - env["HOME"] = "/homeless-shelter"; - - /* Build the environment. */ - for (StringPairs::iterator i = ne.derivation.env.begin(); - i != ne.derivation.env.end(); i++) - env[i->first] = i->second; - - /* We can skip running the builder if all output paths are already - valid. */ - bool fastBuild = true; - for (PathSet::iterator i = ne.derivation.outputs.begin(); - i != ne.derivation.outputs.end(); i++) - { - if (!isValidPath(*i)) { - fastBuild = false; - break; - } - } - - if (!fastBuild) { - - /* If any of the outputs already exist but are not registered, - delete them. */ - for (PathSet::iterator i = ne.derivation.outputs.begin(); - i != ne.derivation.outputs.end(); i++) - { - Path path = *i; - if (isValidPath(path)) - throw Error(format("obstructed build: path `%1%' exists") % path); - if (pathExists(path)) { - debug(format("removing unregistered path `%1%'") % path); - deletePath(path); - } - } - - /* Run the builder. */ - printMsg(lvlChatty, format("building...")); - runProgram(ne.derivation.builder, ne.derivation.args, env, - nixLogDir + "/" + baseNameOf(nePath)); - printMsg(lvlChatty, format("build completed")); - - } else - printMsg(lvlChatty, format("fast build succesful")); - - /* Check whether the output paths were created, and grep each - output path to determine what other paths it references. Also make all - output paths read-only. */ - PathSet usedPaths; - for (PathSet::iterator i = ne.derivation.outputs.begin(); - i != ne.derivation.outputs.end(); i++) - { - Path path = *i; - if (!pathExists(path)) - throw Error(format("path `%1%' does not exist") % path); - nf.closure.roots.insert(path); - - makePathReadOnly(path); - - /* For this output path, find the references to other paths contained - in it. */ - Strings refPaths = filterReferences(path, - Strings(allPaths.begin(), allPaths.end())); - - /* Construct a closure element for this output path. */ - ClosureElem elem; - - /* For each path referenced by this output path, add its id to the - closure element and add the id to the `usedPaths' set (so that the - elements referenced by *its* closure are added below). */ - for (Paths::iterator j = refPaths.begin(); - j != refPaths.end(); j++) - { - Path path = *j; - elem.refs.insert(path); - if (inClosures.find(path) != inClosures.end()) - usedPaths.insert(path); - else if (ne.derivation.outputs.find(path) == ne.derivation.outputs.end()) - abort(); - } - - nf.closure.elems[path] = elem; - } - - /* Close the closure. That is, for any referenced path, add the paths - referenced by it. */ - PathSet donePaths; - - while (!usedPaths.empty()) { - PathSet::iterator i = usedPaths.begin(); - Path path = *i; - usedPaths.erase(i); - - if (donePaths.find(path) != donePaths.end()) continue; - donePaths.insert(path); - - ClosureElems::iterator j = inClosures.find(path); - if (j == inClosures.end()) abort(); - - nf.closure.elems[path] = j->second; - - for (PathSet::iterator k = j->second.refs.begin(); - k != j->second.refs.end(); k++) - usedPaths.insert(*k); - } - - /* For debugging, print out the referenced and unreferenced paths. */ - for (ClosureElems::iterator i = inClosures.begin(); - i != inClosures.end(); i++) - { - PathSet::iterator j = donePaths.find(i->first); - if (j == donePaths.end()) - debug(format("NOT referenced: `%1%'") % i->first); - else - debug(format("referenced: `%1%'") % i->first); - } - - /* Write the normal form. This does not have to occur in the - transaction below because writing terms is idem-potent. */ - ATerm nfTerm = unparseNixExpr(nf); - printMsg(lvlVomit, format("normal form: %1%") % atPrint(nfTerm)); - Path nfPath = writeTerm(nfTerm, "-s"); - - /* Register each outpat path, and register the normal form. This - is wrapped in one database transaction to ensure that if we - crash, either everything is registered or nothing is. This is - for recoverability: unregistered paths in the store can be - deleted arbitrarily, while registered paths can only be deleted - by running the garbage collector. */ - Transaction txn; - createStoreTransaction(txn); - for (PathSet::iterator i = ne.derivation.outputs.begin(); - i != ne.derivation.outputs.end(); i++) - registerValidPath(txn, *i); - registerSuccessor(txn, nePath, nfPath); - txn.commit(); - - return nfPath; -} - - -void realiseClosure(const Path & nePath, PathSet pending) -{ - startNest(nest, lvlDebug, format("realising closure `%1%'") % nePath); - - NixExpr ne = exprFromPath(nePath, pending); - if (ne.type != NixExpr::neClosure) - throw Error(format("expected closure in `%1%'") % nePath); - - for (ClosureElems::const_iterator i = ne.closure.elems.begin(); - i != ne.closure.elems.end(); i++) - ensurePath(i->first, pending); -} - - -void ensurePath(const Path & path, PathSet pending) -{ - /* If the path is already valid, we're done. */ - if (isValidPath(path)) return; - - /* Otherwise, try the substitutes. */ - Paths subPaths = querySubstitutes(path); - - for (Paths::iterator i = subPaths.begin(); - i != subPaths.end(); i++) - { - try { - normaliseNixExpr(*i, pending); - if (isValidPath(path)) return; - throw Error(format("substitute failed to produce expected output path")); - } catch (Error & e) { - printMsg(lvlTalkative, - format("building of substitute `%1%' for `%2%' failed: %3%") - % *i % path % e.what()); - } - } - - throw Error(format("path `%1%' is required, " - "but there are no (successful) substitutes") % path); -} - - -NixExpr exprFromPath(const Path & path, PathSet pending) -{ - ensurePath(path, pending); - ATerm t = ATreadFromNamedFile(path.c_str()); - if (!t) throw Error(format("cannot read aterm from `%1%'") % path); - return parseNixExpr(t); -} - - -PathSet nixExprRoots(const Path & nePath) -{ - PathSet paths; - - NixExpr ne = exprFromPath(nePath); - - if (ne.type == NixExpr::neClosure) - paths.insert(ne.closure.roots.begin(), ne.closure.roots.end()); - else if (ne.type == NixExpr::neDerivation) - paths.insert(ne.derivation.outputs.begin(), - ne.derivation.outputs.end()); - else abort(); - - return paths; -} - - -static void requisitesWorker(const Path & nePath, - bool includeExprs, bool includeSuccessors, - PathSet & paths, PathSet & doneSet) -{ - if (doneSet.find(nePath) != doneSet.end()) return; - doneSet.insert(nePath); - - NixExpr ne = exprFromPath(nePath); - - if (ne.type == NixExpr::neClosure) - for (ClosureElems::iterator i = ne.closure.elems.begin(); - i != ne.closure.elems.end(); i++) - paths.insert(i->first); - - else if (ne.type == NixExpr::neDerivation) - for (PathSet::iterator i = ne.derivation.inputs.begin(); - i != ne.derivation.inputs.end(); i++) - requisitesWorker(*i, - includeExprs, includeSuccessors, paths, doneSet); - - else abort(); - - if (includeExprs) paths.insert(nePath); - - string nfPath; - if (includeSuccessors && (nfPath = useSuccessor(nePath)) != nePath) - requisitesWorker(nfPath, includeExprs, includeSuccessors, - paths, doneSet); -} - - -PathSet nixExprRequisites(const Path & nePath, - bool includeExprs, bool includeSuccessors) -{ - PathSet paths; - PathSet doneSet; - requisitesWorker(nePath, includeExprs, includeSuccessors, - paths, doneSet); - return paths; -} diff --git a/src/libnix/normalise.hh b/src/libnix/normalise.hh deleted file mode 100644 index bbe846404cc0..000000000000 --- a/src/libnix/normalise.hh +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef __NORMALISE_H -#define __NORMALISE_H - -#include "expr.hh" - - -/* Normalise a Nix expression. That is, if the expression is a - derivation, a path containing an equivalent closure expression is - returned. This requires that the derivation is performed, unless a - successor is known. */ -Path normaliseNixExpr(const Path & nePath, PathSet pending = PathSet()); - -/* Realise a closure expression in the file system. - - The pending paths are those that are already being realised. This - prevents infinite recursion for paths realised through a substitute - (since when we build the substitute, we would first try to realise - its output paths through substitutes... kaboom!). */ -void realiseClosure(const Path & nePath, PathSet pending = PathSet()); - -/* Ensure that a path exists, possibly by instantiating it by - realising a substitute. */ -void ensurePath(const Path & path, PathSet pending = PathSet()); - -/* Read a Nix expression, after ensuring its existence through - ensurePath(). */ -NixExpr exprFromPath(const Path & path, PathSet pending = PathSet()); - -/* Get the list of root (output) paths of the given Nix expression. */ -PathSet nixExprRoots(const Path & nePath); - -/* Get the list of paths that are required to realise the given - expression. For a derive expression, this is the union of - requisites of the inputs; for a closure expression, it is the path of - each element in the closure. If `includeExprs' is true, include the - paths of the Nix expressions themselves. If `includeSuccessors' is - true, include the requisites of successors. */ -PathSet nixExprRequisites(const Path & nePath, - bool includeExprs, bool includeSuccessors); - -/* Return the list of the paths of all known Nix expressions whose - output paths are completely contained in the set `outputs'. */ -PathSet findGenerators(const PathSet & outputs); - - -#endif /* !__NORMALISE_H */ diff --git a/src/libnix/pathlocks.cc b/src/libnix/pathlocks.cc deleted file mode 100644 index 3ecbbbcbafd5..000000000000 --- a/src/libnix/pathlocks.cc +++ /dev/null @@ -1,90 +0,0 @@ -#include - -#include -#include -#include - -#include "pathlocks.hh" - - -bool lockFile(int fd, LockType lockType, bool wait) -{ - struct flock lock; - if (lockType == ltRead) lock.l_type = F_RDLCK; - else if (lockType == ltWrite) lock.l_type = F_WRLCK; - else if (lockType == ltNone) lock.l_type = F_UNLCK; - else abort(); - lock.l_whence = SEEK_SET; - lock.l_start = 0; - lock.l_len = 0; /* entire file */ - - if (wait) { - while (fcntl(fd, F_SETLKW, &lock) != 0) - if (errno != EINTR) - throw SysError(format("acquiring/releasing lock")); - } else { - while (fcntl(fd, F_SETLK, &lock) != 0) { - if (errno == EACCES || errno == EAGAIN) return false; - if (errno != EINTR) - throw SysError(format("acquiring/releasing lock")); - } - } - - return true; -} - - -/* This enables us to check whether are not already holding a lock on - a file ourselves. POSIX locks (fcntl) suck in this respect: if we - close a descriptor, the previous lock will be closed as well. And - there is no way to query whether we already have a lock (F_GETLK - only works on locks held by other processes). */ -static StringSet lockedPaths; /* !!! not thread-safe */ - - -PathLocks::PathLocks(const PathSet & _paths) -{ - /* Note that `fds' is built incrementally so that the destructor - will only release those locks that we have already acquired. */ - - /* Sort the paths. This assures that locks are always acquired in - the same order, thus preventing deadlocks. */ - Paths paths(_paths.begin(), _paths.end()); - paths.sort(); - - /* Acquire the lock for each path. */ - for (Paths::iterator i = paths.begin(); i != paths.end(); i++) { - Path path = *i; - Path lockPath = path + ".lock"; - - debug(format("locking path `%1%'") % path); - - if (lockedPaths.find(lockPath) != lockedPaths.end()) { - debug(format("already holding lock on `%1%'") % lockPath); - continue; - } - - /* Open/create the lock file. */ - int fd = open(lockPath.c_str(), O_WRONLY | O_CREAT, 0666); - if (fd == -1) - throw SysError(format("opening lock file `%1%'") % lockPath); - - fds.push_back(fd); - this->paths.push_back(lockPath); - - /* Acquire an exclusive lock. */ - lockFile(fd, ltWrite, true); - - lockedPaths.insert(lockPath); - } -} - - -PathLocks::~PathLocks() -{ - for (list::iterator i = fds.begin(); i != fds.end(); i++) - close(*i); - - for (Paths::iterator i = paths.begin(); i != paths.end(); i++) - lockedPaths.erase(*i); -} diff --git a/src/libnix/pathlocks.hh b/src/libnix/pathlocks.hh deleted file mode 100644 index ce61386d6df0..000000000000 --- a/src/libnix/pathlocks.hh +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef __PATHLOCKS_H -#define __PATHLOCKS_H - -#include "util.hh" - - -typedef enum LockType { ltRead, ltWrite, ltNone }; - -bool lockFile(int fd, LockType lockType, bool wait); - - -class PathLocks -{ -private: - list fds; - Paths paths; - -public: - PathLocks(const PathSet & _paths); - ~PathLocks(); -}; - - -#endif /* !__PATHLOCKS_H */ diff --git a/src/libnix/references.cc b/src/libnix/references.cc deleted file mode 100644 index ab743f76d2d3..000000000000 --- a/src/libnix/references.cc +++ /dev/null @@ -1,106 +0,0 @@ -#include -#include - -#include -#include -#include -#include -#include - -#include "references.hh" -#include "hash.hh" - - -static void search(const string & s, - Strings & ids, Strings & seen) -{ - for (Strings::iterator i = ids.begin(); - i != ids.end(); ) - { - if (s.find(*i) == string::npos) - i++; - else { - debug(format("found reference to `%1%'") % *i); - seen.push_back(*i); - i = ids.erase(i); - } - } -} - - -void checkPath(const string & path, - Strings & ids, Strings & seen) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); - - if (S_ISDIR(st.st_mode)) { - AutoCloseDir dir = opendir(path.c_str()); - - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { - string name = dirent->d_name; - if (name == "." || name == "..") continue; - search(name, ids, seen); - checkPath(path + "/" + name, ids, seen); - } - } - - else if (S_ISREG(st.st_mode)) { - - debug(format("checking `%1%'") % path); - - AutoCloseFD fd = open(path.c_str(), O_RDONLY); - if (fd == -1) throw SysError(format("opening file `%1%'") % path); - - unsigned char * buf = new unsigned char[st.st_size]; - - readFull(fd, buf, st.st_size); - - search(string((char *) buf, st.st_size), ids, seen); - - delete buf; /* !!! autodelete */ - } - - else if (S_ISLNK(st.st_mode)) { - char buf[st.st_size]; - if (readlink(path.c_str(), buf, st.st_size) != st.st_size) - throw SysError(format("reading symbolic link `%1%'") % path); - search(string(buf, st.st_size), ids, seen); - } - - else throw Error(format("unknown file type: %1%") % path); -} - - -Strings filterReferences(const string & path, const Strings & paths) -{ - map backMap; - Strings ids; - Strings seen; - - /* For efficiency (and a higher hit rate), just search for the - hash part of the file name. (This assumes that all references - have the form `HASH-bla'). */ - for (Strings::const_iterator i = paths.begin(); - i != paths.end(); i++) - { - string s = string(baseNameOf(*i), 0, 32); - parseHash(s); - ids.push_back(s); - backMap[s] = *i; - } - - checkPath(path, ids, seen); - - Strings found; - for (Strings::iterator i = seen.begin(); i != seen.end(); i++) - { - map::iterator j; - if ((j = backMap.find(*i)) == backMap.end()) abort(); - found.push_back(j->second); - } - - return found; -} diff --git a/src/libnix/references.hh b/src/libnix/references.hh deleted file mode 100644 index ada23a883302..000000000000 --- a/src/libnix/references.hh +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef __REFERENCES_H -#define __REFERENCES_H - -#include "util.hh" - - -Strings filterReferences(const Path & path, const Strings & refs); - - -#endif /* !__REFERENCES_H */ diff --git a/src/libnix/store.cc b/src/libnix/store.cc deleted file mode 100644 index c83316cf6ad5..000000000000 --- a/src/libnix/store.cc +++ /dev/null @@ -1,407 +0,0 @@ -#include - -#include -#include -#include -#include -#include - -#include "store.hh" -#include "globals.hh" -#include "db.hh" -#include "archive.hh" -#include "pathlocks.hh" - - -/* Nix database. */ -static Database nixDB; - - -/* Database tables. */ - -/* dbValidPaths :: Path -> () - - The existence of a key $p$ indicates that path $p$ is valid (that - is, produced by a succesful build). */ -static TableId dbValidPaths; - -/* dbSuccessors :: Path -> Path - - Each pair $(p_1, p_2)$ in this mapping records the fact that the - Nix expression stored at path $p_1$ has a successor expression - stored at path $p_2$. - - Note that a term $y$ is a successor of $x$ iff there exists a - sequence of rewrite steps that rewrites $x$ into $y$. -*/ -static TableId dbSuccessors; - -/* dbSuccessorsRev :: Path -> [Path] - - The reverse mapping of dbSuccessors (i.e., it stores the - predecessors of a Nix expression). -*/ -static TableId dbSuccessorsRev; - -/* dbSubstitutes :: Path -> [Path] - - Each pair $(p, [ps])$ tells Nix that it can realise any of the - Nix expressions stored at paths $ps$ to produce a path $p$. - - The main purpose of this is for distributed caching of derivates. - One system can compute a derivate and put it on a website (as a Nix - archive), for instance, and then another system can register a - substitute for that derivate. The substitute in this case might be - a Nix expression that fetches the Nix archive. -*/ -static TableId dbSubstitutes; - -/* dbSubstitutesRev :: Path -> [Path] - - The reverse mapping of dbSubstitutes. -*/ -static TableId dbSubstitutesRev; - - -void openDB() -{ - nixDB.open(nixDBPath); - dbValidPaths = nixDB.openTable("validpaths"); - dbSuccessors = nixDB.openTable("successors"); - dbSuccessorsRev = nixDB.openTable("successors-rev"); - dbSubstitutes = nixDB.openTable("substitutes"); - dbSubstitutesRev = nixDB.openTable("substitutes-rev"); -} - - -void initDB() -{ -} - - -void createStoreTransaction(Transaction & txn) -{ - Transaction txn2(nixDB); - txn2.moveTo(txn); -} - - -/* Path copying. */ - -struct CopySink : DumpSink -{ - int fd; - virtual void operator () (const unsigned char * data, unsigned int len) - { - writeFull(fd, data, len); - } -}; - - -struct CopySource : RestoreSource -{ - int fd; - virtual void operator () (unsigned char * data, unsigned int len) - { - readFull(fd, data, len); - } -}; - - -void copyPath(const Path & src, const Path & dst) -{ - debug(format("copying `%1%' to `%2%'") % src % dst); - - /* Unfortunately C++ doesn't support coprocedures, so we have no - nice way to chain CopySink and CopySource together. Instead we - fork off a child to run the sink. (Fork-less platforms should - use a thread). */ - - /* Create a pipe. */ - int fds[2]; - if (pipe(fds) == -1) throw SysError("creating pipe"); - - /* Fork. */ - pid_t pid; - switch (pid = fork()) { - - case -1: - throw SysError("unable to fork"); - - case 0: /* child */ - try { - close(fds[1]); - CopySource source; - source.fd = fds[0]; - restorePath(dst, source); - _exit(0); - } catch (exception & e) { - cerr << "error: " << e.what() << endl; - } - _exit(1); - } - - close(fds[0]); - - /* Parent. */ - - CopySink sink; - sink.fd = fds[1]; - dumpPath(src, sink); - - /* Wait for the child to finish. */ - int status; - if (waitpid(pid, &status, 0) != pid) - throw SysError("waiting for child"); - - if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) - throw Error("cannot copy file: child died"); -} - - -void registerSuccessor(const Transaction & txn, - const Path & srcPath, const Path & sucPath) -{ - Path known; - if (nixDB.queryString(txn, dbSuccessors, srcPath, known) && - known != sucPath) - { - throw Error(format( - "the `impossible' happened: expression in path " - "`%1%' appears to have multiple successors " - "(known `%2%', new `%3%'") - % srcPath % known % sucPath); - } - - Paths revs; - nixDB.queryStrings(txn, dbSuccessorsRev, sucPath, revs); - if (find(revs.begin(), revs.end(), srcPath) == revs.end()) - revs.push_back(srcPath); - - nixDB.setString(txn, dbSuccessors, srcPath, sucPath); - nixDB.setStrings(txn, dbSuccessorsRev, sucPath, revs); -} - - -bool querySuccessor(const Path & srcPath, Path & sucPath) -{ - return nixDB.queryString(noTxn, dbSuccessors, srcPath, sucPath); -} - - -Paths queryPredecessors(const Path & sucPath) -{ - Paths revs; - nixDB.queryStrings(noTxn, dbSuccessorsRev, sucPath, revs); - return revs; -} - - -void registerSubstitute(const Path & srcPath, const Path & subPath) -{ - Transaction txn(nixDB); - - Paths subs; - nixDB.queryStrings(txn, dbSubstitutes, srcPath, subs); - - if (find(subs.begin(), subs.end(), subPath) != subs.end()) { - /* Nothing to do if the substitute is already known. */ - txn.abort(); - return; - } - subs.push_front(subPath); /* new substitutes take precedence */ - - Paths revs; - nixDB.queryStrings(txn, dbSubstitutesRev, subPath, revs); - if (find(revs.begin(), revs.end(), srcPath) == revs.end()) - revs.push_back(srcPath); - - nixDB.setStrings(txn, dbSubstitutes, srcPath, subs); - nixDB.setStrings(txn, dbSubstitutesRev, subPath, revs); - - txn.commit(); -} - - -Paths querySubstitutes(const Path & srcPath) -{ - Paths subPaths; - nixDB.queryStrings(noTxn, dbSubstitutes, srcPath, subPaths); - return subPaths; -} - - -void registerValidPath(const Transaction & txn, const Path & _path) -{ - Path path(canonPath(_path)); - debug(format("registering path `%1%'") % path); - nixDB.setString(txn, dbValidPaths, path, ""); -} - - -bool isValidPath(const Path & path) -{ - string s; - return nixDB.queryString(noTxn, dbValidPaths, path, s); -} - - -void unregisterValidPath(const Path & _path) -{ - Path path(canonPath(_path)); - Transaction txn(nixDB); - - debug(format("unregistering path `%1%'") % path); - - nixDB.delPair(txn, dbValidPaths, path); - - txn.commit(); -} - - -static bool isInPrefix(const string & path, const string & _prefix) -{ - string prefix = canonPath(_prefix + "/"); - return string(path, 0, prefix.size()) == prefix; -} - - -Path addToStore(const Path & _srcPath) -{ - Path srcPath(absPath(_srcPath)); - debug(format("adding `%1%' to the store") % srcPath); - - Hash h = hashPath(srcPath); - - string baseName = baseNameOf(srcPath); - Path dstPath = canonPath(nixStore + "/" + (string) h + "-" + baseName); - - if (!isValidPath(dstPath)) { - - /* The first check above is an optimisation to prevent - unnecessary lock acquisition. */ - - PathSet lockPaths; - lockPaths.insert(dstPath); - PathLocks outputLock(lockPaths); - - if (!isValidPath(dstPath)) { - copyPath(srcPath, dstPath); - - Transaction txn(nixDB); - registerValidPath(txn, dstPath); - txn.commit(); - } - } - - return dstPath; -} - - -void addTextToStore(const Path & dstPath, const string & s) -{ - if (!isValidPath(dstPath)) { - - PathSet lockPaths; - lockPaths.insert(dstPath); - PathLocks outputLock(lockPaths); - - if (!isValidPath(dstPath)) { - - AutoCloseFD fd = open(dstPath.c_str(), O_CREAT | O_EXCL | O_WRONLY, 0666); - if (fd == -1) throw SysError(format("creating store file `%1%'") % dstPath); - - writeFull(fd, (unsigned char *) s.c_str(), s.size()); - - Transaction txn(nixDB); - registerValidPath(txn, dstPath); - txn.commit(); - } - } -} - - -void deleteFromStore(const Path & _path) -{ - Path path(canonPath(_path)); - - if (!isInPrefix(path, nixStore)) - throw Error(format("path `%1%' is not in the store") % path); - - unregisterValidPath(path); - - deletePath(path); -} - - -void verifyStore() -{ - Transaction txn(nixDB); - - Paths paths; - nixDB.enumTable(txn, dbValidPaths, paths); - - for (Paths::iterator i = paths.begin(); - i != paths.end(); i++) - { - Path path = *i; - if (!pathExists(path)) { - debug(format("path `%1%' disappeared") % path); - nixDB.delPair(txn, dbValidPaths, path); - nixDB.delPair(txn, dbSuccessorsRev, path); - nixDB.delPair(txn, dbSubstitutesRev, path); - } - } - -#if 0 - Strings subs; - nixDB.enumTable(txn, dbSubstitutes, subs); - - for (Strings::iterator i = subs.begin(); - i != subs.end(); i++) - { - FSId srcId = parseHash(*i); - - Strings subIds; - nixDB.queryStrings(txn, dbSubstitutes, srcId, subIds); - - for (Strings::iterator j = subIds.begin(); - j != subIds.end(); ) - { - FSId subId = parseHash(*j); - - Strings subPaths; - nixDB.queryStrings(txn, dbId2Paths, subId, subPaths); - if (subPaths.size() == 0) { - debug(format("erasing substitute %1% for %2%") - % (string) subId % (string) srcId); - j = subIds.erase(j); - } else j++; - } - - nixDB.setStrings(txn, dbSubstitutes, srcId, subIds); - } -#endif - - Paths sucs; - nixDB.enumTable(txn, dbSuccessors, sucs); - - for (Paths::iterator i = sucs.begin(); i != sucs.end(); i++) { - Path srcPath = *i; - - Path sucPath; - if (!nixDB.queryString(txn, dbSuccessors, srcPath, sucPath)) abort(); - - Paths revs; - nixDB.queryStrings(txn, dbSuccessorsRev, sucPath, revs); - - if (find(revs.begin(), revs.end(), srcPath) == revs.end()) { - debug(format("reverse successor mapping from `%1%' to `%2%' missing") - % srcPath % sucPath); - revs.push_back(srcPath); - nixDB.setStrings(txn, dbSuccessorsRev, sucPath, revs); - } - } - - txn.commit(); -} diff --git a/src/libnix/store.hh b/src/libnix/store.hh deleted file mode 100644 index dab3d603f802..000000000000 --- a/src/libnix/store.hh +++ /dev/null @@ -1,72 +0,0 @@ -#ifndef __STORE_H -#define __STORE_H - -#include - -#include "hash.hh" -#include "db.hh" - -using namespace std; - - -/* Open the database environment. */ -void openDB(); - -/* Create the required database tables. */ -void initDB(); - -/* Get a transaction object. */ -void createStoreTransaction(Transaction & txn); - -/* Copy a path recursively. */ -void copyPath(const Path & src, const Path & dst); - -/* Register a successor. This function accepts a transaction handle - so that it can be enclosed in an atomic operation with calls to - registerValidPath(). This must be atomic, since if we register a - successor for a derivation without registering the paths built in - the derivation, we have a successor with dangling pointers, and if - we do it in reverse order, we can get an obstructed build (since to - rebuild the successor, the outputs paths must not exist). */ -void registerSuccessor(const Transaction & txn, - const Path & srcPath, const Path & sucPath); - -/* Return the predecessors of the Nix expression stored at the given - path. */ -bool querySuccessor(const Path & srcPath, Path & sucPath); - -/* Return the predecessors of the Nix expression stored at the given - path. */ -Paths queryPredecessors(const Path & sucPath); - -/* Register a substitute. */ -void registerSubstitute(const Path & srcPath, const Path & subPath); - -/* Return the substitutes expression for the given path. */ -Paths querySubstitutes(const Path & srcPath); - -/* Register the validity of a path. */ -void registerValidPath(const Transaction & txn, const Path & path); - -/* Unregister the validity of a path. */ -void unregisterValidPath(const Path & path); - -/* Checks whether a path is valid. */ -bool isValidPath(const Path & path); - -/* Copy the contents of a path to the store and register the validity - the resulting path. The resulting path is returned. */ -Path addToStore(const Path & srcPath); - -/* Like addToStore, but the path of the output is given, and the - contents written to the output path is a regular file containing - the given string. */ -void addTextToStore(const Path & dstPath, const string & s); - -/* Delete a value from the nixStore directory. */ -void deleteFromStore(const Path & path); - -void verifyStore(); - - -#endif /* !__STORE_H */ diff --git a/src/libnix/test-builder-1.sh b/src/libnix/test-builder-1.sh deleted file mode 100755 index 80e23354c3b9..000000000000 --- a/src/libnix/test-builder-1.sh +++ /dev/null @@ -1,3 +0,0 @@ -#! /bin/sh - -echo "Hello World" > $out diff --git a/src/libnix/test-builder-2.sh b/src/libnix/test-builder-2.sh deleted file mode 100755 index 0794fa96a401..000000000000 --- a/src/libnix/test-builder-2.sh +++ /dev/null @@ -1,9 +0,0 @@ -#! /bin/sh - -echo "builder 2" - -/bin/mkdir $out || exit 1 -cd $out || exit 1 -echo "Hallo Wereld" > bla -echo $builder >> bla -echo $out >> bla diff --git a/src/libnix/test.cc b/src/libnix/test.cc deleted file mode 100644 index 457fecf245f0..000000000000 --- a/src/libnix/test.cc +++ /dev/null @@ -1,162 +0,0 @@ -#include - -#include -#include - -#include "hash.hh" -#include "archive.hh" -#include "util.hh" -#include "normalise.hh" -#include "globals.hh" - - -void realise(Path nePath) -{ - Nest nest(lvlDebug, format("TEST: realising `%1%'") % nePath); - realiseClosure(normaliseNixExpr(nePath)); -} - - -struct MySink : DumpSink -{ - virtual void operator () (const unsigned char * data, unsigned int len) - { - /* Don't use cout, it's slow as hell! */ - writeFull(STDOUT_FILENO, data, len); - } -}; - - -struct MySource : RestoreSource -{ - virtual void operator () (unsigned char * data, unsigned int len) - { - readFull(STDIN_FILENO, data, len); - } -}; - - -void runTests() -{ - verbosity = (Verbosity) 100; - - /* Hashing. */ - string s = "0b0ffd0538622bfe20b92c4aa57254d9"; - Hash h = parseHash(s); - if ((string) h != s) abort(); - - try { - h = parseHash("blah blah"); - abort(); - } catch (Error err) { }; - - try { - h = parseHash("0b0ffd0538622bfe20b92c4aa57254d99"); - abort(); - } catch (Error err) { }; - - /* Path canonicalisation. */ - cout << canonPath("/./../././//") << endl; - cout << canonPath("/foo/bar") << endl; - cout << canonPath("///foo/////bar//") << endl; - cout << canonPath("/././/foo/////bar//.") << endl; - cout << canonPath("/foo////bar//..///x/") << endl; - cout << canonPath("/foo////bar//..//..//x/y/../z/") << endl; - cout << canonPath("/foo/bar/../../../..///") << endl; - - /* Dumping. */ - -#if 0 - MySink sink; - dumpPath("scratch", sink); - cout << (string) hashPath("scratch") << endl; -#endif - - /* Restoring. */ -#if 0 - MySource source; - restorePath("outdir", source); - cout << (string) hashPath("outdir") << endl; - return; -#endif - - /* Set up the test environment. */ - - mkdir("scratch", 0777); - mkdir("scratch/db", 0777); - - string testDir = absPath("scratch"); - cout << testDir << endl; - - nixStore = testDir; - nixLogDir = testDir; - nixDBPath = testDir + "/db"; - - openDB(); - initDB(); - - /* Expression evaluation. */ - - Path builder1fn; - builder1fn = addToStore("./test-builder-1.sh"); - - ATerm fs1 = ATmake( - "Closure([], [(, [])])", - builder1fn.c_str(), - builder1fn.c_str()); - Path fs1ne = writeTerm(fs1, "-c"); - - realise(fs1ne); - realise(fs1ne); - - string out1h = hashString("foo"); /* !!! bad */ - Path out1fn = nixStore + "/" + (string) out1h + "-hello.txt"; - ATerm fs3 = ATmake( - "Derive([], [], , , [], [(\"out\", )])", - out1fn.c_str(), - fs1ne.c_str(), - thisSystem.c_str(), - builder1fn.c_str(), - out1fn.c_str()); - debug(printTerm(fs3)); - Path fs3ne = writeTerm(fs3, "-d"); - - realise(fs3ne); - realise(fs3ne); - - - Path builder4fn = addToStore("./test-builder-2.sh"); - - ATerm fs4 = ATmake( - "Closure([], [(, [])])", - builder4fn.c_str(), - builder4fn.c_str()); - Path fs4ne = writeTerm(fs4, "-c"); - - realise(fs4ne); - - string out5h = hashString("bar"); /* !!! bad */ - Path out5fn = nixStore + "/" + (string) out5h + "-hello2"; - ATerm fs5 = ATmake( - "Derive([], [], , , [], [(\"out\", ), (\"builder\", )])", - out5fn.c_str(), - fs4ne.c_str(), - thisSystem.c_str(), - builder4fn.c_str(), - out5fn.c_str(), - builder4fn.c_str()); - debug(printTerm(fs5)); - Path fs5ne = writeTerm(fs5, "-d"); - - realise(fs5ne); - realise(fs5ne); -} - - -void run(Strings args) -{ - runTests(); -} - - -string programId = "test"; diff --git a/src/libstore/Makefile.am b/src/libstore/Makefile.am new file mode 100644 index 000000000000..472b1b079313 --- /dev/null +++ b/src/libstore/Makefile.am @@ -0,0 +1,17 @@ +noinst_LIBRARIES = libstore.a + +libstore_a_SOURCES = \ + store.cc expr.cc normalise.cc exec.cc \ + globals.cc db.cc references.cc pathlocks.cc + +AM_CXXFLAGS = -DSYSTEM=\"@host@\" -Wall \ + -I.. -I../../externals/inst/include -I../libutil + +EXTRA_DIST = *.hh *.h test-builder-*.sh + +check_PROGRAMS = test-aterm + +test_aterm_SOURCES = test-aterm.cc +test_aterm_LDADD = libstore.a $(LDADD) ../boost/format/libformat.a \ + -L../../externals/inst/lib -ldb_cxx -lATerm + diff --git a/src/libstore/db.cc b/src/libstore/db.cc new file mode 100644 index 000000000000..63ec2724fcb3 --- /dev/null +++ b/src/libstore/db.cc @@ -0,0 +1,425 @@ +#include +#include +#include + +#include + +#include "db.hh" +#include "util.hh" +#include "pathlocks.hh" + + +/* Wrapper class to ensure proper destruction. */ +class DestroyDbc +{ + Dbc * dbc; +public: + DestroyDbc(Dbc * _dbc) : dbc(_dbc) { } + ~DestroyDbc() { dbc->close(); /* close() frees dbc */ } +}; + + +static void rethrow(DbException & e) +{ + throw Error(e.what()); +} + + +Transaction::Transaction() + : txn(0) +{ +} + + +Transaction::Transaction(Database & db) +{ + db.requireEnv(); + try { + db.env->txn_begin(0, &txn, 0); + } catch (DbException e) { rethrow(e); } +} + + +Transaction::~Transaction() +{ + if (txn) abort(); +} + + +void Transaction::commit() +{ + if (!txn) throw Error("commit called on null transaction"); + debug(format("committing transaction %1%") % (void *) txn); + DbTxn * txn2 = txn; + txn = 0; + try { + txn2->commit(0); + } catch (DbException e) { rethrow(e); } +} + + +void Transaction::abort() +{ + if (!txn) throw Error("abort called on null transaction"); + debug(format("aborting transaction %1%") % (void *) txn); + DbTxn * txn2 = txn; + txn = 0; + try { + txn2->abort(); + } catch (DbException e) { rethrow(e); } +} + + +void Transaction::moveTo(Transaction & t) +{ + if (t.txn) throw Error("target txn already exists"); + t.txn = txn; + txn = 0; +} + + +void Database::requireEnv() +{ + if (!env) throw Error("database environment not open"); +} + + +Db * Database::getDb(TableId table) +{ + map::iterator i = tables.find(table); + if (i == tables.end()) + throw Error("unknown table id"); + return i->second; +} + + +Database::Database() + : env(0) + , nextId(1) +{ +} + + +Database::~Database() +{ + close(); +} + + +int getAccessorCount(int fd) +{ + if (lseek(fd, 0, SEEK_SET) == -1) + throw SysError("seeking accessor count"); + char buf[128]; + int len; + if ((len = read(fd, buf, sizeof(buf) - 1)) == -1) + throw SysError("reading accessor count"); + buf[len] = 0; + int count; + if (sscanf(buf, "%d", &count) != 1) { + debug(format("accessor count is invalid: `%1%'") % buf); + return -1; + } + return count; +} + + +void setAccessorCount(int fd, int n) +{ + if (lseek(fd, 0, SEEK_SET) == -1) + throw SysError("seeking accessor count"); + string s = (format("%1%") % n).str(); + const char * s2 = s.c_str(); + if (write(fd, s2, strlen(s2)) != (ssize_t) strlen(s2) || + ftruncate(fd, strlen(s2)) != 0) + throw SysError("writing accessor count"); +} + + +void openEnv(DbEnv * env, const string & path, u_int32_t flags) +{ + env->open(path.c_str(), + DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | + DB_CREATE | flags, + 0666); +} + + +void Database::open(const string & path) +{ + if (env) throw Error(format("environment already open")); + + try { + + debug(format("opening database environment")); + + + /* Create the database environment object. */ + env = new DbEnv(0); + + env->set_lg_bsize(32 * 1024); /* default */ + env->set_lg_max(256 * 1024); /* must be > 4 * lg_bsize */ + env->set_lk_detect(DB_LOCK_DEFAULT); + env->set_flags(DB_TXN_WRITE_NOSYNC, 1); + + + /* The following code provides automatic recovery of the + database environment. Recovery is necessary when a process + dies while it has the database open. To detect this, + processes atomically increment a counter when the open the + database, and decrement it when they close it. If we see + that counter is > 0 but no processes are accessing the + database---determined by attempting to obtain a write lock + on a lock file on which all accessors have a read lock---we + must run recovery. Note that this also ensures that we + only run recovery when there are no other accessors (which + could cause database corruption). */ + + /* !!! close fdAccessors / fdLock on exception */ + + /* Open the accessor count file. */ + string accessorsPath = path + "/accessor_count"; + fdAccessors = ::open(accessorsPath.c_str(), O_RDWR | O_CREAT, 0666); + if (fdAccessors == -1) + throw SysError(format("opening file `%1%'") % accessorsPath); + + /* Open the lock file. */ + string lockPath = path + "/access_lock"; + fdLock = ::open(lockPath.c_str(), O_RDWR | O_CREAT, 0666); + if (fdLock == -1) + throw SysError(format("opening lock file `%1%'") % lockPath); + + /* Try to acquire a write lock. */ + debug(format("attempting write lock on `%1%'") % lockPath); + if (lockFile(fdLock, ltWrite, false)) { /* don't wait */ + + debug(format("write lock granted")); + + /* We have a write lock, which means that there are no + other readers or writers. */ + + int n = getAccessorCount(fdAccessors); + setAccessorCount(fdAccessors, 1); + + if (n != 0) { + printMsg(lvlTalkative, + format("accessor count is %1%, running recovery") % n); + + /* Open the environment after running recovery. */ + openEnv(env, path, DB_RECOVER); + } + + else + /* Open the environment normally. */ + openEnv(env, path, 0); + + /* Downgrade to a read lock. */ + debug(format("downgrading to read lock on `%1%'") % lockPath); + lockFile(fdLock, ltRead, true); + + } else { + /* There are other accessors. */ + debug(format("write lock refused")); + + /* Acquire a read lock. */ + debug(format("acquiring read lock on `%1%'") % lockPath); + lockFile(fdLock, ltRead, true); /* wait indefinitely */ + + /* Increment the accessor count. */ + lockFile(fdAccessors, ltWrite, true); + int n = getAccessorCount(fdAccessors) + 1; + setAccessorCount(fdAccessors, n); + debug(format("incremented accessor count to %1%") % n); + lockFile(fdAccessors, ltNone, true); + + /* Open the environment normally. */ + openEnv(env, path, 0); + } + + } catch (DbException e) { rethrow(e); } +} + + +void Database::close() +{ + if (!env) return; + + /* Close the database environment. */ + debug(format("closing database environment")); + + try { + + for (map::iterator i = tables.begin(); + i != tables.end(); i++) + { + Db * db = i->second; + db->close(DB_NOSYNC); + delete db; + } + +// env->txn_checkpoint(0, 0, 0); + env->close(0); + + } catch (DbException e) { rethrow(e); } + + delete env; + + /* Decrement the accessor count. */ + lockFile(fdAccessors, ltWrite, true); + int n = getAccessorCount(fdAccessors) - 1; + setAccessorCount(fdAccessors, n); + debug(format("decremented accessor count to %1%") % n); + lockFile(fdAccessors, ltNone, true); + + ::close(fdAccessors); + ::close(fdLock); +} + + +TableId Database::openTable(const string & tableName) +{ + requireEnv(); + TableId table = nextId++; + + try { + + Db * db = new Db(env, 0); + + try { + db->open(0, tableName.c_str(), 0, + DB_HASH, DB_CREATE | DB_AUTO_COMMIT, 0666); + } catch (...) { + delete db; + throw; + } + + tables[table] = db; + + } catch (DbException e) { rethrow(e); } + + return table; +} + + +bool Database::queryString(const Transaction & txn, TableId table, + const string & key, string & data) +{ + try { + Db * db = getDb(table); + + Dbt kt((void *) key.c_str(), key.length()); + Dbt dt; + + int err = db->get(txn.txn, &kt, &dt, 0); + if (err) return false; + + if (!dt.get_data()) + data = ""; + else + data = string((char *) dt.get_data(), dt.get_size()); + + } catch (DbException e) { rethrow(e); } + + return true; +} + + +bool Database::queryStrings(const Transaction & txn, TableId table, + const string & key, Strings & data) +{ + string d; + + if (!queryString(txn, table, key, d)) + return false; + + string::iterator it = d.begin(); + + while (it != d.end()) { + + if (it + 4 > d.end()) + throw Error(format("short db entry: `%1%'") % d); + + unsigned int len; + len = (unsigned char) *it++; + len |= ((unsigned char) *it++) << 8; + len |= ((unsigned char) *it++) << 16; + len |= ((unsigned char) *it++) << 24; + + if (it + len > d.end()) + throw Error(format("short db entry: `%1%'") % d); + + string s; + while (len--) s += *it++; + + data.push_back(s); + } + + return true; +} + + +void Database::setString(const Transaction & txn, TableId table, + const string & key, const string & data) +{ + try { + Db * db = getDb(table); + Dbt kt((void *) key.c_str(), key.length()); + Dbt dt((void *) data.c_str(), data.length()); + db->put(txn.txn, &kt, &dt, 0); + } catch (DbException e) { rethrow(e); } +} + + +void Database::setStrings(const Transaction & txn, TableId table, + const string & key, const Strings & data) +{ + string d; + + for (Strings::const_iterator it = data.begin(); + it != data.end(); it++) + { + string s = *it; + unsigned int len = s.size(); + + d += len & 0xff; + d += (len >> 8) & 0xff; + d += (len >> 16) & 0xff; + d += (len >> 24) & 0xff; + + d += s; + } + + setString(txn, table, key, d); +} + + +void Database::delPair(const Transaction & txn, TableId table, + const string & key) +{ + try { + Db * db = getDb(table); + Dbt kt((void *) key.c_str(), key.length()); + db->del(txn.txn, &kt, 0); + /* Non-existence of a pair with the given key is not an + error. */ + } catch (DbException e) { rethrow(e); } +} + + +void Database::enumTable(const Transaction & txn, TableId table, + Strings & keys) +{ + try { + Db * db = getDb(table); + + Dbc * dbc; + db->cursor(txn.txn, &dbc, 0); + DestroyDbc destroyDbc(dbc); + + Dbt kt, dt; + while (dbc->get(&kt, &dt, DB_NEXT) != DB_NOTFOUND) + keys.push_back( + string((char *) kt.get_data(), kt.get_size())); + + } catch (DbException e) { rethrow(e); } +} diff --git a/src/libstore/db.hh b/src/libstore/db.hh new file mode 100644 index 000000000000..1c681b9b5419 --- /dev/null +++ b/src/libstore/db.hh @@ -0,0 +1,89 @@ +#ifndef __DB_H +#define __DB_H + +#include +#include +#include + +#include + +#include "util.hh" + +using namespace std; + + +class Database; + + +class Transaction +{ + friend class Database; + +private: + DbTxn * txn; + +public: + Transaction(); + Transaction(Database & _db); + ~Transaction(); + + void abort(); + void commit(); + + void moveTo(Transaction & t); +}; + + +#define noTxn Transaction() + + +typedef unsigned int TableId; /* table handles */ + + +class Database +{ + friend class Transaction; + +private: + DbEnv * env; + + int fdLock; + int fdAccessors; + + TableId nextId; + map tables; + + void requireEnv(); + + Db * getDb(TableId table); + +public: + Database(); + ~Database(); + + void open(const string & path); + void close(); + + TableId openTable(const string & table); + + bool queryString(const Transaction & txn, TableId table, + const string & key, string & data); + + bool queryStrings(const Transaction & txn, TableId table, + const string & key, Strings & data); + + void setString(const Transaction & txn, TableId table, + const string & key, const string & data); + + void setStrings(const Transaction & txn, TableId table, + const string & key, const Strings & data); + + void delPair(const Transaction & txn, TableId table, + const string & key); + + void enumTable(const Transaction & txn, TableId table, + Strings & keys); +}; + + +#endif /* !__DB_H */ diff --git a/src/libstore/exec.cc b/src/libstore/exec.cc new file mode 100644 index 000000000000..47a385f147d5 --- /dev/null +++ b/src/libstore/exec.cc @@ -0,0 +1,127 @@ +#include + +#include +#include +#include +#include +#include + +#include "exec.hh" +#include "util.hh" +#include "globals.hh" + + +static string pathNullDevice = "/dev/null"; + + +/* Run a program. */ +void runProgram(const string & program, + const Strings & args, const Environment & env, + const string & logFileName) +{ + /* Create a log file. */ + string logCommand = + verbosity >= lvlDebug + ? "tee " + logFileName + " >&2" + : "cat > " + logFileName; + /* !!! auto-pclose on exit */ + FILE * logFile = popen(logCommand.c_str(), "w"); /* !!! escaping */ + if (!logFile) + throw SysError(format("creating log file `%1%'") % logFileName); + + /* Create a temporary directory where the build will take + place. */ + string tmpDir = createTempDir(); + + AutoDelete delTmpDir(tmpDir); + + /* Fork a child to build the package. */ + pid_t pid; + switch (pid = fork()) { + + case -1: + throw SysError("unable to fork"); + + case 0: + + try { /* child */ + + if (chdir(tmpDir.c_str()) == -1) + throw SysError(format("changing into to `%1%'") % tmpDir); + + /* Fill in the arguments. */ + const char * argArr[args.size() + 2]; + const char * * p = argArr; + string progName = baseNameOf(program); + *p++ = progName.c_str(); + for (Strings::const_iterator i = args.begin(); + i != args.end(); i++) + *p++ = i->c_str(); + *p = 0; + + /* Fill in the environment. */ + Strings envStrs; + const char * envArr[env.size() + 1]; + p = envArr; + for (Environment::const_iterator i = env.begin(); + i != env.end(); i++) + *p++ = envStrs.insert(envStrs.end(), + i->first + "=" + i->second)->c_str(); + *p = 0; + + /* Dup the log handle into stderr. */ + if (dup2(fileno(logFile), STDERR_FILENO) == -1) + throw SysError("cannot pipe standard error into log file"); + + /* Dup stderr to stdin. */ + if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) + throw SysError("cannot dup stderr into stdout"); + + /* Reroute stdin to /dev/null. */ + int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); + if (fdDevNull == -1) + throw SysError(format("cannot open `%1%'") % pathNullDevice); + if (dup2(fdDevNull, STDIN_FILENO) == -1) + throw SysError("cannot dup null device into stdin"); + + /* Execute the program. This should not return. */ + execve(program.c_str(), (char * *) argArr, (char * *) envArr); + + throw SysError(format("unable to execute %1%") % program); + + } catch (exception & e) { + cerr << format("build error: %1%\n") % e.what(); + } + _exit(1); + + } + + /* parent */ + + /* Close the logging pipe. Note that this should not cause + the logger to exit until builder exits (because the latter + has an open file handle to the former). */ + pclose(logFile); + + /* Wait for the child to finish. */ + int status; + if (waitpid(pid, &status, 0) != pid) + throw Error("unable to wait for child"); + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + if (keepFailed) { + printMsg(lvlTalkative, + format("program `%1%' failed; keeping build directory `%2%'") + % program % tmpDir); + delTmpDir.cancel(); + } + if (WIFEXITED(status)) + throw Error(format("program `%1%' failed with exit code %2%") + % program % WEXITSTATUS(status)); + else if (WIFSIGNALED(status)) + throw Error(format("program `%1%' failed due to signal %2%") + % program % WTERMSIG(status)); + else + throw Error(format("program `%1%' died abnormally") % program); + } +} diff --git a/src/libstore/exec.hh b/src/libstore/exec.hh new file mode 100644 index 000000000000..fc5bd6ac8d0b --- /dev/null +++ b/src/libstore/exec.hh @@ -0,0 +1,22 @@ +#ifndef __EXEC_H +#define __EXEC_H + +#include +#include + +#include "util.hh" + +using namespace std; + + +/* A Unix environment is a mapping from strings to strings. */ +typedef map Environment; + + +/* Run a program. */ +void runProgram(const string & program, + const Strings & args, const Environment & env, + const string & logFileName); + + +#endif /* !__EXEC_H */ diff --git a/src/libstore/expr.cc b/src/libstore/expr.cc new file mode 100644 index 000000000000..7bb1f5306129 --- /dev/null +++ b/src/libstore/expr.cc @@ -0,0 +1,210 @@ +#include "expr.hh" +#include "globals.hh" +#include "store.hh" + + +Error badTerm(const format & f, ATerm t) +{ + char * s = ATwriteToString(t); + if (!s) throw Error("cannot print term"); + if (strlen(s) > 1000) { + int len; + s = ATwriteToSharedString(t, &len); + if (!s) throw Error("cannot print term"); + } + return Error(format("%1%, in `%2%'") % f.str() % (string) s); +} + + +Hash hashTerm(ATerm t) +{ + return hashString(atPrint(t)); +} + + +Path writeTerm(ATerm t, const string & suffix) +{ + /* The id of a term is its hash. */ + Hash h = hashTerm(t); + + Path path = canonPath(nixStore + "/" + + (string) h + suffix + ".nix"); + + if (!isValidPath(path)) { + char * s = ATwriteToString(t); + if (!s) throw Error(format("cannot write aterm to `%1%'") % path); + addTextToStore(path, string(s)); + } + + return path; +} + + +static void parsePaths(ATermList paths, PathSet & out) +{ + ATMatcher m; + for (ATermIterator i(paths); i; ++i) { + string s; + if (!(atMatch(m, *i) >> s)) + throw badTerm("not a path", *i); + out.insert(s); + } +} + + +static void checkClosure(const Closure & closure) +{ + if (closure.elems.size() == 0) + throw Error("empty closure"); + + PathSet decl; + for (ClosureElems::const_iterator i = closure.elems.begin(); + i != closure.elems.end(); i++) + decl.insert(i->first); + + for (PathSet::const_iterator i = closure.roots.begin(); + i != closure.roots.end(); i++) + if (decl.find(*i) == decl.end()) + throw Error(format("undefined root path `%1%'") % *i); + + for (ClosureElems::const_iterator i = closure.elems.begin(); + i != closure.elems.end(); i++) + for (PathSet::const_iterator j = i->second.refs.begin(); + j != i->second.refs.end(); j++) + if (decl.find(*j) == decl.end()) + throw Error( + format("undefined path `%1%' referenced by `%2%'") + % *j % i->first); +} + + +/* Parse a closure. */ +static bool parseClosure(ATerm t, Closure & closure) +{ + ATermList roots, elems; + ATMatcher m; + + if (!(atMatch(m, t) >> "Closure" >> roots >> elems)) + return false; + + parsePaths(roots, closure.roots); + + for (ATermIterator i(elems); i; ++i) { + string path; + ATermList refs; + if (!(atMatch(m, *i) >> "" >> path >> refs)) + throw badTerm("not a closure element", *i); + ClosureElem elem; + parsePaths(refs, elem.refs); + closure.elems[path] = elem; + } + + checkClosure(closure); + return true; +} + + +static bool parseDerivation(ATerm t, Derivation & derivation) +{ + ATMatcher m; + ATermList outs, ins, args, bnds; + string builder, platform; + + if (!(atMatch(m, t) >> "Derive" >> outs >> ins >> platform + >> builder >> args >> bnds)) + return false; + + parsePaths(outs, derivation.outputs); + parsePaths(ins, derivation.inputs); + + derivation.builder = builder; + derivation.platform = platform; + + for (ATermIterator i(args); i; ++i) { + string s; + if (!(atMatch(m, *i) >> s)) + throw badTerm("string expected", *i); + derivation.args.push_back(s); + } + + for (ATermIterator i(bnds); i; ++i) { + string s1, s2; + if (!(atMatch(m, *i) >> "" >> s1 >> s2)) + throw badTerm("tuple of strings expected", *i); + derivation.env[s1] = s2; + } + + return true; +} + + +NixExpr parseNixExpr(ATerm t) +{ + NixExpr ne; + if (parseClosure(t, ne.closure)) + ne.type = NixExpr::neClosure; + else if (parseDerivation(t, ne.derivation)) + ne.type = NixExpr::neDerivation; + else throw badTerm("not a Nix expression", t); + return ne; +} + + +static ATermList unparsePaths(const PathSet & paths) +{ + ATermList l = ATempty; + for (PathSet::const_iterator i = paths.begin(); + i != paths.end(); i++) + l = ATinsert(l, ATmake("", i->c_str())); + return ATreverse(l); +} + + +static ATerm unparseClosure(const Closure & closure) +{ + ATermList roots = unparsePaths(closure.roots); + + ATermList elems = ATempty; + for (ClosureElems::const_iterator i = closure.elems.begin(); + i != closure.elems.end(); i++) + elems = ATinsert(elems, + ATmake("(, )", + i->first.c_str(), + unparsePaths(i->second.refs))); + + return ATmake("Closure(, )", roots, elems); +} + + +static ATerm unparseDerivation(const Derivation & derivation) +{ + ATermList args = ATempty; + for (Strings::const_iterator i = derivation.args.begin(); + i != derivation.args.end(); i++) + args = ATinsert(args, ATmake("", i->c_str())); + + ATermList env = ATempty; + for (StringPairs::const_iterator i = derivation.env.begin(); + i != derivation.env.end(); i++) + env = ATinsert(env, + ATmake("(, )", + i->first.c_str(), i->second.c_str())); + + return ATmake("Derive(, , , , , )", + unparsePaths(derivation.outputs), + unparsePaths(derivation.inputs), + derivation.platform.c_str(), + derivation.builder.c_str(), + ATreverse(args), + ATreverse(env)); +} + + +ATerm unparseNixExpr(const NixExpr & ne) +{ + if (ne.type == NixExpr::neClosure) + return unparseClosure(ne.closure); + else if (ne.type == NixExpr::neDerivation) + return unparseDerivation(ne.derivation); + else abort(); +} diff --git a/src/libstore/expr.hh b/src/libstore/expr.hh new file mode 100644 index 000000000000..f5abf9af0d3a --- /dev/null +++ b/src/libstore/expr.hh @@ -0,0 +1,60 @@ +#ifndef __FSTATE_H +#define __FSTATE_H + +#include "aterm.hh" +#include "store.hh" + + +/* Abstract syntax of Nix expressions. */ + +struct ClosureElem +{ + PathSet refs; +}; + +typedef map ClosureElems; + +struct Closure +{ + PathSet roots; + ClosureElems elems; +}; + +typedef map StringPairs; + +struct Derivation +{ + PathSet outputs; + PathSet inputs; /* Nix expressions, not actual inputs */ + string platform; + Path builder; + Strings args; + StringPairs env; +}; + +struct NixExpr +{ + enum { neClosure, neDerivation } type; + Closure closure; + Derivation derivation; +}; + + +/* Throw an exception with an error message containing the given + aterm. */ +Error badTerm(const format & f, ATerm t); + +/* Hash an aterm. */ +Hash hashTerm(ATerm t); + +/* Write an aterm to the Nix store directory, and return its path. */ +Path writeTerm(ATerm t, const string & suffix); + +/* Parse a Nix expression. */ +NixExpr parseNixExpr(ATerm t); + +/* Parse a Nix expression. */ +ATerm unparseNixExpr(const NixExpr & ne); + + +#endif /* !__FSTATE_H */ diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc new file mode 100644 index 000000000000..a292b49aeae0 --- /dev/null +++ b/src/libstore/globals.cc @@ -0,0 +1,8 @@ +#include "globals.hh" + +string nixStore = "/UNINIT"; +string nixDataDir = "/UNINIT"; +string nixLogDir = "/UNINIT"; +string nixDBPath = "/UNINIT"; + +bool keepFailed = false; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh new file mode 100644 index 000000000000..1b4d0bde3ffe --- /dev/null +++ b/src/libstore/globals.hh @@ -0,0 +1,29 @@ +#ifndef __GLOBALS_H +#define __GLOBALS_H + +#include + +using namespace std; + +/* Path names. */ + +/* nixStore is the directory where we generally store atomic and + derived files. */ +extern string nixStore; + +extern string nixDataDir; /* !!! fix */ + +/* nixLogDir is the directory where we log various operations. */ +extern string nixLogDir; + +/* nixDBPath is the path name of our Berkeley DB environment. */ +extern string nixDBPath; + + +/* Misc. global flags. */ + +/* Whether to keep temporary directories of failed builds. */ +extern bool keepFailed; + + +#endif /* !__GLOBALS_H */ diff --git a/src/libstore/normalise.cc b/src/libstore/normalise.cc new file mode 100644 index 000000000000..cb2bf4f5b183 --- /dev/null +++ b/src/libstore/normalise.cc @@ -0,0 +1,369 @@ +#include + +#include "normalise.hh" +#include "references.hh" +#include "exec.hh" +#include "pathlocks.hh" +#include "globals.hh" + + +static Path useSuccessor(const Path & path) +{ + string pathSucc; + if (querySuccessor(path, pathSucc)) { + debug(format("successor %1% -> %2%") % (string) path % pathSucc); + return pathSucc; + } else + return path; +} + + +Path normaliseNixExpr(const Path & _nePath, PathSet pending) +{ + startNest(nest, lvlTalkative, + format("normalising expression in `%1%'") % (string) _nePath); + + /* Try to substitute the expression by any known successors in + order to speed up the rewrite process. */ + Path nePath = useSuccessor(_nePath); + + /* Get the Nix expression. */ + NixExpr ne = exprFromPath(nePath, pending); + + /* If this is a normal form (i.e., a closure) we are done. */ + if (ne.type == NixExpr::neClosure) return nePath; + if (ne.type != NixExpr::neDerivation) abort(); + + + /* Otherwise, it's a derivation expression, and we have to build it to + determine its normal form. */ + + + /* Some variables. */ + + /* Input paths, with their closure elements. */ + ClosureElems inClosures; + + /* Referenceable paths (i.e., input and output paths). */ + PathSet allPaths; + + /* The environment to be passed to the builder. */ + Environment env; + + /* The result. */ + NixExpr nf; + nf.type = NixExpr::neClosure; + + + /* The outputs are referenceable paths. */ + for (PathSet::iterator i = ne.derivation.outputs.begin(); + i != ne.derivation.outputs.end(); i++) + { + debug(format("building path `%1%'") % *i); + allPaths.insert(*i); + } + + /* Obtain locks on all output paths. The locks are automatically + released when we exit this function or Nix crashes. */ + PathLocks outputLocks(ne.derivation.outputs); + + /* Now check again whether there is a successor. This is because + another process may have started building in parallel. After + it has finished and released the locks, we can (and should) + reuse its results. (Strictly speaking the first successor + check above can be omitted, but that would be less efficient.) + Note that since we now hold the locks on the output paths, no + other process can build this expression, so no further checks + are necessary. */ + { + Path nePath2 = useSuccessor(nePath); + if (nePath != nePath2) { + NixExpr ne = exprFromPath(nePath2, pending); + debug(format("skipping build of expression `%1%', someone beat us to it") + % (string) nePath); + if (ne.type != NixExpr::neClosure) abort(); + return nePath2; + } + } + + /* Right platform? */ + if (ne.derivation.platform != thisSystem) + throw Error(format("a `%1%' is required, but I am a `%2%'") + % ne.derivation.platform % thisSystem); + + /* Realise inputs (and remember all input paths). */ + for (PathSet::iterator i = ne.derivation.inputs.begin(); + i != ne.derivation.inputs.end(); i++) + { + Path nfPath = normaliseNixExpr(*i, pending); + realiseClosure(nfPath, pending); + /* !!! nfPath should be a root of the garbage collector while + we are building */ + NixExpr ne = exprFromPath(nfPath, pending); + if (ne.type != NixExpr::neClosure) abort(); + for (ClosureElems::iterator j = ne.closure.elems.begin(); + j != ne.closure.elems.end(); j++) + { + inClosures[j->first] = j->second; + allPaths.insert(j->first); + } + } + + /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when + PATH is not set. We don't want this, so we fill it in with some dummy + value. */ + env["PATH"] = "/path-not-set"; + + /* Set HOME to a non-existing path to prevent certain programs from using + /etc/passwd (or NIS, or whatever) to locate the home directory (for + example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd + if HOME is not set, but they will just assume that the settings file + they are looking for does not exist if HOME is set but points to some + non-existing path. */ + env["HOME"] = "/homeless-shelter"; + + /* Build the environment. */ + for (StringPairs::iterator i = ne.derivation.env.begin(); + i != ne.derivation.env.end(); i++) + env[i->first] = i->second; + + /* We can skip running the builder if all output paths are already + valid. */ + bool fastBuild = true; + for (PathSet::iterator i = ne.derivation.outputs.begin(); + i != ne.derivation.outputs.end(); i++) + { + if (!isValidPath(*i)) { + fastBuild = false; + break; + } + } + + if (!fastBuild) { + + /* If any of the outputs already exist but are not registered, + delete them. */ + for (PathSet::iterator i = ne.derivation.outputs.begin(); + i != ne.derivation.outputs.end(); i++) + { + Path path = *i; + if (isValidPath(path)) + throw Error(format("obstructed build: path `%1%' exists") % path); + if (pathExists(path)) { + debug(format("removing unregistered path `%1%'") % path); + deletePath(path); + } + } + + /* Run the builder. */ + printMsg(lvlChatty, format("building...")); + runProgram(ne.derivation.builder, ne.derivation.args, env, + nixLogDir + "/" + baseNameOf(nePath)); + printMsg(lvlChatty, format("build completed")); + + } else + printMsg(lvlChatty, format("fast build succesful")); + + /* Check whether the output paths were created, and grep each + output path to determine what other paths it references. Also make all + output paths read-only. */ + PathSet usedPaths; + for (PathSet::iterator i = ne.derivation.outputs.begin(); + i != ne.derivation.outputs.end(); i++) + { + Path path = *i; + if (!pathExists(path)) + throw Error(format("path `%1%' does not exist") % path); + nf.closure.roots.insert(path); + + makePathReadOnly(path); + + /* For this output path, find the references to other paths contained + in it. */ + Strings refPaths = filterReferences(path, + Strings(allPaths.begin(), allPaths.end())); + + /* Construct a closure element for this output path. */ + ClosureElem elem; + + /* For each path referenced by this output path, add its id to the + closure element and add the id to the `usedPaths' set (so that the + elements referenced by *its* closure are added below). */ + for (Paths::iterator j = refPaths.begin(); + j != refPaths.end(); j++) + { + Path path = *j; + elem.refs.insert(path); + if (inClosures.find(path) != inClosures.end()) + usedPaths.insert(path); + else if (ne.derivation.outputs.find(path) == ne.derivation.outputs.end()) + abort(); + } + + nf.closure.elems[path] = elem; + } + + /* Close the closure. That is, for any referenced path, add the paths + referenced by it. */ + PathSet donePaths; + + while (!usedPaths.empty()) { + PathSet::iterator i = usedPaths.begin(); + Path path = *i; + usedPaths.erase(i); + + if (donePaths.find(path) != donePaths.end()) continue; + donePaths.insert(path); + + ClosureElems::iterator j = inClosures.find(path); + if (j == inClosures.end()) abort(); + + nf.closure.elems[path] = j->second; + + for (PathSet::iterator k = j->second.refs.begin(); + k != j->second.refs.end(); k++) + usedPaths.insert(*k); + } + + /* For debugging, print out the referenced and unreferenced paths. */ + for (ClosureElems::iterator i = inClosures.begin(); + i != inClosures.end(); i++) + { + PathSet::iterator j = donePaths.find(i->first); + if (j == donePaths.end()) + debug(format("NOT referenced: `%1%'") % i->first); + else + debug(format("referenced: `%1%'") % i->first); + } + + /* Write the normal form. This does not have to occur in the + transaction below because writing terms is idem-potent. */ + ATerm nfTerm = unparseNixExpr(nf); + printMsg(lvlVomit, format("normal form: %1%") % atPrint(nfTerm)); + Path nfPath = writeTerm(nfTerm, "-s"); + + /* Register each outpat path, and register the normal form. This + is wrapped in one database transaction to ensure that if we + crash, either everything is registered or nothing is. This is + for recoverability: unregistered paths in the store can be + deleted arbitrarily, while registered paths can only be deleted + by running the garbage collector. */ + Transaction txn; + createStoreTransaction(txn); + for (PathSet::iterator i = ne.derivation.outputs.begin(); + i != ne.derivation.outputs.end(); i++) + registerValidPath(txn, *i); + registerSuccessor(txn, nePath, nfPath); + txn.commit(); + + return nfPath; +} + + +void realiseClosure(const Path & nePath, PathSet pending) +{ + startNest(nest, lvlDebug, format("realising closure `%1%'") % nePath); + + NixExpr ne = exprFromPath(nePath, pending); + if (ne.type != NixExpr::neClosure) + throw Error(format("expected closure in `%1%'") % nePath); + + for (ClosureElems::const_iterator i = ne.closure.elems.begin(); + i != ne.closure.elems.end(); i++) + ensurePath(i->first, pending); +} + + +void ensurePath(const Path & path, PathSet pending) +{ + /* If the path is already valid, we're done. */ + if (isValidPath(path)) return; + + /* Otherwise, try the substitutes. */ + Paths subPaths = querySubstitutes(path); + + for (Paths::iterator i = subPaths.begin(); + i != subPaths.end(); i++) + { + try { + normaliseNixExpr(*i, pending); + if (isValidPath(path)) return; + throw Error(format("substitute failed to produce expected output path")); + } catch (Error & e) { + printMsg(lvlTalkative, + format("building of substitute `%1%' for `%2%' failed: %3%") + % *i % path % e.what()); + } + } + + throw Error(format("path `%1%' is required, " + "but there are no (successful) substitutes") % path); +} + + +NixExpr exprFromPath(const Path & path, PathSet pending) +{ + ensurePath(path, pending); + ATerm t = ATreadFromNamedFile(path.c_str()); + if (!t) throw Error(format("cannot read aterm from `%1%'") % path); + return parseNixExpr(t); +} + + +PathSet nixExprRoots(const Path & nePath) +{ + PathSet paths; + + NixExpr ne = exprFromPath(nePath); + + if (ne.type == NixExpr::neClosure) + paths.insert(ne.closure.roots.begin(), ne.closure.roots.end()); + else if (ne.type == NixExpr::neDerivation) + paths.insert(ne.derivation.outputs.begin(), + ne.derivation.outputs.end()); + else abort(); + + return paths; +} + + +static void requisitesWorker(const Path & nePath, + bool includeExprs, bool includeSuccessors, + PathSet & paths, PathSet & doneSet) +{ + if (doneSet.find(nePath) != doneSet.end()) return; + doneSet.insert(nePath); + + NixExpr ne = exprFromPath(nePath); + + if (ne.type == NixExpr::neClosure) + for (ClosureElems::iterator i = ne.closure.elems.begin(); + i != ne.closure.elems.end(); i++) + paths.insert(i->first); + + else if (ne.type == NixExpr::neDerivation) + for (PathSet::iterator i = ne.derivation.inputs.begin(); + i != ne.derivation.inputs.end(); i++) + requisitesWorker(*i, + includeExprs, includeSuccessors, paths, doneSet); + + else abort(); + + if (includeExprs) paths.insert(nePath); + + string nfPath; + if (includeSuccessors && (nfPath = useSuccessor(nePath)) != nePath) + requisitesWorker(nfPath, includeExprs, includeSuccessors, + paths, doneSet); +} + + +PathSet nixExprRequisites(const Path & nePath, + bool includeExprs, bool includeSuccessors) +{ + PathSet paths; + PathSet doneSet; + requisitesWorker(nePath, includeExprs, includeSuccessors, + paths, doneSet); + return paths; +} diff --git a/src/libstore/normalise.hh b/src/libstore/normalise.hh new file mode 100644 index 000000000000..bbe846404cc0 --- /dev/null +++ b/src/libstore/normalise.hh @@ -0,0 +1,46 @@ +#ifndef __NORMALISE_H +#define __NORMALISE_H + +#include "expr.hh" + + +/* Normalise a Nix expression. That is, if the expression is a + derivation, a path containing an equivalent closure expression is + returned. This requires that the derivation is performed, unless a + successor is known. */ +Path normaliseNixExpr(const Path & nePath, PathSet pending = PathSet()); + +/* Realise a closure expression in the file system. + + The pending paths are those that are already being realised. This + prevents infinite recursion for paths realised through a substitute + (since when we build the substitute, we would first try to realise + its output paths through substitutes... kaboom!). */ +void realiseClosure(const Path & nePath, PathSet pending = PathSet()); + +/* Ensure that a path exists, possibly by instantiating it by + realising a substitute. */ +void ensurePath(const Path & path, PathSet pending = PathSet()); + +/* Read a Nix expression, after ensuring its existence through + ensurePath(). */ +NixExpr exprFromPath(const Path & path, PathSet pending = PathSet()); + +/* Get the list of root (output) paths of the given Nix expression. */ +PathSet nixExprRoots(const Path & nePath); + +/* Get the list of paths that are required to realise the given + expression. For a derive expression, this is the union of + requisites of the inputs; for a closure expression, it is the path of + each element in the closure. If `includeExprs' is true, include the + paths of the Nix expressions themselves. If `includeSuccessors' is + true, include the requisites of successors. */ +PathSet nixExprRequisites(const Path & nePath, + bool includeExprs, bool includeSuccessors); + +/* Return the list of the paths of all known Nix expressions whose + output paths are completely contained in the set `outputs'. */ +PathSet findGenerators(const PathSet & outputs); + + +#endif /* !__NORMALISE_H */ diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc new file mode 100644 index 000000000000..3ecbbbcbafd5 --- /dev/null +++ b/src/libstore/pathlocks.cc @@ -0,0 +1,90 @@ +#include + +#include +#include +#include + +#include "pathlocks.hh" + + +bool lockFile(int fd, LockType lockType, bool wait) +{ + struct flock lock; + if (lockType == ltRead) lock.l_type = F_RDLCK; + else if (lockType == ltWrite) lock.l_type = F_WRLCK; + else if (lockType == ltNone) lock.l_type = F_UNLCK; + else abort(); + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 0; /* entire file */ + + if (wait) { + while (fcntl(fd, F_SETLKW, &lock) != 0) + if (errno != EINTR) + throw SysError(format("acquiring/releasing lock")); + } else { + while (fcntl(fd, F_SETLK, &lock) != 0) { + if (errno == EACCES || errno == EAGAIN) return false; + if (errno != EINTR) + throw SysError(format("acquiring/releasing lock")); + } + } + + return true; +} + + +/* This enables us to check whether are not already holding a lock on + a file ourselves. POSIX locks (fcntl) suck in this respect: if we + close a descriptor, the previous lock will be closed as well. And + there is no way to query whether we already have a lock (F_GETLK + only works on locks held by other processes). */ +static StringSet lockedPaths; /* !!! not thread-safe */ + + +PathLocks::PathLocks(const PathSet & _paths) +{ + /* Note that `fds' is built incrementally so that the destructor + will only release those locks that we have already acquired. */ + + /* Sort the paths. This assures that locks are always acquired in + the same order, thus preventing deadlocks. */ + Paths paths(_paths.begin(), _paths.end()); + paths.sort(); + + /* Acquire the lock for each path. */ + for (Paths::iterator i = paths.begin(); i != paths.end(); i++) { + Path path = *i; + Path lockPath = path + ".lock"; + + debug(format("locking path `%1%'") % path); + + if (lockedPaths.find(lockPath) != lockedPaths.end()) { + debug(format("already holding lock on `%1%'") % lockPath); + continue; + } + + /* Open/create the lock file. */ + int fd = open(lockPath.c_str(), O_WRONLY | O_CREAT, 0666); + if (fd == -1) + throw SysError(format("opening lock file `%1%'") % lockPath); + + fds.push_back(fd); + this->paths.push_back(lockPath); + + /* Acquire an exclusive lock. */ + lockFile(fd, ltWrite, true); + + lockedPaths.insert(lockPath); + } +} + + +PathLocks::~PathLocks() +{ + for (list::iterator i = fds.begin(); i != fds.end(); i++) + close(*i); + + for (Paths::iterator i = paths.begin(); i != paths.end(); i++) + lockedPaths.erase(*i); +} diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh new file mode 100644 index 000000000000..ce61386d6df0 --- /dev/null +++ b/src/libstore/pathlocks.hh @@ -0,0 +1,24 @@ +#ifndef __PATHLOCKS_H +#define __PATHLOCKS_H + +#include "util.hh" + + +typedef enum LockType { ltRead, ltWrite, ltNone }; + +bool lockFile(int fd, LockType lockType, bool wait); + + +class PathLocks +{ +private: + list fds; + Paths paths; + +public: + PathLocks(const PathSet & _paths); + ~PathLocks(); +}; + + +#endif /* !__PATHLOCKS_H */ diff --git a/src/libstore/references.cc b/src/libstore/references.cc new file mode 100644 index 000000000000..ab743f76d2d3 --- /dev/null +++ b/src/libstore/references.cc @@ -0,0 +1,106 @@ +#include +#include + +#include +#include +#include +#include +#include + +#include "references.hh" +#include "hash.hh" + + +static void search(const string & s, + Strings & ids, Strings & seen) +{ + for (Strings::iterator i = ids.begin(); + i != ids.end(); ) + { + if (s.find(*i) == string::npos) + i++; + else { + debug(format("found reference to `%1%'") % *i); + seen.push_back(*i); + i = ids.erase(i); + } + } +} + + +void checkPath(const string & path, + Strings & ids, Strings & seen) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + + if (S_ISDIR(st.st_mode)) { + AutoCloseDir dir = opendir(path.c_str()); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { + string name = dirent->d_name; + if (name == "." || name == "..") continue; + search(name, ids, seen); + checkPath(path + "/" + name, ids, seen); + } + } + + else if (S_ISREG(st.st_mode)) { + + debug(format("checking `%1%'") % path); + + AutoCloseFD fd = open(path.c_str(), O_RDONLY); + if (fd == -1) throw SysError(format("opening file `%1%'") % path); + + unsigned char * buf = new unsigned char[st.st_size]; + + readFull(fd, buf, st.st_size); + + search(string((char *) buf, st.st_size), ids, seen); + + delete buf; /* !!! autodelete */ + } + + else if (S_ISLNK(st.st_mode)) { + char buf[st.st_size]; + if (readlink(path.c_str(), buf, st.st_size) != st.st_size) + throw SysError(format("reading symbolic link `%1%'") % path); + search(string(buf, st.st_size), ids, seen); + } + + else throw Error(format("unknown file type: %1%") % path); +} + + +Strings filterReferences(const string & path, const Strings & paths) +{ + map backMap; + Strings ids; + Strings seen; + + /* For efficiency (and a higher hit rate), just search for the + hash part of the file name. (This assumes that all references + have the form `HASH-bla'). */ + for (Strings::const_iterator i = paths.begin(); + i != paths.end(); i++) + { + string s = string(baseNameOf(*i), 0, 32); + parseHash(s); + ids.push_back(s); + backMap[s] = *i; + } + + checkPath(path, ids, seen); + + Strings found; + for (Strings::iterator i = seen.begin(); i != seen.end(); i++) + { + map::iterator j; + if ((j = backMap.find(*i)) == backMap.end()) abort(); + found.push_back(j->second); + } + + return found; +} diff --git a/src/libstore/references.hh b/src/libstore/references.hh new file mode 100644 index 000000000000..ada23a883302 --- /dev/null +++ b/src/libstore/references.hh @@ -0,0 +1,10 @@ +#ifndef __REFERENCES_H +#define __REFERENCES_H + +#include "util.hh" + + +Strings filterReferences(const Path & path, const Strings & refs); + + +#endif /* !__REFERENCES_H */ diff --git a/src/libstore/store.cc b/src/libstore/store.cc new file mode 100644 index 000000000000..c83316cf6ad5 --- /dev/null +++ b/src/libstore/store.cc @@ -0,0 +1,407 @@ +#include + +#include +#include +#include +#include +#include + +#include "store.hh" +#include "globals.hh" +#include "db.hh" +#include "archive.hh" +#include "pathlocks.hh" + + +/* Nix database. */ +static Database nixDB; + + +/* Database tables. */ + +/* dbValidPaths :: Path -> () + + The existence of a key $p$ indicates that path $p$ is valid (that + is, produced by a succesful build). */ +static TableId dbValidPaths; + +/* dbSuccessors :: Path -> Path + + Each pair $(p_1, p_2)$ in this mapping records the fact that the + Nix expression stored at path $p_1$ has a successor expression + stored at path $p_2$. + + Note that a term $y$ is a successor of $x$ iff there exists a + sequence of rewrite steps that rewrites $x$ into $y$. +*/ +static TableId dbSuccessors; + +/* dbSuccessorsRev :: Path -> [Path] + + The reverse mapping of dbSuccessors (i.e., it stores the + predecessors of a Nix expression). +*/ +static TableId dbSuccessorsRev; + +/* dbSubstitutes :: Path -> [Path] + + Each pair $(p, [ps])$ tells Nix that it can realise any of the + Nix expressions stored at paths $ps$ to produce a path $p$. + + The main purpose of this is for distributed caching of derivates. + One system can compute a derivate and put it on a website (as a Nix + archive), for instance, and then another system can register a + substitute for that derivate. The substitute in this case might be + a Nix expression that fetches the Nix archive. +*/ +static TableId dbSubstitutes; + +/* dbSubstitutesRev :: Path -> [Path] + + The reverse mapping of dbSubstitutes. +*/ +static TableId dbSubstitutesRev; + + +void openDB() +{ + nixDB.open(nixDBPath); + dbValidPaths = nixDB.openTable("validpaths"); + dbSuccessors = nixDB.openTable("successors"); + dbSuccessorsRev = nixDB.openTable("successors-rev"); + dbSubstitutes = nixDB.openTable("substitutes"); + dbSubstitutesRev = nixDB.openTable("substitutes-rev"); +} + + +void initDB() +{ +} + + +void createStoreTransaction(Transaction & txn) +{ + Transaction txn2(nixDB); + txn2.moveTo(txn); +} + + +/* Path copying. */ + +struct CopySink : DumpSink +{ + int fd; + virtual void operator () (const unsigned char * data, unsigned int len) + { + writeFull(fd, data, len); + } +}; + + +struct CopySource : RestoreSource +{ + int fd; + virtual void operator () (unsigned char * data, unsigned int len) + { + readFull(fd, data, len); + } +}; + + +void copyPath(const Path & src, const Path & dst) +{ + debug(format("copying `%1%' to `%2%'") % src % dst); + + /* Unfortunately C++ doesn't support coprocedures, so we have no + nice way to chain CopySink and CopySource together. Instead we + fork off a child to run the sink. (Fork-less platforms should + use a thread). */ + + /* Create a pipe. */ + int fds[2]; + if (pipe(fds) == -1) throw SysError("creating pipe"); + + /* Fork. */ + pid_t pid; + switch (pid = fork()) { + + case -1: + throw SysError("unable to fork"); + + case 0: /* child */ + try { + close(fds[1]); + CopySource source; + source.fd = fds[0]; + restorePath(dst, source); + _exit(0); + } catch (exception & e) { + cerr << "error: " << e.what() << endl; + } + _exit(1); + } + + close(fds[0]); + + /* Parent. */ + + CopySink sink; + sink.fd = fds[1]; + dumpPath(src, sink); + + /* Wait for the child to finish. */ + int status; + if (waitpid(pid, &status, 0) != pid) + throw SysError("waiting for child"); + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) + throw Error("cannot copy file: child died"); +} + + +void registerSuccessor(const Transaction & txn, + const Path & srcPath, const Path & sucPath) +{ + Path known; + if (nixDB.queryString(txn, dbSuccessors, srcPath, known) && + known != sucPath) + { + throw Error(format( + "the `impossible' happened: expression in path " + "`%1%' appears to have multiple successors " + "(known `%2%', new `%3%'") + % srcPath % known % sucPath); + } + + Paths revs; + nixDB.queryStrings(txn, dbSuccessorsRev, sucPath, revs); + if (find(revs.begin(), revs.end(), srcPath) == revs.end()) + revs.push_back(srcPath); + + nixDB.setString(txn, dbSuccessors, srcPath, sucPath); + nixDB.setStrings(txn, dbSuccessorsRev, sucPath, revs); +} + + +bool querySuccessor(const Path & srcPath, Path & sucPath) +{ + return nixDB.queryString(noTxn, dbSuccessors, srcPath, sucPath); +} + + +Paths queryPredecessors(const Path & sucPath) +{ + Paths revs; + nixDB.queryStrings(noTxn, dbSuccessorsRev, sucPath, revs); + return revs; +} + + +void registerSubstitute(const Path & srcPath, const Path & subPath) +{ + Transaction txn(nixDB); + + Paths subs; + nixDB.queryStrings(txn, dbSubstitutes, srcPath, subs); + + if (find(subs.begin(), subs.end(), subPath) != subs.end()) { + /* Nothing to do if the substitute is already known. */ + txn.abort(); + return; + } + subs.push_front(subPath); /* new substitutes take precedence */ + + Paths revs; + nixDB.queryStrings(txn, dbSubstitutesRev, subPath, revs); + if (find(revs.begin(), revs.end(), srcPath) == revs.end()) + revs.push_back(srcPath); + + nixDB.setStrings(txn, dbSubstitutes, srcPath, subs); + nixDB.setStrings(txn, dbSubstitutesRev, subPath, revs); + + txn.commit(); +} + + +Paths querySubstitutes(const Path & srcPath) +{ + Paths subPaths; + nixDB.queryStrings(noTxn, dbSubstitutes, srcPath, subPaths); + return subPaths; +} + + +void registerValidPath(const Transaction & txn, const Path & _path) +{ + Path path(canonPath(_path)); + debug(format("registering path `%1%'") % path); + nixDB.setString(txn, dbValidPaths, path, ""); +} + + +bool isValidPath(const Path & path) +{ + string s; + return nixDB.queryString(noTxn, dbValidPaths, path, s); +} + + +void unregisterValidPath(const Path & _path) +{ + Path path(canonPath(_path)); + Transaction txn(nixDB); + + debug(format("unregistering path `%1%'") % path); + + nixDB.delPair(txn, dbValidPaths, path); + + txn.commit(); +} + + +static bool isInPrefix(const string & path, const string & _prefix) +{ + string prefix = canonPath(_prefix + "/"); + return string(path, 0, prefix.size()) == prefix; +} + + +Path addToStore(const Path & _srcPath) +{ + Path srcPath(absPath(_srcPath)); + debug(format("adding `%1%' to the store") % srcPath); + + Hash h = hashPath(srcPath); + + string baseName = baseNameOf(srcPath); + Path dstPath = canonPath(nixStore + "/" + (string) h + "-" + baseName); + + if (!isValidPath(dstPath)) { + + /* The first check above is an optimisation to prevent + unnecessary lock acquisition. */ + + PathSet lockPaths; + lockPaths.insert(dstPath); + PathLocks outputLock(lockPaths); + + if (!isValidPath(dstPath)) { + copyPath(srcPath, dstPath); + + Transaction txn(nixDB); + registerValidPath(txn, dstPath); + txn.commit(); + } + } + + return dstPath; +} + + +void addTextToStore(const Path & dstPath, const string & s) +{ + if (!isValidPath(dstPath)) { + + PathSet lockPaths; + lockPaths.insert(dstPath); + PathLocks outputLock(lockPaths); + + if (!isValidPath(dstPath)) { + + AutoCloseFD fd = open(dstPath.c_str(), O_CREAT | O_EXCL | O_WRONLY, 0666); + if (fd == -1) throw SysError(format("creating store file `%1%'") % dstPath); + + writeFull(fd, (unsigned char *) s.c_str(), s.size()); + + Transaction txn(nixDB); + registerValidPath(txn, dstPath); + txn.commit(); + } + } +} + + +void deleteFromStore(const Path & _path) +{ + Path path(canonPath(_path)); + + if (!isInPrefix(path, nixStore)) + throw Error(format("path `%1%' is not in the store") % path); + + unregisterValidPath(path); + + deletePath(path); +} + + +void verifyStore() +{ + Transaction txn(nixDB); + + Paths paths; + nixDB.enumTable(txn, dbValidPaths, paths); + + for (Paths::iterator i = paths.begin(); + i != paths.end(); i++) + { + Path path = *i; + if (!pathExists(path)) { + debug(format("path `%1%' disappeared") % path); + nixDB.delPair(txn, dbValidPaths, path); + nixDB.delPair(txn, dbSuccessorsRev, path); + nixDB.delPair(txn, dbSubstitutesRev, path); + } + } + +#if 0 + Strings subs; + nixDB.enumTable(txn, dbSubstitutes, subs); + + for (Strings::iterator i = subs.begin(); + i != subs.end(); i++) + { + FSId srcId = parseHash(*i); + + Strings subIds; + nixDB.queryStrings(txn, dbSubstitutes, srcId, subIds); + + for (Strings::iterator j = subIds.begin(); + j != subIds.end(); ) + { + FSId subId = parseHash(*j); + + Strings subPaths; + nixDB.queryStrings(txn, dbId2Paths, subId, subPaths); + if (subPaths.size() == 0) { + debug(format("erasing substitute %1% for %2%") + % (string) subId % (string) srcId); + j = subIds.erase(j); + } else j++; + } + + nixDB.setStrings(txn, dbSubstitutes, srcId, subIds); + } +#endif + + Paths sucs; + nixDB.enumTable(txn, dbSuccessors, sucs); + + for (Paths::iterator i = sucs.begin(); i != sucs.end(); i++) { + Path srcPath = *i; + + Path sucPath; + if (!nixDB.queryString(txn, dbSuccessors, srcPath, sucPath)) abort(); + + Paths revs; + nixDB.queryStrings(txn, dbSuccessorsRev, sucPath, revs); + + if (find(revs.begin(), revs.end(), srcPath) == revs.end()) { + debug(format("reverse successor mapping from `%1%' to `%2%' missing") + % srcPath % sucPath); + revs.push_back(srcPath); + nixDB.setStrings(txn, dbSuccessorsRev, sucPath, revs); + } + } + + txn.commit(); +} diff --git a/src/libstore/store.hh b/src/libstore/store.hh new file mode 100644 index 000000000000..dab3d603f802 --- /dev/null +++ b/src/libstore/store.hh @@ -0,0 +1,72 @@ +#ifndef __STORE_H +#define __STORE_H + +#include + +#include "hash.hh" +#include "db.hh" + +using namespace std; + + +/* Open the database environment. */ +void openDB(); + +/* Create the required database tables. */ +void initDB(); + +/* Get a transaction object. */ +void createStoreTransaction(Transaction & txn); + +/* Copy a path recursively. */ +void copyPath(const Path & src, const Path & dst); + +/* Register a successor. This function accepts a transaction handle + so that it can be enclosed in an atomic operation with calls to + registerValidPath(). This must be atomic, since if we register a + successor for a derivation without registering the paths built in + the derivation, we have a successor with dangling pointers, and if + we do it in reverse order, we can get an obstructed build (since to + rebuild the successor, the outputs paths must not exist). */ +void registerSuccessor(const Transaction & txn, + const Path & srcPath, const Path & sucPath); + +/* Return the predecessors of the Nix expression stored at the given + path. */ +bool querySuccessor(const Path & srcPath, Path & sucPath); + +/* Return the predecessors of the Nix expression stored at the given + path. */ +Paths queryPredecessors(const Path & sucPath); + +/* Register a substitute. */ +void registerSubstitute(const Path & srcPath, const Path & subPath); + +/* Return the substitutes expression for the given path. */ +Paths querySubstitutes(const Path & srcPath); + +/* Register the validity of a path. */ +void registerValidPath(const Transaction & txn, const Path & path); + +/* Unregister the validity of a path. */ +void unregisterValidPath(const Path & path); + +/* Checks whether a path is valid. */ +bool isValidPath(const Path & path); + +/* Copy the contents of a path to the store and register the validity + the resulting path. The resulting path is returned. */ +Path addToStore(const Path & srcPath); + +/* Like addToStore, but the path of the output is given, and the + contents written to the output path is a regular file containing + the given string. */ +void addTextToStore(const Path & dstPath, const string & s); + +/* Delete a value from the nixStore directory. */ +void deleteFromStore(const Path & path); + +void verifyStore(); + + +#endif /* !__STORE_H */ diff --git a/src/libstore/test-builder-1.sh b/src/libstore/test-builder-1.sh new file mode 100755 index 000000000000..80e23354c3b9 --- /dev/null +++ b/src/libstore/test-builder-1.sh @@ -0,0 +1,3 @@ +#! /bin/sh + +echo "Hello World" > $out diff --git a/src/libstore/test-builder-2.sh b/src/libstore/test-builder-2.sh new file mode 100755 index 000000000000..0794fa96a401 --- /dev/null +++ b/src/libstore/test-builder-2.sh @@ -0,0 +1,9 @@ +#! /bin/sh + +echo "builder 2" + +/bin/mkdir $out || exit 1 +cd $out || exit 1 +echo "Hallo Wereld" > bla +echo $builder >> bla +echo $out >> bla diff --git a/src/libstore/test.cc b/src/libstore/test.cc new file mode 100644 index 000000000000..457fecf245f0 --- /dev/null +++ b/src/libstore/test.cc @@ -0,0 +1,162 @@ +#include + +#include +#include + +#include "hash.hh" +#include "archive.hh" +#include "util.hh" +#include "normalise.hh" +#include "globals.hh" + + +void realise(Path nePath) +{ + Nest nest(lvlDebug, format("TEST: realising `%1%'") % nePath); + realiseClosure(normaliseNixExpr(nePath)); +} + + +struct MySink : DumpSink +{ + virtual void operator () (const unsigned char * data, unsigned int len) + { + /* Don't use cout, it's slow as hell! */ + writeFull(STDOUT_FILENO, data, len); + } +}; + + +struct MySource : RestoreSource +{ + virtual void operator () (unsigned char * data, unsigned int len) + { + readFull(STDIN_FILENO, data, len); + } +}; + + +void runTests() +{ + verbosity = (Verbosity) 100; + + /* Hashing. */ + string s = "0b0ffd0538622bfe20b92c4aa57254d9"; + Hash h = parseHash(s); + if ((string) h != s) abort(); + + try { + h = parseHash("blah blah"); + abort(); + } catch (Error err) { }; + + try { + h = parseHash("0b0ffd0538622bfe20b92c4aa57254d99"); + abort(); + } catch (Error err) { }; + + /* Path canonicalisation. */ + cout << canonPath("/./../././//") << endl; + cout << canonPath("/foo/bar") << endl; + cout << canonPath("///foo/////bar//") << endl; + cout << canonPath("/././/foo/////bar//.") << endl; + cout << canonPath("/foo////bar//..///x/") << endl; + cout << canonPath("/foo////bar//..//..//x/y/../z/") << endl; + cout << canonPath("/foo/bar/../../../..///") << endl; + + /* Dumping. */ + +#if 0 + MySink sink; + dumpPath("scratch", sink); + cout << (string) hashPath("scratch") << endl; +#endif + + /* Restoring. */ +#if 0 + MySource source; + restorePath("outdir", source); + cout << (string) hashPath("outdir") << endl; + return; +#endif + + /* Set up the test environment. */ + + mkdir("scratch", 0777); + mkdir("scratch/db", 0777); + + string testDir = absPath("scratch"); + cout << testDir << endl; + + nixStore = testDir; + nixLogDir = testDir; + nixDBPath = testDir + "/db"; + + openDB(); + initDB(); + + /* Expression evaluation. */ + + Path builder1fn; + builder1fn = addToStore("./test-builder-1.sh"); + + ATerm fs1 = ATmake( + "Closure([], [(, [])])", + builder1fn.c_str(), + builder1fn.c_str()); + Path fs1ne = writeTerm(fs1, "-c"); + + realise(fs1ne); + realise(fs1ne); + + string out1h = hashString("foo"); /* !!! bad */ + Path out1fn = nixStore + "/" + (string) out1h + "-hello.txt"; + ATerm fs3 = ATmake( + "Derive([], [], , , [], [(\"out\", )])", + out1fn.c_str(), + fs1ne.c_str(), + thisSystem.c_str(), + builder1fn.c_str(), + out1fn.c_str()); + debug(printTerm(fs3)); + Path fs3ne = writeTerm(fs3, "-d"); + + realise(fs3ne); + realise(fs3ne); + + + Path builder4fn = addToStore("./test-builder-2.sh"); + + ATerm fs4 = ATmake( + "Closure([], [(, [])])", + builder4fn.c_str(), + builder4fn.c_str()); + Path fs4ne = writeTerm(fs4, "-c"); + + realise(fs4ne); + + string out5h = hashString("bar"); /* !!! bad */ + Path out5fn = nixStore + "/" + (string) out5h + "-hello2"; + ATerm fs5 = ATmake( + "Derive([], [], , , [], [(\"out\", ), (\"builder\", )])", + out5fn.c_str(), + fs4ne.c_str(), + thisSystem.c_str(), + builder4fn.c_str(), + out5fn.c_str(), + builder4fn.c_str()); + debug(printTerm(fs5)); + Path fs5ne = writeTerm(fs5, "-d"); + + realise(fs5ne); + realise(fs5ne); +} + + +void run(Strings args) +{ + runTests(); +} + + +string programId = "test"; diff --git a/src/nix-hash/Makefile.am b/src/nix-hash/Makefile.am index 7377a62e7c29..8609cb216453 100644 --- a/src/nix-hash/Makefile.am +++ b/src/nix-hash/Makefile.am @@ -1,8 +1,8 @@ bin_PROGRAMS = nix-hash nix_hash_SOURCES = nix-hash.cc -nix_hash_LDADD = ../libmain/libmain.a ../libnix/libnix.a ../libutil/libutil.a \ +nix_hash_LDADD = ../libmain/libmain.a ../libstore/libstore.a ../libutil/libutil.a \ ../boost/format/libformat.a -L../../externals/inst/lib -ldb_cxx -lATerm AM_CXXFLAGS = \ - -I.. -I../../externals/inst/include -I../libutil -I../libnix -I../libmain + -I.. -I../../externals/inst/include -I../libutil -I../libstore -I../libmain diff --git a/src/nix/Makefile.am b/src/nix/Makefile.am index c4368a37c6dd..8a52be364f1a 100644 --- a/src/nix/Makefile.am +++ b/src/nix/Makefile.am @@ -1,7 +1,7 @@ bin_PROGRAMS = nix nix_SOURCES = nix.cc dotgraph.cc -nix_LDADD = ../libmain/libmain.a ../libnix/libnix.a ../libutil/libutil.a \ +nix_LDADD = ../libmain/libmain.a ../libstore/libstore.a ../libutil/libutil.a \ ../boost/format/libformat.a -L../../externals/inst/lib -ldb_cxx -lATerm nix.o: nix-help.txt.hh @@ -12,7 +12,7 @@ nix.o: nix-help.txt.hh echo '"' >> $@ AM_CXXFLAGS = \ - -I.. -I../../externals/inst/include -I../libutil -I../libnix -I../libmain + -I.. -I../../externals/inst/include -I../libutil -I../libstore -I../libmain install-data-local: $(INSTALL) -d $(localstatedir)/nix -- cgit 1.4.1