diff options
| -rw-r--r-- | src/Makefile.am | 1 | ||||
| -rw-r--r-- | src/db.cc | 160 | ||||
| -rw-r--r-- | src/db.hh | 71 | ||||
| -rw-r--r-- | src/fix.cc | 2 | ||||
| -rw-r--r-- | src/globals.cc | 19 | ||||
| -rw-r--r-- | src/globals.hh | 17 | ||||
| -rw-r--r-- | src/nix.cc | 2 | ||||
| -rw-r--r-- | src/normalise.cc | 12 | ||||
| -rw-r--r-- | src/shared.cc | 2 | ||||
| -rw-r--r-- | src/store.cc | 57 | ||||
| -rw-r--r-- | src/test.cc | 6 | ||||
| -rwxr-xr-x | testpkgs/slow2/slow-build.sh | 4 |
12 files changed, 251 insertions, 102 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index cc51eff36577..847f3eb19fa9 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -42,6 +42,7 @@ nix.o: nix-help.txt.hh install-data-local: $(INSTALL) -d $(localstatedir)/nix + $(INSTALL) -d $(localstatedir)/nix/db $(INSTALL) -d $(localstatedir)/nix/links ln -sf $(localstatedir)/nix/links/current $(prefix)/current $(INSTALL) -d $(localstatedir)/log/nix diff --git a/src/db.cc b/src/db.cc index a8741342ce92..4362fe405583 100644 --- a/src/db.cc +++ b/src/db.cc @@ -3,66 +3,142 @@ #include <memory> -#include <db_cxx.h> - -/* Wrapper classes that ensures that the database is closed upon - object destruction. */ -class Db2 : public Db +/* Wrapper class to ensure proper destruction. */ +class DestroyDb { + Db * db; public: - Db2(DbEnv *env, u_int32_t flags) : Db(env, flags) { } - ~Db2() { close(0); } + DestroyDb(Db * _db) : db(_db) { } + ~DestroyDb() { db->close(0); delete db; } }; -class DbcClose +class DestroyDbc { - Dbc * cursor; + Dbc * dbc; public: - DbcClose(Dbc * c) : cursor(c) { } - ~DbcClose() { cursor->close(); } + DestroyDbc(Dbc * _dbc) : dbc(_dbc) { } + ~DestroyDbc() { dbc->close(); /* close() frees dbc */ } }; -static auto_ptr<Db2> openDB(const string & filename, const string & dbname, - bool readonly) +static void rethrow(DbException & e) +{ + throw Error(e.what()); +} + + +Transaction::Transaction() + : txn(0) { - auto_ptr<Db2> db(new Db2(0, 0)); +} + + +Transaction::Transaction(Database & db) +{ + db.requireEnv(); + db.env->txn_begin(0, &txn, 0); +} + + +Transaction::~Transaction() +{ + if (txn) { + txn->abort(); + txn = 0; + } +} + + +void Transaction::commit() +{ + if (!txn) throw Error("commit called on null transaction"); + txn->commit(0); + txn = 0; +} + + +void Database::requireEnv() +{ + if (!env) throw Error("database environment not open"); +} - db->open(filename.c_str(), dbname.c_str(), - DB_HASH, readonly ? DB_RDONLY : DB_CREATE, 0666); + +Db * Database::openDB(const Transaction & txn, + const string & table, bool create) +{ + requireEnv(); + + Db * db = new Db(env, 0); + + try { + // !!! fixme when switching to BDB 4.1: use txn. + db->open(table.c_str(), 0, + DB_HASH, create ? DB_CREATE : 0, 0666); + } catch (...) { + delete db; + throw; + } return db; } -static void rethrow(DbException & e) +Database::Database() + : env(0) { - throw Error(e.what()); } -void createDB(const string & filename, const string & dbname) +Database::~Database() +{ + if (env) { + env->close(0); + delete env; + } +} + + +void Database::open(const string & path) +{ + try { + + if (env) throw Error(format("environment already open")); + + env = new DbEnv(0); + + debug("foo" + path); + env->open(path.c_str(), + DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | + DB_CREATE, + 0666); + + } catch (DbException e) { rethrow(e); } +} + + +void Database::createTable(const string & table) { try { - openDB(filename, dbname, false); + Db * db = openDB(noTxn, table, true); + DestroyDb destroyDb(db); } catch (DbException e) { rethrow(e); } } -bool queryDB(const string & filename, const string & dbname, +bool Database::queryString(const Transaction & txn, const string & table, const string & key, string & data) { try { - int err; - auto_ptr<Db2> db = openDB(filename, dbname, true); + Db * db = openDB(txn, table, false); + DestroyDb destroyDb(db); Dbt kt((void *) key.c_str(), key.length()); Dbt dt; - err = db->get(0, &kt, &dt, 0); + int err = db->get(txn.txn, &kt, &dt, 0); if (err) return false; if (!dt.get_data()) @@ -76,12 +152,12 @@ bool queryDB(const string & filename, const string & dbname, } -bool queryListDB(const string & filename, const string & dbname, +bool Database::queryStrings(const Transaction & txn, const string & table, const string & key, Strings & data) { string d; - if (!queryDB(filename, dbname, key, d)) + if (!queryString(txn, table, key, d)) return false; string::iterator it = d.begin(); @@ -110,19 +186,21 @@ bool queryListDB(const string & filename, const string & dbname, } -void setDB(const string & filename, const string & dbname, +void Database::setString(const Transaction & txn, const string & table, const string & key, const string & data) { try { - auto_ptr<Db2> db = openDB(filename, dbname, false); + Db * db = openDB(txn, table, false); + DestroyDb destroyDb(db); + Dbt kt((void *) key.c_str(), key.length()); Dbt dt((void *) data.c_str(), data.length()); - db->put(0, &kt, &dt, 0); + db->put(txn.txn, &kt, &dt, 0); } catch (DbException e) { rethrow(e); } } -void setListDB(const string & filename, const string & dbname, +void Database::setStrings(const Transaction & txn, const string & table, const string & key, const Strings & data) { string d; @@ -141,34 +219,36 @@ void setListDB(const string & filename, const string & dbname, d += s; } - setDB(filename, dbname, key, d); + setString(txn, table, key, d); } -void delDB(const string & filename, const string & dbname, +void Database::delPair(const Transaction & txn, const string & table, const string & key) { try { - auto_ptr<Db2> db = openDB(filename, dbname, false); + Db * db = openDB(txn, table, false); + DestroyDb destroyDb(db); Dbt kt((void *) key.c_str(), key.length()); - db->del(0, &kt, 0); + db->del(txn.txn, &kt, 0); } catch (DbException e) { rethrow(e); } } -void enumDB(const string & filename, const string & dbname, +void Database::enumTable(const Transaction & txn, const string & table, Strings & keys) { try { - auto_ptr<Db2> db = openDB(filename, dbname, true); + Db * db = openDB(txn, table, false); + DestroyDb destroyDb(db); - Dbc * cursor; - db->cursor(0, &cursor, 0); - DbcClose cursorCloser(cursor); + Dbc * dbc; + db->cursor(0, &dbc, 0); + DestroyDbc destroyDbc(dbc); Dbt kt, dt; - while (cursor->get(&kt, &dt, DB_NEXT) != DB_NOTFOUND) + while (dbc->get(&kt, &dt, DB_NEXT) != DB_NOTFOUND) keys.push_back( string((char *) kt.get_data(), kt.get_size())); diff --git a/src/db.hh b/src/db.hh index 6f13eb30c813..57b6e4d8ebfb 100644 --- a/src/db.hh +++ b/src/db.hh @@ -4,28 +4,73 @@ #include <string> #include <list> +#include <db_cxx.h> + #include "util.hh" using namespace std; -void createDB(const string & filename, const string & dbname); -bool queryDB(const string & filename, const string & dbname, - const string & key, string & data); +class Database; + + +class Transaction +{ + friend class Database; + +private: + DbTxn * txn; + +public: + Transaction(); + Transaction(Database & _db); + ~Transaction(); + + void commit(); +}; + + +#define noTxn Transaction() + + +class Database +{ + friend class Transaction; + +private: + DbEnv * env; + + void requireEnv(); + + Db * openDB(const Transaction & txn, + const string & table, bool create); + +public: + Database(); + ~Database(); + + void open(const string & path); + + void createTable(const string & table); + + bool queryString(const Transaction & txn, const string & table, + const string & key, string & data); + + bool queryStrings(const Transaction & txn, const string & table, + const string & key, Strings & data); -bool queryListDB(const string & filename, const string & dbname, - const string & key, Strings & data); + void setString(const Transaction & txn, const string & table, + const string & key, const string & data); -void setDB(const string & filename, const string & dbname, - const string & key, const string & data); + void setStrings(const Transaction & txn, const string & table, + const string & key, const Strings & data); -void setListDB(const string & filename, const string & dbname, - const string & key, const Strings & data); + void delPair(const Transaction & txn, const string & table, + const string & key); -void delDB(const string & filename, const string & dbname, - const string & key); + void enumTable(const Transaction & txn, const string & table, + Strings & keys); +}; -void enumDB(const string & filename, const string & dbname, - Strings & keys); #endif /* !__DB_H */ diff --git a/src/fix.cc b/src/fix.cc index 9b0d95912f10..4235165ccdd4 100644 --- a/src/fix.cc +++ b/src/fix.cc @@ -301,6 +301,8 @@ static Expr evalFile(EvalState & state, string relPath) void run(Strings args) { + openDB(); + EvalState state; Strings files; diff --git a/src/globals.cc b/src/globals.cc index 1edb38f7483e..8c3ec38283ec 100644 --- a/src/globals.cc +++ b/src/globals.cc @@ -2,6 +2,9 @@ #include "db.hh" +Database nixDB; + + string dbPath2Id = "path2id"; string dbId2Paths = "id2paths"; string dbSuccessors = "successors"; @@ -11,13 +14,19 @@ string dbSubstitutes = "substitutes"; string nixStore = "/UNINIT"; string nixDataDir = "/UNINIT"; string nixLogDir = "/UNINIT"; -string nixDB = "/UNINIT"; +string nixDBPath = "/UNINIT"; + + +void openDB() +{ + nixDB.open(nixDBPath); +} void initDB() { - createDB(nixDB, dbPath2Id); - createDB(nixDB, dbId2Paths); - createDB(nixDB, dbSuccessors); - createDB(nixDB, dbSubstitutes); + nixDB.createTable(dbPath2Id); + nixDB.createTable(dbId2Paths); + nixDB.createTable(dbSuccessors); + nixDB.createTable(dbSubstitutes); } diff --git a/src/globals.hh b/src/globals.hh index fbb020df7c91..9df827622237 100644 --- a/src/globals.hh +++ b/src/globals.hh @@ -3,10 +3,15 @@ #include <string> +#include "db.hh" + using namespace std; -/* Database names. */ +extern Database nixDB; + + +/* Database tables. */ /* dbPath2Id :: Path -> FSId @@ -60,12 +65,14 @@ extern string nixDataDir; /* !!! fix */ /* nixLogDir is the directory where we log various operations. */ extern string nixLogDir; -/* nixDB is the file name of the Berkeley DB database where we - maintain the dbXXX mappings. */ -extern string nixDB; +/* nixDBPath is the path name of our Berkeley DB environment. */ +extern string nixDBPath; + +/* Open the database environment. */ +void openDB(); -/* Initialize the databases. */ +/* Create the required database tables. */ void initDB(); diff --git a/src/nix.cc b/src/nix.cc index 68d01f2f87a3..42cc4a87c228 100644 --- a/src/nix.cc +++ b/src/nix.cc @@ -335,6 +335,8 @@ static void opVerify(Strings opFlags, Strings opArgs) list. */ void run(Strings args) { + openDB(); + Strings opFlags, opArgs; Operation op = 0; diff --git a/src/normalise.cc b/src/normalise.cc index 5ef4d82ac139..5a8cb9a0d0ff 100644 --- a/src/normalise.cc +++ b/src/normalise.cc @@ -9,7 +9,7 @@ void registerSuccessor(const FSId & id1, const FSId & id2) { - setDB(nixDB, dbSuccessors, id1, id2); + nixDB.setString(noTxn, dbSuccessors, id1, id2); } @@ -31,7 +31,7 @@ FSId normaliseFState(FSId id, FSIdSet pending) /* Try to substitute $id$ by any known successors in order to speed up the rewrite process. */ string idSucc; - while (queryDB(nixDB, dbSuccessors, id, idSucc)) { + while (nixDB.queryString(noTxn, dbSuccessors, id, idSucc)) { debug(format("successor %1% -> %2%") % (string) id % idSucc); id = parseHash(idSucc); } @@ -191,7 +191,7 @@ void realiseSlice(const FSId & id, FSIdSet pending) { SliceElem elem = *i; string id; - if (!queryDB(nixDB, dbPath2Id, elem.path, id)) { + if (!nixDB.queryString(noTxn, dbPath2Id, elem.path, id)) { if (pathExists(elem.path)) throw Error(format("path `%1%' obstructed") % elem.path); missing = true; @@ -269,7 +269,7 @@ static void fstateRequisitesSet(const FSId & id, string idSucc; if (includeSuccessors && - queryDB(nixDB, dbSuccessors, id, idSucc)) + nixDB.queryString(noTxn, dbSuccessors, id, idSucc)) fstateRequisitesSet(parseHash(idSucc), includeExprs, includeSuccessors, paths); } @@ -293,13 +293,13 @@ FSIds findGenerators(const FSIds & _ids) mappings, since we know that those are Nix expressions. */ Strings sucs; - enumDB(nixDB, dbSuccessors, sucs); + nixDB.enumTable(noTxn, dbSuccessors, sucs); for (Strings::iterator i = sucs.begin(); i != sucs.end(); i++) { string s; - if (!queryDB(nixDB, dbSuccessors, *i, s)) continue; + if (!nixDB.queryString(noTxn, dbSuccessors, *i, s)) continue; FSId id = parseHash(s); FState fs; diff --git a/src/shared.cc b/src/shared.cc index 75145f6db2fa..c0f07e955c73 100644 --- a/src/shared.cc +++ b/src/shared.cc @@ -19,7 +19,7 @@ static void initAndRun(int argc, char * * argv) nixStore = NIX_STORE_DIR; nixDataDir = NIX_DATA_DIR; nixLogDir = NIX_LOG_DIR; - nixDB = (string) NIX_STATE_DIR + "/nixstate.db"; + nixDBPath = (string) NIX_STATE_DIR + "/db"; /* Put the arguments in a vector. */ Strings args; diff --git a/src/store.cc b/src/store.cc index 3493ba384bb6..3dc625a7b239 100644 --- a/src/store.cc +++ b/src/store.cc @@ -96,7 +96,7 @@ void registerSubstitute(const FSId & srcId, const FSId & subId) /* For now, accept only one substitute per id. */ Strings subs; subs.push_back(subId); - setListDB(nixDB, dbSubstitutes, srcId, subs); + nixDB.setStrings(noTxn, dbSubstitutes, srcId, subs); } @@ -104,10 +104,10 @@ void registerPath(const string & _path, const FSId & id) { string path(canonPath(_path)); - setDB(nixDB, dbPath2Id, path, id); + nixDB.setString(noTxn, dbPath2Id, path, id); Strings paths; - queryListDB(nixDB, dbId2Paths, id, paths); /* non-existence = ok */ + nixDB.queryStrings(noTxn, dbId2Paths, id, paths); /* non-existence = ok */ for (Strings::iterator it = paths.begin(); it != paths.end(); it++) @@ -115,7 +115,7 @@ void registerPath(const string & _path, const FSId & id) paths.push_back(path); - setListDB(nixDB, dbId2Paths, id, paths); + nixDB.setStrings(noTxn, dbId2Paths, id, paths); } @@ -124,16 +124,15 @@ void unregisterPath(const string & _path) string path(canonPath(_path)); string _id; - if (!queryDB(nixDB, dbPath2Id, path, _id)) - return; + if (!nixDB.queryString(noTxn, dbPath2Id, path, _id)) return; FSId id(parseHash(_id)); - delDB(nixDB, dbPath2Id, path); + nixDB.delPair(noTxn, dbPath2Id, path); /* begin transaction */ Strings paths, paths2; - queryListDB(nixDB, dbId2Paths, id, paths); /* non-existence = ok */ + nixDB.queryStrings(noTxn, dbId2Paths, id, paths); /* non-existence = ok */ bool changed = false; for (Strings::iterator it = paths.begin(); @@ -141,7 +140,7 @@ void unregisterPath(const string & _path) if (*it != path) paths2.push_back(*it); else changed = true; if (changed) - setListDB(nixDB, dbId2Paths, id, paths2); + nixDB.setStrings(noTxn, dbId2Paths, id, paths2); /* end transaction */ @@ -151,7 +150,7 @@ void unregisterPath(const string & _path) bool queryPathId(const string & path, FSId & id) { string s; - if (!queryDB(nixDB, dbPath2Id, absPath(path), s)) return false; + if (!nixDB.queryString(noTxn, dbPath2Id, absPath(path), s)) return false; id = parseHash(s); return true; } @@ -174,7 +173,7 @@ string expandId(const FSId & id, const string & target, if (!target.empty() && !isInPrefix(target, prefix)) abort(); - queryListDB(nixDB, dbId2Paths, id, paths); + nixDB.queryStrings(noTxn, dbId2Paths, id, paths); /* Pick one equal to `target'. */ if (!target.empty()) { @@ -212,7 +211,7 @@ string expandId(const FSId & id, const string & target, /* Try to realise the substitutes, but only if this id is not already being realised by a substitute. */ Strings subs; - queryListDB(nixDB, dbSubstitutes, id, subs); /* non-existence = ok */ + nixDB.queryStrings(noTxn, dbSubstitutes, id, subs); /* non-existence = ok */ for (Strings::iterator it = subs.begin(); it != subs.end(); it++) { FSId subId = parseHash(*it); @@ -264,7 +263,7 @@ void deleteFromStore(const string & path) void verifyStore() { Strings paths; - enumDB(nixDB, dbPath2Id, paths); + nixDB.enumTable(noTxn, dbPath2Id, paths); for (Strings::iterator i = paths.begin(); i != paths.end(); i++) @@ -278,10 +277,10 @@ void verifyStore() else { string id; - if (!queryDB(nixDB, dbPath2Id, path, id)) abort(); + if (!nixDB.queryString(noTxn, dbPath2Id, path, id)) abort(); Strings idPaths; - queryListDB(nixDB, dbId2Paths, id, idPaths); + nixDB.queryStrings(noTxn, dbId2Paths, id, idPaths); bool found = false; for (Strings::iterator j = idPaths.begin(); @@ -298,11 +297,11 @@ void verifyStore() debug(format("reverse mapping for path `%1%' missing") % path); } - if (erase) delDB(nixDB, dbPath2Id, path); + if (erase) nixDB.delPair(noTxn, dbPath2Id, path); } Strings ids; - enumDB(nixDB, dbId2Paths, ids); + nixDB.enumTable(noTxn, dbId2Paths, ids); for (Strings::iterator i = ids.begin(); i != ids.end(); i++) @@ -310,13 +309,13 @@ void verifyStore() FSId id = parseHash(*i); Strings idPaths; - queryListDB(nixDB, dbId2Paths, id, idPaths); + nixDB.queryStrings(noTxn, dbId2Paths, id, idPaths); for (Strings::iterator j = idPaths.begin(); j != idPaths.end(); ) { string id2; - if (!queryDB(nixDB, dbPath2Id, *j, id2) || + if (!nixDB.queryString(noTxn, dbPath2Id, *j, id2) || id != parseHash(id2)) { debug(format("erasing path `%1%' from mapping for id %2%") % *j % (string) id); @@ -324,12 +323,12 @@ void verifyStore() } else j++; } - setListDB(nixDB, dbId2Paths, id, idPaths); + nixDB.setStrings(noTxn, dbId2Paths, id, idPaths); } Strings subs; - enumDB(nixDB, dbSubstitutes, subs); + nixDB.enumTable(noTxn, dbSubstitutes, subs); for (Strings::iterator i = subs.begin(); i != subs.end(); i++) @@ -337,7 +336,7 @@ void verifyStore() FSId srcId = parseHash(*i); Strings subIds; - queryListDB(nixDB, dbSubstitutes, srcId, subIds); + nixDB.queryStrings(noTxn, dbSubstitutes, srcId, subIds); for (Strings::iterator j = subIds.begin(); j != subIds.end(); ) @@ -345,7 +344,7 @@ void verifyStore() FSId subId = parseHash(*j); Strings subPaths; - queryListDB(nixDB, dbId2Paths, subId, subPaths); + nixDB.queryStrings(noTxn, dbId2Paths, subId, subPaths); if (subPaths.size() == 0) { debug(format("erasing substitute %1% for %2%") % (string) subId % (string) srcId); @@ -353,11 +352,11 @@ void verifyStore() } else j++; } - setListDB(nixDB, dbSubstitutes, srcId, subIds); + nixDB.setStrings(noTxn, dbSubstitutes, srcId, subIds); } Strings sucs; - enumDB(nixDB, dbSuccessors, sucs); + nixDB.enumTable(noTxn, dbSuccessors, sucs); for (Strings::iterator i = sucs.begin(); i != sucs.end(); i++) @@ -365,17 +364,17 @@ void verifyStore() FSId id1 = parseHash(*i); string id2; - if (!queryDB(nixDB, dbSuccessors, id1, id2)) abort(); + if (!nixDB.queryString(noTxn, dbSuccessors, id1, id2)) abort(); Strings id2Paths; - queryListDB(nixDB, dbId2Paths, id2, id2Paths); + nixDB.queryStrings(noTxn, dbId2Paths, id2, id2Paths); if (id2Paths.size() == 0) { Strings id2Subs; - queryListDB(nixDB, dbSubstitutes, id2, id2Subs); + nixDB.queryStrings(noTxn, dbSubstitutes, id2, id2Subs); if (id2Subs.size() == 0) { debug(format("successor %1% for %2% missing") % id2 % (string) id1); - delDB(nixDB, dbSuccessors, (string) id1); + nixDB.delPair(noTxn, dbSuccessors, (string) id1); } } } diff --git a/src/test.cc b/src/test.cc index cff094e2f89d..d640e335ae77 100644 --- a/src/test.cc +++ b/src/test.cc @@ -51,6 +51,8 @@ struct MySource : RestoreSource void runTests() { + verbosity = (Verbosity) 100; + /* Hashing. */ string s = "0b0ffd0538622bfe20b92c4aa57254d9"; Hash h = parseHash(s); @@ -94,14 +96,16 @@ void runTests() /* Set up the test environment. */ mkdir("scratch", 0777); + mkdir("scratch/db", 0777); string testDir = absPath("scratch"); cout << testDir << endl; nixStore = testDir; nixLogDir = testDir; - nixDB = testDir + "/db"; + nixDBPath = testDir + "/db"; + openDB(); initDB(); /* Expression evaluation. */ diff --git a/testpkgs/slow2/slow-build.sh b/testpkgs/slow2/slow-build.sh index 99e18c0fb779..4a9d7d48287d 100755 --- a/testpkgs/slow2/slow-build.sh +++ b/testpkgs/slow2/slow-build.sh @@ -2,13 +2,13 @@ echo "builder started..." -for i in $(seq 1 30); do +for i in $(seq 1 10); do echo $i sleep 1 done mkdir $out -echo "done" > $out/bla +echo "done" >> $out/bla echo "builder finished" |