From 01d56c1eeca497de247413a64a544605c53d9d41 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Aug 2012 22:34:46 -0400 Subject: Drop the block count in the garbage collector --- src/libutil/util.cc | 21 ++++++++------------- src/libutil/util.hh | 3 +-- 2 files changed, 9 insertions(+), 15 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 689fc543af31..9d8e4afed37d 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -297,8 +297,7 @@ void computePathSize(const Path & path, } -static void _deletePath(const Path & path, unsigned long long & bytesFreed, - unsigned long long & blocksFreed) +static void _deletePath(const Path & path, unsigned long long & bytesFreed) { checkInterrupt(); @@ -308,10 +307,8 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed, if (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode)) makeMutable(path); - if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) { - bytesFreed += st.st_size; - blocksFreed += st.st_blocks; - } + if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) + bytesFreed += st.st_blocks * 512; if (S_ISDIR(st.st_mode)) { Strings names = readDirectory(path); @@ -323,7 +320,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed, } for (Strings::iterator i = names.begin(); i != names.end(); ++i) - _deletePath(path + "/" + *i, bytesFreed, blocksFreed); + _deletePath(path + "/" + *i, bytesFreed); } if (remove(path.c_str()) == -1) @@ -333,19 +330,17 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed, void deletePath(const Path & path) { - unsigned long long dummy1, dummy2; - deletePath(path, dummy1, dummy2); + unsigned long long dummy; + deletePath(path, dummy); } -void deletePath(const Path & path, unsigned long long & bytesFreed, - unsigned long long & blocksFreed) +void deletePath(const Path & path, unsigned long long & bytesFreed) { startNest(nest, lvlDebug, format("recursively deleting path `%1%'") % path); bytesFreed = 0; - blocksFreed = 0; - _deletePath(path, bytesFreed, blocksFreed); + _deletePath(path, bytesFreed); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 9b8656f70485..edb7c0fa6c60 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -80,8 +80,7 @@ void computePathSize(const Path & path, returns the number of bytes and blocks freed. */ void deletePath(const Path & path); -void deletePath(const Path & path, unsigned long long & bytesFreed, - unsigned long long & blocksFreed); +void deletePath(const Path & path, unsigned long long & bytesFreed); /* Make a path read-only recursively. */ void makePathReadOnly(const Path & path); -- cgit 1.4.1 From e82767910c649f160d6701e47f606f3b8dde4b29 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 12 Aug 2012 23:29:28 -0400 Subject: Add some basic profiling support to the evaluator Setting the environment variable NIX_COUNT_CALLS to 1 enables some basic profiling in the evaluator. It will count calls to functions and primops as well as evaluations of attributes. For example, to see where evaluation of a NixOS configuration spends its time: $ NIX_SHOW_STATS=1 NIX_COUNT_CALLS=1 ./src/nix-instantiate/nix-instantiate '' -A system --readonly-mode ... calls to 39 primops: 239532 head 233962 tail 191252 hasAttr ... calls to 1595 functions: 224157 `/nix/var/nix/profiles/per-user/root/channels/nixos/nixpkgs/pkgs/lib/lists.nix:17:19' 221767 `/nix/var/nix/profiles/per-user/root/channels/nixos/nixpkgs/pkgs/lib/lists.nix:17:14' 221767 `/nix/var/nix/profiles/per-user/root/channels/nixos/nixpkgs/pkgs/lib/lists.nix:17:10' ... evaluations of 7088 attributes: 167377 undefined position 132459 `/nix/var/nix/profiles/per-user/root/channels/nixos/nixpkgs/pkgs/lib/attrsets.nix:119:41' 47322 `/nix/var/nix/profiles/per-user/root/channels/nixos/nixpkgs/pkgs/lib/attrsets.nix:13:21' ... --- src/libexpr/eval.cc | 38 +++++++++++++++++++++++++++++++++++++- src/libexpr/eval.hh | 16 ++++++++++++++-- src/libexpr/nixexpr.hh | 9 +++++++++ src/libexpr/primops.cc | 3 ++- src/libutil/util.hh | 3 +++ 5 files changed, 65 insertions(+), 4 deletions(-) (limited to 'src/libutil') diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index cf7c62ad20cf..2d41eba0e088 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -144,6 +144,7 @@ EvalState::EvalState() { nrEnvs = nrValuesInEnvs = nrValues = nrListElems = 0; nrAttrsets = nrOpUpdates = nrOpUpdateValuesCopied = 0; + countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0"; #if HAVE_BOEHMGC static bool gcInitialised = true; @@ -300,8 +301,10 @@ inline Value * EvalState::lookupVar(Env * env, const VarRef & var) if (var.fromWith) { while (1) { Bindings::iterator j = env->values[0]->attrs->find(var.name); - if (j != env->values[0]->attrs->end()) + if (j != env->values[0]->attrs->end()) { + if (countCalls && j->pos) attrSelects[*j->pos]++; return j->value; + } if (env->prevWith == 0) throwEvalError("undefined variable `%1%'", var.name); for (unsigned int l = env->prevWith; l; --l, env = env->up) ; @@ -619,8 +622,10 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) } vAttrs = j->value; pos = j->pos; + if (state.countCalls && pos) state.attrSelects[*pos]++; } + state.forceValue(*vAttrs); } catch (Error & e) { @@ -700,6 +705,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v) vArgs[n--] = arg->primOpApp.right; /* And call the primop. */ + if (countCalls) primOpCalls[primOp->primOp->name]++; try { primOp->primOp->fun(*this, vArgs, v); } catch (Error & e) { @@ -760,6 +766,8 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v) throwTypeError("function at %1% called with unexpected argument", fun.lambda.fun->pos); } + if (countCalls) functionCalls[fun.lambda.fun->pos]++; + try { fun.lambda.fun->body->eval(*this, env2, v); } catch (Error & e) { @@ -1216,6 +1224,34 @@ void EvalState::printStats() printMsg(v, format(" number of thunks: %1%") % nrThunks); printMsg(v, format(" number of thunks avoided: %1%") % nrAvoided); printMsg(v, format(" number of attr lookups: %1%") % nrLookups); + + if (countCalls) { + + printMsg(v, format("calls to %1% primops:") % primOpCalls.size()); + typedef std::multimap PrimOpCalls_; + std::multimap primOpCalls_; + foreach (PrimOpCalls::iterator, i, primOpCalls) + primOpCalls_.insert(std::pair(i->second, i->first)); + foreach_reverse (PrimOpCalls_::reverse_iterator, i, primOpCalls_) + printMsg(v, format("%1$10d %2%") % i->first % i->second); + + printMsg(v, format("calls to %1% functions:") % functionCalls.size()); + typedef std::multimap FunctionCalls_; + std::multimap functionCalls_; + foreach (FunctionCalls::iterator, i, functionCalls) + functionCalls_.insert(std::pair(i->second, i->first)); + foreach_reverse (FunctionCalls_::reverse_iterator, i, functionCalls_) + printMsg(v, format("%1$10d %2%") % i->first % i->second); + + printMsg(v, format("evaluations of %1% attributes:") % attrSelects.size()); + typedef std::multimap AttrSelects_; + std::multimap attrSelects_; + foreach (AttrSelects::iterator, i, attrSelects) + attrSelects_.insert(std::pair(i->second, i->first)); + foreach_reverse (AttrSelects_::reverse_iterator, i, attrSelects_) + printMsg(v, format("%1$10d %2%") % i->first % i->second); + + } } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 5103ae8cefe9..540f1e200ef3 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -244,9 +244,21 @@ private: unsigned long nrAttrsets; unsigned long nrOpUpdates; unsigned long nrOpUpdateValuesCopied; - - friend class RecursionCounter; + + bool countCalls; + + typedef std::map PrimOpCalls; + PrimOpCalls primOpCalls; + + typedef std::map FunctionCalls; + FunctionCalls functionCalls; + + typedef std::map AttrSelects; + AttrSelects attrSelects; + friend class ExprOpUpdate; + friend class ExprSelect; + friend void prim_getAttr(EvalState & state, Value * * args, Value & v); }; diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 4c1a0bb2d5fb..bc6c3287c79d 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -27,6 +27,15 @@ struct Pos Pos() : line(0), column(0) { }; Pos(const string & file, unsigned int line, unsigned int column) : file(file), line(line), column(column) { }; + bool operator < (const Pos & p2) const + { + int d = file.compare(p2.file); + if (d < 0) return true; + if (d > 0) return false; + if (line < p2.line) return true; + if (line > p2.line) return false; + return column < p2.column; + } }; extern Pos noPos; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 0d4efc47e6d6..b833413ea5ca 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -713,7 +713,7 @@ static void prim_attrNames(EvalState & state, Value * * args, Value & v) /* Dynamic version of the `.' operator. */ -static void prim_getAttr(EvalState & state, Value * * args, Value & v) +void prim_getAttr(EvalState & state, Value * * args, Value & v) { string attr = state.forceStringNoCtx(*args[0]); state.forceAttrs(*args[1]); @@ -722,6 +722,7 @@ static void prim_getAttr(EvalState & state, Value * * args, Value & v) if (i == args[1]->attrs->end()) throw EvalError(format("attribute `%1%' missing") % attr); // !!! add to stack trace? + if (state.countCalls && i->pos) state.attrSelects[*i->pos]++; state.forceValue(*i->value); v = *i->value; } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index edb7c0fa6c60..dc38a53ca2fe 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -17,6 +17,9 @@ namespace nix { #define foreach(it_type, it, collection) \ for (it_type it = (collection).begin(); it != (collection).end(); ++it) +#define foreach_reverse(it_type, it, collection) \ + for (it_type it = (collection).rbegin(); it != (collection).rend(); ++it) + /* Return an environment variable. */ string getEnv(const string & key, const string & def = ""); -- cgit 1.4.1 From 56e30e161cd309addb5aa95ba02a8d3371846228 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Aug 2012 15:27:30 -0400 Subject: In the chroot, make all mounted filesystems private This is required on systemd, which mounts filesystems as "shared" subtrees. Changes to shared trees in a private mount namespace are propagated to the outside world, which is bad. --- src/libstore/build.cc | 18 ++++++++++++++++++ src/libutil/util.cc | 4 ++-- src/libutil/util.hh | 2 +- 3 files changed, 21 insertions(+), 3 deletions(-) (limited to 'src/libutil') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9da8084bf5c7..0ed69614b80b 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1853,6 +1853,24 @@ void DerivationGoal::initChild() char domainname[] = "(none)"; // kernel default setdomainname(domainname, sizeof(domainname)); + /* Make all filesystems private. This is necessary + because subtrees may have been mounted as "shared" + (MS_SHARED). (Systemd does this, for instance.) Even + though we have a private mount namespace, mounting + filesystems on top of a shared subtree still propagates + outside of the namespace. Making a subtree private is + local to the namespace, though, so setting MS_PRIVATE + does not affect the outside world. */ + Strings mounts = tokenizeString(readFile("/proc/self/mountinfo", true), "\n"); + foreach (Strings::iterator, i, mounts) { + Strings fields = tokenizeString(*i, " "); + assert(fields.size() >= 5); + Strings::iterator j = fields.begin(); + std::advance(j, 4); + if (mount(0, j->c_str(), 0, MS_PRIVATE, 0) == -1) + throw SysError(format("unable to make filesystem `%1%' private") % *j); + } + /* Bind-mount all the directories from the "host" filesystem that we want in the chroot environment. */ diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 9d8e4afed37d..fe4fedfa597d 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -224,12 +224,12 @@ string readFile(int fd) } -string readFile(const Path & path) +string readFile(const Path & path, bool drain) { AutoCloseFD fd = open(path.c_str(), O_RDONLY); if (fd == -1) throw SysError(format("opening file `%1%'") % path); - return readFile(fd); + return drain ? drainFD(fd) : readFile(fd); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index dc38a53ca2fe..22992bbafee7 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -63,7 +63,7 @@ Strings readDirectory(const Path & path); /* Read the contents of a file into a string. */ string readFile(int fd); -string readFile(const Path & path); +string readFile(const Path & path, bool drain = false); /* Write a string to a file. */ void writeFile(const Path & path, const string & s); -- cgit 1.4.1