Delay evaulation of `with` attrs until a variable lookup needs them

Evaluation of attribute sets is strict in the attribute names, which
means immediate evaluation of `with` attribute sets rules out some
potentially interesting use cases (e.g. where the attribute names of one
set depend in some way on another but we want to bring those names into
scope for some values in the second set).

The major example of this is overridable self-referential package sets
(e.g. all-packages.nix). With immediate `with` evaluation, the only
options for such sets are to either make them non-recursive and
explicitly use the name of the overridden set in non-overridden one
every time you want to reference another package, or make the set
recursive and use the `__overrides` hack. As shown in the test case that
comes with this commit, though, delayed `with` evaluation allows a nicer
third alternative.

Signed-off-by: Shea Levy <shea@shealevy.com>

4 files changed, 40 insertions, 7 deletions

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index c927db2c10c6..76bace1d4b3b 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -310,6 +310,10 @@ inline Value * EvalState::lookupVar(Env * env, const VarRef & var)
     
     if (var.fromWith) {
         while (1) {
+            if (env->values[0] == NULL) {
+                env->values[0] = allocValue();
+                evalAttrs(*env->up, env->withAttrs, *env->values[0]);
+            }
             Bindings::iterator j = env->values[0]->attrs->find(var.name);
             if (j != env->values[0]->attrs->end()) {
                 if (countCalls && j->pos) attrSelects[*j->pos]++;
@@ -337,7 +341,7 @@ Env & EvalState::allocEnv(unsigned int size)
     nrValuesInEnvs += size;
     Env * env = (Env *) GC_MALLOC(sizeof(Env) + size * sizeof(Value *));
 
-    /* Clear the values because maybeThunk() expects this. */
+    /* Clear the values because maybeThunk() and lookupVar fromWith expects this. */
     for (unsigned i = 0; i < size; ++i)
         env->values[i] = 0;
     
@@ -405,10 +409,12 @@ unsigned long nrAvoided = 0;
 
 Value * ExprVar::maybeThunk(EvalState & state, Env & env)
 {
-    Value * v = state.lookupVar(&env, info);
-    /* The value might not be initialised in the environment yet.
-       In that case, ignore it. */
-    if (v) { nrAvoided++; return v; }
+    if (!info.fromWith) {
+        Value * v = state.lookupVar(&env, info);
+        /* The value might not be initialised in the environment yet.
+           In that case, ignore it. */
+        if (v) { nrAvoided++; return v; }
+    }
     return Expr::maybeThunk(state, env);
 }
 
@@ -825,8 +831,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
     env2.up = &env;
     env2.prevWith = prevWith;
 
-    env2.values[0] = state.allocValue();
-    state.evalAttrs(env, attrs, *env2.values[0]);
+    env2.withAttrs = attrs;
 
     body->eval(state, env2, v);
 }
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index a57efa08f184..0e4ad3db2325 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -53,6 +53,7 @@ struct Env
 {
     Env * up;
     unsigned int prevWith; // nr of levels up to next `with' environment
+    Expr * withAttrs;
     Value * values[0];
 };
 
diff --git a/tests/lang/eval-okay-delayed-with.exp b/tests/lang/eval-okay-delayed-with.exp
new file mode 100644
index 000000000000..eaacb55c1aff
--- /dev/null
+++ b/tests/lang/eval-okay-delayed-with.exp
@@ -0,0 +1 @@
+"b-overridden"
diff --git a/tests/lang/eval-okay-delayed-with.nix b/tests/lang/eval-okay-delayed-with.nix
new file mode 100644
index 000000000000..82934d6a9d5e
--- /dev/null
+++ b/tests/lang/eval-okay-delayed-with.nix
@@ -0,0 +1,26 @@
+let
+  pkgs_ = with pkgs; {
+    a = derivation {
+      name = "a";
+      system = builtins.currentSystem;
+      builder = "/bin/sh";
+      args = [ "-c" "touch $out" ];
+      inherit b;
+    };
+
+    b = derivation {
+      name = "b";
+      system = builtins.currentSystem;
+      builder = "/bin/sh";
+      args = [ "-c" "touch $out" ];
+    };
+
+    c = b;
+  };
+
+  packageOverrides = p: {
+    b = derivation (p.b.drvAttrs // { name = "b-overridden"; });
+  };
+
+  pkgs = pkgs_ // (packageOverrides pkgs_);
+in pkgs.a.b.name