about summary refs log tree commit diff
path: root/src/libexpr
diff options
context:
space:
mode:
Diffstat (limited to 'src/libexpr')
-rw-r--r--src/libexpr/attr-set.cc24
-rw-r--r--src/libexpr/eval-inline.hh14
-rw-r--r--src/libexpr/eval.cc355
-rw-r--r--src/libexpr/eval.hh61
-rw-r--r--src/libexpr/get-drvs.cc2
-rw-r--r--src/libexpr/get-drvs.hh4
-rw-r--r--src/libexpr/json-to-value.cc2
-rw-r--r--src/libexpr/lexer.l49
-rw-r--r--src/libexpr/local.mk2
-rw-r--r--src/libexpr/nix-expr.pc.in2
-rw-r--r--src/libexpr/nixexpr.hh3
-rw-r--r--src/libexpr/parser.y27
-rw-r--r--src/libexpr/primops.cc211
-rw-r--r--src/libexpr/primops/context.cc187
-rw-r--r--src/libexpr/primops/fetchGit.cc20
-rw-r--r--src/libexpr/primops/fetchMercurial.cc21
-rw-r--r--src/libexpr/primops/fromTOML.cc77
-rw-r--r--src/libexpr/symbol-table.hh9
-rw-r--r--src/libexpr/value.hh8
19 files changed, 772 insertions, 306 deletions
diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc
index b284daa3c2f7..0785897d2513 100644
--- a/src/libexpr/attr-set.cc
+++ b/src/libexpr/attr-set.cc
@@ -1,5 +1,5 @@
 #include "attr-set.hh"
-#include "eval.hh"
+#include "eval-inline.hh"
 
 #include <algorithm>
 
@@ -7,30 +7,18 @@
 namespace nix {
 
 
-/* Note: Various places expect the allocated memory to be zeroed. */
-static void * allocBytes(size_t n)
-{
-    void * p;
-#if HAVE_BOEHMGC
-    p = GC_malloc(n);
-#else
-    p = calloc(n, 1);
-#endif
-    if (!p) throw std::bad_alloc();
-    return p;
-}
-
-
 /* Allocate a new array of attributes for an attribute set with a specific
    capacity. The space is implicitly reserved after the Bindings
    structure. */
-Bindings * EvalState::allocBindings(Bindings::size_t capacity)
+Bindings * EvalState::allocBindings(size_t capacity)
 {
-    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings(capacity);
+    if (capacity > std::numeric_limits<Bindings::size_t>::max())
+        throw Error("attribute set of size %d is too big", capacity);
+    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity);
 }
 
 
-void EvalState::mkAttrs(Value & v, unsigned int capacity)
+void EvalState::mkAttrs(Value & v, size_t capacity)
 {
     if (capacity == 0) {
         v = vEmptySet;
diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index 8cc50e561354..c27116e3b448 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -78,4 +78,18 @@ inline void EvalState::forceList(Value & v, const Pos & pos)
         throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
 }
 
+/* Note: Various places expect the allocated memory to be zeroed. */
+inline void * allocBytes(size_t n)
+{
+    void * p;
+#if HAVE_BOEHMGC
+    p = GC_MALLOC(n);
+#else
+    p = calloc(n, 1);
+#endif
+    if (!p) throw std::bad_alloc();
+    return p;
+}
+
+
 }
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 37b977736e28..d8e10d9f20e1 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -6,27 +6,26 @@
 #include "globals.hh"
 #include "eval-inline.hh"
 #include "download.hh"
+#include "json.hh"
 
 #include <algorithm>
 #include <cstring>
 #include <unistd.h>
 #include <sys/time.h>
 #include <sys/resource.h>
+#include <iostream>
+#include <fstream>
+
+#include <sys/time.h>
+#include <sys/resource.h>
 
 #if HAVE_BOEHMGC
 
 #include <gc/gc.h>
 #include <gc/gc_cpp.h>
 
-#define NEW new (UseGC)
-
-#else
-
-#define NEW new
-
 #endif
 
-
 namespace nix {
 
 
@@ -34,7 +33,7 @@ static char * dupString(const char * s)
 {
     char * t;
 #if HAVE_BOEHMGC
-    t = GC_strdup(s);
+    t = GC_STRDUP(s);
 #else
     t = strdup(s);
 #endif
@@ -43,20 +42,6 @@ static char * dupString(const char * s)
 }
 
 
-/* Note: Various places expect the allocated memory to be zeroed. */
-static void * allocBytes(size_t n)
-{
-    void * p;
-#if HAVE_BOEHMGC
-    p = GC_malloc(n);
-#else
-    p = calloc(n, 1);
-#endif
-    if (!p) throw std::bad_alloc();
-    return p;
-}
-
-
 static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
 {
     checkInterrupt();
@@ -145,6 +130,16 @@ std::ostream & operator << (std::ostream & str, const Value & v)
 }
 
 
+const Value *getPrimOp(const Value &v) {
+    const Value * primOp = &v;
+    while (primOp->type == tPrimOpApp) {
+        primOp = primOp->primOpApp.left;
+    }
+    assert(primOp->type == tPrimOp);
+    return primOp;
+}
+
+
 string showType(const Value & v)
 {
     switch (v.type) {
@@ -159,8 +154,10 @@ string showType(const Value & v)
         case tApp: return "a function application";
         case tLambda: return "a function";
         case tBlackhole: return "a black hole";
-        case tPrimOp: return "a built-in function";
-        case tPrimOpApp: return "a partially applied built-in function";
+        case tPrimOp:
+            return fmt("the built-in function '%s'", string(v.primOp->name));
+        case tPrimOpApp:
+            return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
         case tExternal: return v.external->showType();
         case tFloat: return "a float";
     }
@@ -199,8 +196,15 @@ void initGC()
 
 #if HAVE_BOEHMGC
     /* Initialise the Boehm garbage collector. */
+
+    /* Don't look for interior pointers. This reduces the odds of
+       misdetection a bit. */
     GC_set_all_interior_pointers(0);
 
+    /* We don't have any roots in data segments, so don't scan from
+       there. */
+    GC_set_no_dls(1);
+
     GC_INIT();
 
     GC_set_oom_fn(oomHandler);
@@ -307,20 +311,32 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
     assert(gcInitialised);
 
+    static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
+
     /* Initialise the Nix expression search path. */
-    if (!settings.pureEval) {
+    if (!evalSettings.pureEval) {
         Strings paths = parseNixPath(getEnv("NIX_PATH", ""));
         for (auto & i : _searchPath) addToSearchPath(i);
         for (auto & i : paths) addToSearchPath(i);
     }
     addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true));
 
-    if (settings.restrictEval || settings.pureEval) {
+    if (evalSettings.restrictEval || evalSettings.pureEval) {
         allowedPaths = PathSet();
+
         for (auto & i : searchPath) {
             auto r = resolveSearchPathElem(i);
             if (!r.first) continue;
-            allowedPaths->insert(r.second);
+
+            auto path = r.second;
+
+            if (store->isInStore(r.second)) {
+                PathSet closure;
+                store->computeFSClosure(store->toStorePath(r.second), closure);
+                for (auto & path : closure)
+                    allowedPaths->insert(path);
+            } else
+                allowedPaths->insert(r.second);
         }
     }
 
@@ -334,7 +350,6 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
 EvalState::~EvalState()
 {
-    fileEvalCache.clear();
 }
 
 
@@ -342,25 +357,37 @@ Path EvalState::checkSourcePath(const Path & path_)
 {
     if (!allowedPaths) return path_;
 
+    auto i = resolvedPaths.find(path_);
+    if (i != resolvedPaths.end())
+        return i->second;
+
     bool found = false;
 
+    /* First canonicalize the path without symlinks, so we make sure an
+     * attacker can't append ../../... to a path that would be in allowedPaths
+     * and thus leak symlink targets.
+     */
+    Path abspath = canonPath(path_);
+
     for (auto & i : *allowedPaths) {
-        if (isDirOrInDir(path_, i)) {
+        if (isDirOrInDir(abspath, i)) {
             found = true;
             break;
         }
     }
 
     if (!found)
-        throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path_);
+        throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", abspath);
 
     /* Resolve symlinks. */
-    debug(format("checking access to '%s'") % path_);
-    Path path = canonPath(path_, true);
+    debug(format("checking access to '%s'") % abspath);
+    Path path = canonPath(abspath, true);
 
     for (auto & i : *allowedPaths) {
-        if (isDirOrInDir(path, i))
+        if (isDirOrInDir(path, i)) {
+            resolvedPaths[path_] = path;
             return path;
+        }
     }
 
     throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path);
@@ -369,13 +396,13 @@ Path EvalState::checkSourcePath(const Path & path_)
 
 void EvalState::checkURI(const std::string & uri)
 {
-    if (!settings.restrictEval) return;
+    if (!evalSettings.restrictEval) return;
 
     /* 'uri' should be equal to a prefix, or in a subdirectory of a
        prefix. Thus, the prefix https://github.co does not permit
        access to https://github.com. Note: this allows 'http://' and
        'https://' as prefixes for any http/https URI. */
-    for (auto & prefix : settings.allowedUris.get())
+    for (auto & prefix : evalSettings.allowedUris.get())
         if (uri == prefix ||
             (uri.size() > prefix.size()
             && prefix.size() > 0
@@ -422,7 +449,7 @@ Value * EvalState::addConstant(const string & name, Value & v)
 
 
 Value * EvalState::addPrimOp(const string & name,
-    unsigned int arity, PrimOpFun primOp)
+    size_t arity, PrimOpFun primOp)
 {
     if (arity == 0) {
         Value v;
@@ -433,7 +460,7 @@ Value * EvalState::addPrimOp(const string & name,
     string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
     Symbol sym = symbols.create(name2);
     v->type = tPrimOp;
-    v->primOp = NEW PrimOp(primOp, arity, sym);
+    v->primOp = new PrimOp(primOp, arity, sym);
     staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
     baseEnv.values[baseEnvDispl++] = v;
     baseEnv.values[0]->attrs->push_back(Attr(sym, v));
@@ -528,7 +555,7 @@ Value & mkString(Value & v, const string & s, const PathSet & context)
 {
     mkString(v, s.c_str());
     if (!context.empty()) {
-        unsigned int n = 0;
+        size_t n = 0;
         v.string.context = (const char * *)
             allocBytes((context.size() + 1) * sizeof(char *));
         for (auto & i : context)
@@ -547,17 +574,17 @@ void mkPath(Value & v, const char * s)
 
 inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 {
-    for (unsigned int l = var.level; l; --l, env = env->up) ;
+    for (size_t l = var.level; l; --l, env = env->up) ;
 
     if (!var.fromWith) return env->values[var.displ];
 
     while (1) {
-        if (!env->haveWithAttrs) {
+        if (env->type == Env::HasWithExpr) {
             if (noEval) return 0;
             Value * v = allocValue();
             evalAttrs(*env->up, (Expr *) env->values[0], *v);
             env->values[0] = v;
-            env->haveWithAttrs = true;
+            env->type = Env::HasWithAttrs;
         }
         Bindings::iterator j = env->values[0]->attrs->find(var.name);
         if (j != env->values[0]->attrs->end()) {
@@ -566,26 +593,37 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
         }
         if (!env->prevWith)
             throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, var.pos);
-        for (unsigned int l = env->prevWith; l; --l, env = env->up) ;
+        for (size_t l = env->prevWith; l; --l, env = env->up) ;
     }
 }
 
 
+std::atomic<uint64_t> nrValuesFreed{0};
+
+void finalizeValue(void * obj, void * data)
+{
+    nrValuesFreed++;
+}
+
 Value * EvalState::allocValue()
 {
     nrValues++;
-    return (Value *) allocBytes(sizeof(Value));
+    auto v = (Value *) allocBytes(sizeof(Value));
+    //GC_register_finalizer_no_order(v, finalizeValue, nullptr, nullptr, nullptr);
+    return v;
 }
 
 
-Env & EvalState::allocEnv(unsigned int size)
+Env & EvalState::allocEnv(size_t size)
 {
-    assert(size <= std::numeric_limits<decltype(Env::size)>::max());
+    if (size > std::numeric_limits<decltype(Env::size)>::max())
+        throw Error("environment size %d is too big", size);
 
     nrEnvs++;
     nrValuesInEnvs += size;
     Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
-    env->size = size;
+    env->size = (decltype(Env::size)) size;
+    env->type = Env::Plain;
 
     /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
 
@@ -593,7 +631,7 @@ Env & EvalState::allocEnv(unsigned int size)
 }
 
 
-void EvalState::mkList(Value & v, unsigned int size)
+void EvalState::mkList(Value & v, size_t size)
 {
     clearValue(v);
     if (size == 1)
@@ -628,7 +666,7 @@ void EvalState::mkThunk_(Value & v, Expr * expr)
 
 void EvalState::mkPos(Value & v, Pos * pos)
 {
-    if (pos) {
+    if (pos && pos->file.set()) {
         mkAttrs(v, 3);
         mkString(*allocAttr(v, sFile), pos->file);
         mkInt(*allocAttr(v, sLine), pos->line);
@@ -705,7 +743,17 @@ void EvalState::evalFile(const Path & path_, Value & v)
     }
 
     printTalkative("evaluating file '%1%'", path2);
-    Expr * e = parseExprFromFile(checkSourcePath(path2));
+    Expr * e = nullptr;
+
+    auto j = fileParseCache.find(path2);
+    if (j != fileParseCache.end())
+        e = j->second;
+
+    if (!e)
+        e = parseExprFromFile(checkSourcePath(path2));
+
+    fileParseCache[path2] = e;
+
     try {
         eval(e, v);
     } catch (Error & e) {
@@ -721,6 +769,7 @@ void EvalState::evalFile(const Path & path_, Value & v)
 void EvalState::resetFileCache()
 {
     fileEvalCache.clear();
+    fileParseCache.clear();
 }
 
 
@@ -805,7 +854,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         /* The recursive attributes are evaluated in the new
            environment, while the inherited attributes are evaluated
            in the original environment. */
-        unsigned int displ = 0;
+        size_t displ = 0;
         for (auto & i : attrs) {
             Value * vAttr;
             if (hasOverrides && !i.second.inherited) {
@@ -879,7 +928,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
     /* The recursive attributes are evaluated in the new environment,
        while the inherited attributes are evaluated in the original
        environment. */
-    unsigned int displ = 0;
+    size_t displ = 0;
     for (auto & i : attrs->attrs)
         env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2);
 
@@ -890,7 +939,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
 void ExprList::eval(EvalState & state, Env & env, Value & v)
 {
     state.mkList(v, elems.size());
-    for (unsigned int n = 0; n < elems.size(); ++n)
+    for (size_t n = 0; n < elems.size(); ++n)
         v.listElems()[n] = elems[n]->maybeThunk(state, env);
 }
 
@@ -1012,22 +1061,22 @@ void ExprApp::eval(EvalState & state, Env & env, Value & v)
 void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
 {
     /* Figure out the number of arguments still needed. */
-    unsigned int argsDone = 0;
+    size_t argsDone = 0;
     Value * primOp = &fun;
     while (primOp->type == tPrimOpApp) {
         argsDone++;
         primOp = primOp->primOpApp.left;
     }
     assert(primOp->type == tPrimOp);
-    unsigned int arity = primOp->primOp->arity;
-    unsigned int argsLeft = arity - argsDone;
+    auto arity = primOp->primOp->arity;
+    auto argsLeft = arity - argsDone;
 
     if (argsLeft == 1) {
         /* We have all the arguments, so call the primop. */
 
         /* Put all the arguments in an array. */
         Value * vArgs[arity];
-        unsigned int n = arity - 1;
+        auto n = arity - 1;
         vArgs[n--] = &arg;
         for (Value * arg = &fun; arg->type == tPrimOpApp; arg = arg->primOpApp.left)
             vArgs[n--] = arg->primOpApp.right;
@@ -1048,6 +1097,8 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
 
 void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos)
 {
+    forceValue(fun, pos);
+
     if (fun.type == tPrimOp || fun.type == tPrimOpApp) {
         callPrimOp(fun, arg, v, pos);
         return;
@@ -1063,10 +1114,8 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
         auto & fun2 = *allocValue();
         fun2 = fun;
         /* !!! Should we use the attr pos here? */
-        forceValue(*found->value, pos);
         Value v2;
         callFunction(*found->value, fun2, v2, pos);
-        forceValue(v2, pos);
         return callFunction(v2, arg, v, pos);
       }
     }
@@ -1076,13 +1125,13 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
 
     ExprLambda & lambda(*fun.lambda.fun);
 
-    unsigned int size =
+    auto size =
         (lambda.arg.empty() ? 0 : 1) +
         (lambda.matchAttrs ? lambda.formals->formals.size() : 0);
     Env & env2(allocEnv(size));
     env2.up = fun.lambda.env;
 
-    unsigned int displ = 0;
+    size_t displ = 0;
 
     if (!lambda.matchAttrs)
         env2.values[displ++] = &arg;
@@ -1096,7 +1145,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
         /* For each formal argument, get the actual argument.  If
            there is no matching actual argument but the formal
            argument has a default, use the default. */
-        unsigned int attrsUsed = 0;
+        size_t attrsUsed = 0;
         for (auto & i : lambda.formals->formals) {
             Bindings::iterator j = arg.attrs->find(i.name);
             if (j == arg.attrs->end()) {
@@ -1153,7 +1202,6 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
     if (fun.type == tAttrs) {
         auto found = fun.attrs->find(sFunctor);
         if (found != fun.attrs->end()) {
-            forceValue(*found->value);
             Value * v = allocValue();
             callFunction(*found->value, fun, *v, noPos);
             forceValue(*v);
@@ -1188,7 +1236,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
     Env & env2(state.allocEnv(1));
     env2.up = &env;
     env2.prevWith = prevWith;
-    env2.haveWithAttrs = false;
+    env2.type = Env::HasWithExpr;
     env2.values[0] = (Value *) attrs;
 
     body->eval(state, env2, v);
@@ -1294,15 +1342,15 @@ void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v)
 }
 
 
-void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists, const Pos & pos)
+void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos)
 {
     nrListConcats++;
 
     Value * nonEmpty = 0;
-    unsigned int len = 0;
-    for (unsigned int n = 0; n < nrLists; ++n) {
+    size_t len = 0;
+    for (size_t n = 0; n < nrLists; ++n) {
         forceList(*lists[n], pos);
-        unsigned int l = lists[n]->listSize();
+        auto l = lists[n]->listSize();
         len += l;
         if (l) nonEmpty = lists[n];
     }
@@ -1314,8 +1362,8 @@ void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists, co
 
     mkList(v, len);
     auto out = v.listElems();
-    for (unsigned int n = 0, pos = 0; n < nrLists; ++n) {
-        unsigned int l = lists[n]->listSize();
+    for (size_t n = 0, pos = 0; n < nrLists; ++n) {
+        auto l = lists[n]->listSize();
         if (l)
             memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *));
         pos += l;
@@ -1410,7 +1458,7 @@ void EvalState::forceValueDeep(Value & v)
         }
 
         else if (v.isList()) {
-            for (unsigned int n = 0; n < v.listSize(); ++n)
+            for (size_t n = 0; n < v.listSize(); ++n)
                 recurse(*v.listElems()[n]);
         }
     };
@@ -1537,7 +1585,6 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
     if (v.type == tAttrs) {
         auto i = v.attrs->find(sToString);
         if (i != v.attrs->end()) {
-            forceValue(*i->value, pos);
             Value v1;
             callFunction(*i->value, v, v1, pos);
             return coerceToString(pos, v1, context, coerceMore, copyToStore);
@@ -1562,7 +1609,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
 
         if (v.isList()) {
             string result;
-            for (unsigned int n = 0; n < v.listSize(); ++n) {
+            for (size_t n = 0; n < v.listSize(); ++n) {
                 result += coerceToString(pos, *v.listElems()[n],
                     context, coerceMore, copyToStore);
                 if (n < v.listSize() - 1
@@ -1649,7 +1696,7 @@ bool EvalState::eqValues(Value & v1, Value & v2)
         case tList2:
         case tListN:
             if (v1.listSize() != v2.listSize()) return false;
-            for (unsigned int n = 0; n < v1.listSize(); ++n)
+            for (size_t n = 0; n < v1.listSize(); ++n)
                 if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false;
             return true;
 
@@ -1691,12 +1738,9 @@ bool EvalState::eqValues(Value & v1, Value & v2)
     }
 }
 
-
 void EvalState::printStats()
 {
     bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0";
-    Verbosity v = showStats ? lvlInfo : lvlDebug;
-    printMsg(v, "evaluation statistics:");
 
     struct rusage buf;
     getrusage(RUSAGE_SELF, &buf);
@@ -1707,62 +1751,107 @@ void EvalState::printStats()
     uint64_t bValues = nrValues * sizeof(Value);
     uint64_t bAttrsets = nrAttrsets * sizeof(Bindings) + nrAttrsInAttrsets * sizeof(Attr);
 
-    printMsg(v, format("  time elapsed: %1%") % cpuTime);
-    printMsg(v, format("  size of a value: %1%") % sizeof(Value));
-    printMsg(v, format("  size of an attr: %1%") % sizeof(Attr));
-    printMsg(v, format("  environments allocated count: %1%") % nrEnvs);
-    printMsg(v, format("  environments allocated bytes: %1%") % bEnvs);
-    printMsg(v, format("  list elements count: %1%") % nrListElems);
-    printMsg(v, format("  list elements bytes: %1%") % bLists);
-    printMsg(v, format("  list concatenations: %1%") % nrListConcats);
-    printMsg(v, format("  values allocated count: %1%") % nrValues);
-    printMsg(v, format("  values allocated bytes: %1%") % bValues);
-    printMsg(v, format("  sets allocated: %1% (%2% bytes)") % nrAttrsets % bAttrsets);
-    printMsg(v, format("  right-biased unions: %1%") % nrOpUpdates);
-    printMsg(v, format("  values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied);
-    printMsg(v, format("  symbols in symbol table: %1%") % symbols.size());
-    printMsg(v, format("  size of symbol table: %1%") % symbols.totalSize());
-    printMsg(v, format("  number of thunks: %1%") % nrThunks);
-    printMsg(v, format("  number of thunks avoided: %1%") % nrAvoided);
-    printMsg(v, format("  number of attr lookups: %1%") % nrLookups);
-    printMsg(v, format("  number of primop calls: %1%") % nrPrimOpCalls);
-    printMsg(v, format("  number of function calls: %1%") % nrFunctionCalls);
-    printMsg(v, format("  total allocations: %1% bytes") % (bEnvs + bLists + bValues + bAttrsets));
-
 #if HAVE_BOEHMGC
     GC_word heapSize, totalBytes;
     GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes);
-    printMsg(v, format("  current Boehm heap size: %1% bytes") % heapSize);
-    printMsg(v, format("  total Boehm heap allocations: %1% bytes") % totalBytes);
+#endif
+    if (showStats) {
+        auto outPath = getEnv("NIX_SHOW_STATS_PATH","-");
+        std::fstream fs;
+        if (outPath != "-")
+            fs.open(outPath, std::fstream::out);
+        JSONObject topObj(outPath == "-" ? std::cerr : fs, true);
+        topObj.attr("cpuTime",cpuTime);
+        {
+            auto envs = topObj.object("envs");
+            envs.attr("number", nrEnvs);
+            envs.attr("elements", nrValuesInEnvs);
+            envs.attr("bytes", bEnvs);
+        }
+        {
+            auto lists = topObj.object("list");
+            lists.attr("elements", nrListElems);
+            lists.attr("bytes", bLists);
+            lists.attr("concats", nrListConcats);
+        }
+        {
+            auto values = topObj.object("values");
+            values.attr("number", nrValues);
+            values.attr("bytes", bValues);
+        }
+        {
+            auto syms = topObj.object("symbols");
+            syms.attr("number", symbols.size());
+            syms.attr("bytes", symbols.totalSize());
+        }
+        {
+            auto sets = topObj.object("sets");
+            sets.attr("number", nrAttrsets);
+            sets.attr("bytes", bAttrsets);
+            sets.attr("elements", nrAttrsInAttrsets);
+        }
+        {
+            auto sizes = topObj.object("sizes");
+            sizes.attr("Env", sizeof(Env));
+            sizes.attr("Value", sizeof(Value));
+            sizes.attr("Bindings", sizeof(Bindings));
+            sizes.attr("Attr", sizeof(Attr));
+        }
+        topObj.attr("nrOpUpdates", nrOpUpdates);
+        topObj.attr("nrOpUpdateValuesCopied", nrOpUpdateValuesCopied);
+        topObj.attr("nrThunks", nrThunks);
+        topObj.attr("nrAvoided", nrAvoided);
+        topObj.attr("nrLookups", nrLookups);
+        topObj.attr("nrPrimOpCalls", nrPrimOpCalls);
+        topObj.attr("nrFunctionCalls", nrFunctionCalls);
+#if HAVE_BOEHMGC
+        {
+            auto gc = topObj.object("gc");
+            gc.attr("heapSize", heapSize);
+            gc.attr("totalBytes", totalBytes);
+        }
 #endif
 
-    if (countCalls) {
-        v = lvlInfo;
-
-        printMsg(v, format("calls to %1% primops:") % primOpCalls.size());
-        typedef std::multimap<unsigned int, Symbol> PrimOpCalls_;
-        PrimOpCalls_ primOpCalls_;
-        for (auto & i : primOpCalls)
-            primOpCalls_.insert(std::pair<unsigned int, Symbol>(i.second, i.first));
-        for (auto i = primOpCalls_.rbegin(); i != primOpCalls_.rend(); ++i)
-            printMsg(v, format("%1$10d %2%") % i->first % i->second);
-
-        printMsg(v, format("calls to %1% functions:") % functionCalls.size());
-        typedef std::multimap<unsigned int, ExprLambda *> FunctionCalls_;
-        FunctionCalls_ functionCalls_;
-        for (auto & i : functionCalls)
-            functionCalls_.insert(std::pair<unsigned int, ExprLambda *>(i.second, i.first));
-        for (auto i = functionCalls_.rbegin(); i != functionCalls_.rend(); ++i)
-            printMsg(v, format("%1$10d %2%") % i->first % i->second->showNamePos());
-
-        printMsg(v, format("evaluations of %1% attributes:") % attrSelects.size());
-        typedef std::multimap<unsigned int, Pos> AttrSelects_;
-        AttrSelects_ attrSelects_;
-        for (auto & i : attrSelects)
-            attrSelects_.insert(std::pair<unsigned int, Pos>(i.second, i.first));
-        for (auto i = attrSelects_.rbegin(); i != attrSelects_.rend(); ++i)
-            printMsg(v, format("%1$10d %2%") % i->first % i->second);
+        if (countCalls) {
+            {
+                auto obj = topObj.object("primops");
+                for (auto & i : primOpCalls)
+                    obj.attr(i.first, i.second);
+            }
+            {
+                auto list = topObj.list("functions");
+                for (auto & i : functionCalls) {
+                    auto obj = list.object();
+                    if (i.first->name.set())
+                        obj.attr("name", (const string &) i.first->name);
+                    else
+                        obj.attr("name", nullptr);
+                    if (i.first->pos) {
+                        obj.attr("file", (const string &) i.first->pos.file);
+                        obj.attr("line", i.first->pos.line);
+                        obj.attr("column", i.first->pos.column);
+                    }
+                    obj.attr("count", i.second);
+                }
+            }
+            {
+                auto list = topObj.list("attributes");
+                for (auto & i : attrSelects) {
+                    auto obj = list.object();
+                    if (i.first) {
+                        obj.attr("file", (const string &) i.first.file);
+                        obj.attr("line", i.first.line);
+                        obj.attr("column", i.first.column);
+                    }
+                    obj.attr("count", i.second);
+                }
+            }
+        }
 
+        if (getEnv("NIX_SHOW_SYMBOLS", "0") != "0") {
+            auto list = topObj.list("symbols");
+            symbols.dump([&](const std::string & s) { list.elem(s); });
+        }
     }
 }
 
@@ -1810,7 +1899,7 @@ size_t valueSize(Value & v)
             if (seen.find(v.listElems()) == seen.end()) {
                 seen.insert(v.listElems());
                 sz += v.listSize() * sizeof(Value *);
-                for (unsigned int n = 0; n < v.listSize(); ++n)
+                for (size_t n = 0; n < v.listSize(); ++n)
                     sz += doValue(*v.listElems()[n]);
             }
             break;
@@ -1846,9 +1935,10 @@ size_t valueSize(Value & v)
 
         size_t sz = sizeof(Env) + sizeof(Value *) * env.size;
 
-        for (unsigned int i = 0; i < env.size; ++i)
-            if (env.values[i])
-                sz += doValue(*env.values[i]);
+        if (env.type != Env::HasWithExpr)
+            for (size_t i = 0; i < env.size; ++i)
+                if (env.values[i])
+                    sz += doValue(*env.values[i]);
 
         if (env.up) sz += doEnv(*env.up);
 
@@ -1877,4 +1967,9 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) {
 }
 
 
+EvalSettings evalSettings;
+
+static GlobalConfig::Register r1(&evalSettings);
+
+
 }
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 9d8799b7906b..a314e01e0a71 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -5,8 +5,10 @@
 #include "nixexpr.hh"
 #include "symbol-table.hh"
 #include "hash.hh"
+#include "config.hh"
 
 #include <map>
+#include <unordered_map>
 
 
 namespace nix {
@@ -34,8 +36,8 @@ struct Env
 {
     Env * up;
     unsigned short size; // used by ‘valueSize’
-    unsigned short prevWith:15; // nr of levels up to next `with' environment
-    unsigned short haveWithAttrs:1;
+    unsigned short prevWith:14; // nr of levels up to next `with' environment
+    enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2;
     Value * values[0];
 };
 
@@ -79,7 +81,7 @@ public:
 
     /* The allowed filesystem paths in restricted or pure evaluation
        mode. */
-    std::experimental::optional<PathSet> allowedPaths;
+    std::optional<PathSet> allowedPaths;
 
     Value vEmptySet;
 
@@ -88,6 +90,14 @@ public:
 private:
     SrcToStore srcToStore;
 
+    /* A cache from path names to parse trees. */
+#if HAVE_BOEHMGC
+    typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *> > > FileParseCache;
+#else
+    typedef std::map<Path, Expr *> FileParseCache;
+#endif
+    FileParseCache fileParseCache;
+
     /* A cache from path names to values. */
 #if HAVE_BOEHMGC
     typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value> > > FileEvalCache;
@@ -100,6 +110,9 @@ private:
 
     std::map<std::string, std::pair<bool, std::string>> searchPathResolved;
 
+    /* Cache used by checkSourcePath(). */
+    std::unordered_map<Path, Path> resolvedPaths;
+
 public:
 
     EvalState(const Strings & _searchPath, ref<Store> store);
@@ -214,7 +227,7 @@ private:
     Value * addConstant(const string & name, Value & v);
 
     Value * addPrimOp(const string & name,
-        unsigned int arity, PrimOpFun primOp);
+        size_t arity, PrimOpFun primOp);
 
 public:
 
@@ -248,18 +261,18 @@ public:
 
     /* Allocation primitives. */
     Value * allocValue();
-    Env & allocEnv(unsigned int size);
+    Env & allocEnv(size_t size);
 
     Value * allocAttr(Value & vAttrs, const Symbol & name);
 
-    Bindings * allocBindings(Bindings::size_t capacity);
+    Bindings * allocBindings(size_t capacity);
 
-    void mkList(Value & v, unsigned int length);
-    void mkAttrs(Value & v, unsigned int capacity);
+    void mkList(Value & v, size_t length);
+    void mkAttrs(Value & v, size_t capacity);
     void mkThunk_(Value & v, Expr * expr);
     void mkPos(Value & v, Pos * pos);
 
-    void concatLists(Value & v, unsigned int nrLists, Value * * lists, const Pos & pos);
+    void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos);
 
     /* Print statistics. */
     void printStats();
@@ -282,15 +295,15 @@ private:
 
     bool countCalls;
 
-    typedef std::map<Symbol, unsigned int> PrimOpCalls;
+    typedef std::map<Symbol, size_t> PrimOpCalls;
     PrimOpCalls primOpCalls;
 
-    typedef std::map<ExprLambda *, unsigned int> FunctionCalls;
+    typedef std::map<ExprLambda *, size_t> FunctionCalls;
     FunctionCalls functionCalls;
 
     void incrFunctionCall(ExprLambda * fun);
 
-    typedef std::map<Pos, unsigned int> AttrSelects;
+    typedef std::map<Pos, size_t> AttrSelects;
     AttrSelects attrSelects;
 
     friend struct ExprOpUpdate;
@@ -303,6 +316,9 @@ private:
 /* Return a string representing the type of the value `v'. */
 string showType(const Value & v);
 
+/* Decode a context string ‘!<name>!<path>’ into a pair <path,
+   name>. */
+std::pair<string, string> decodeContext(const string & s);
 
 /* If `path' refers to a directory, then append "/default.nix". */
 Path resolveExprPath(Path path);
@@ -316,4 +332,25 @@ struct InvalidPathError : EvalError
 #endif
 };
 
+struct EvalSettings : Config
+{
+    Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
+        "Whether builtin functions that allow executing native code should be enabled."};
+
+    Setting<bool> restrictEval{this, false, "restrict-eval",
+        "Whether to restrict file system access to paths in $NIX_PATH, "
+        "and network access to the URI prefixes listed in 'allowed-uris'."};
+
+    Setting<bool> pureEval{this, false, "pure-eval",
+        "Whether to restrict file system and network access to files specified by cryptographic hash."};
+
+    Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation",
+        "Whether the evaluator allows importing the result of a derivation."};
+
+    Setting<Strings> allowedUris{this, {}, "allowed-uris",
+        "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
+};
+
+extern EvalSettings evalSettings;
+
 }
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index d38ed2df3b18..21a4d7917fce 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -295,7 +295,7 @@ static bool getDerivation(EvalState & state, Value & v,
 }
 
 
-std::experimental::optional<DrvInfo> getDerivation(EvalState & state, Value & v,
+std::optional<DrvInfo> getDerivation(EvalState & state, Value & v,
     bool ignoreAssertionFailures)
 {
     Done done;
diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh
index 4d9128e3f448..d7860fc6a4bc 100644
--- a/src/libexpr/get-drvs.hh
+++ b/src/libexpr/get-drvs.hh
@@ -44,7 +44,7 @@ public:
     string queryDrvPath() const;
     string queryOutPath() const;
     string queryOutputName() const;
-    /** Return the list of outputs. The "outputs to install" are determined by `mesa.outputsToInstall`. */
+    /** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */
     Outputs queryOutputs(bool onlyOutputsToInstall = false);
 
     StringSet queryMetaNames();
@@ -78,7 +78,7 @@ typedef list<DrvInfo> DrvInfos;
 
 /* If value `v' denotes a derivation, return a DrvInfo object
    describing it. Otherwise return nothing. */
-std::experimental::optional<DrvInfo> getDerivation(EvalState & state,
+std::optional<DrvInfo> getDerivation(EvalState & state,
     Value & v, bool ignoreAssertionFailures);
 
 void getDerivations(EvalState & state, Value & v, const string & pathPrefix,
diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc
index 8b1404595548..3f6017957782 100644
--- a/src/libexpr/json-to-value.cc
+++ b/src/libexpr/json-to-value.cc
@@ -110,7 +110,7 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
             if (number_type == tFloat)
                 mkFloat(v, stod(tmp_number));
             else
-                mkInt(v, stoi(tmp_number));
+                mkInt(v, stol(tmp_number));
         } catch (std::invalid_argument e) {
             throw JSONParseError("invalid JSON number");
         } catch (std::out_of_range e) {
diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index 1e9c29afa133..c34e5c383923 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -6,12 +6,14 @@
 %option nounput noyy_top_state
 
 
+%s DEFAULT
 %x STRING
 %x IND_STRING
-%x INSIDE_DOLLAR_CURLY
 
 
 %{
+#include <boost/lexical_cast.hpp>
+
 #include "nixexpr.hh"
 #include "parser-tab.hh"
 
@@ -97,8 +99,6 @@ URI         [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~
 
 %%
 
-<INITIAL,INSIDE_DOLLAR_CURLY>{
-
 
 if          { return IF; }
 then        { return THEN; }
@@ -124,9 +124,11 @@ or          { return OR_KW; }
 
 {ID}        { yylval->id = strdup(yytext); return ID; }
 {INT}       { errno = 0;
-              yylval->n = strtol(yytext, 0, 10);
-              if (errno != 0)
+              try {
+                  yylval->n = boost::lexical_cast<int64_t>(yytext);
+              } catch (const boost::bad_lexical_cast &) {
                   throw ParseError(format("invalid integer '%1%'") % yytext);
+              }
               return INT;
             }
 {FLOAT}     { errno = 0;
@@ -136,17 +138,19 @@ or          { return OR_KW; }
               return FLOAT;
             }
 
-\$\{        { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }
-}
+\$\{        { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
 
-\}                           { return '}'; }
-<INSIDE_DOLLAR_CURLY>\}      { POP_STATE(); return '}'; }
-\{                           { return '{'; }
-<INSIDE_DOLLAR_CURLY>\{      { PUSH_STATE(INSIDE_DOLLAR_CURLY); return '{'; }
+\}          { /* State INITIAL only exists at the bottom of the stack and is
+                 used as a marker. DEFAULT replaces it everywhere else.
+                 Popping when in INITIAL state causes an empty stack exception,
+                 so don't */
+              if (YYSTATE != INITIAL)
+                POP_STATE();
+              return '}';
+            }
+\{          { PUSH_STATE(DEFAULT); return '{'; }
 
-<INITIAL,INSIDE_DOLLAR_CURLY>\" {
-                PUSH_STATE(STRING); return '"';
-              }
+\"          { PUSH_STATE(STRING); return '"'; }
 <STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" |
 <STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ {
                 /* It is impossible to match strings ending with '$' with one
@@ -155,7 +159,7 @@ or          { return OR_KW; }
                 yylval->e = unescapeStr(data->symbols, yytext, yyleng);
                 return STR;
               }
-<STRING>\$\{  { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }
+<STRING>\$\{  { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
 <STRING>\"    { POP_STATE(); return '"'; }
 <STRING>\$|\\|\$\\ {
                 /* This can only occur when we reach EOF, otherwise the above
@@ -165,7 +169,7 @@ or          { return OR_KW; }
                 return STR;
               }
 
-<INITIAL,INSIDE_DOLLAR_CURLY>\'\'(\ *\n)?     { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
+\'\'(\ *\n)?     { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
 <IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ {
                    yylval->e = new ExprIndStr(yytext);
                    return IND_STR;
@@ -183,14 +187,13 @@ or          { return OR_KW; }
                    yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2);
                    return IND_STR;
                  }
-<IND_STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }
+<IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
 <IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; }
 <IND_STRING>\'   {
                    yylval->e = new ExprIndStr("'");
                    return IND_STR;
                  }
 
-<INITIAL,INSIDE_DOLLAR_CURLY>{
 
 {PATH}      { if (yytext[yyleng-1] == '/')
                   throw ParseError("path '%s' has a trailing slash", yytext);
@@ -209,11 +212,11 @@ or          { return OR_KW; }
 \#[^\r\n]*    /* single-line comments */
 \/\*([^*]|\*+[^*/])*\*+\/  /* long comments */
 
-{ANY}           return yytext[0];
-
-}
-
-<<EOF>> { data->atEnd = true; return 0; }
+{ANY}       {
+              /* Don't return a negative number, as this will cause
+                 Bison to stop parsing without an error. */
+              return (unsigned char) yytext[0];
+            }
 
 %%
 
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index daa3258f0d3c..ccd5293e4e5e 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -6,7 +6,7 @@ libexpr_DIR := $(d)
 
 libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
 
-libexpr_LIBS = libutil libstore libformat
+libexpr_LIBS = libutil libstore
 
 libexpr_LDFLAGS =
 ifneq ($(OS), FreeBSD)
diff --git a/src/libexpr/nix-expr.pc.in b/src/libexpr/nix-expr.pc.in
index 79f3e2f4506e..80f7a492b1a1 100644
--- a/src/libexpr/nix-expr.pc.in
+++ b/src/libexpr/nix-expr.pc.in
@@ -7,4 +7,4 @@ Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Requires: nix-store bdw-gc
 Libs: -L${libdir} -lnixexpr
-Cflags: -I${includedir}/nix -std=c++14
+Cflags: -I${includedir}/nix -std=c++17
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 8c8f39640681..665a42987dc1 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -11,7 +11,6 @@ namespace nix {
 
 MakeError(EvalError, Error)
 MakeError(ParseError, Error)
-MakeError(IncompleteParseError, ParseError)
 MakeError(AssertionError, EvalError)
 MakeError(ThrownError, AssertionError)
 MakeError(Abort, EvalError)
@@ -255,7 +254,7 @@ struct ExprWith : Expr
 {
     Pos pos;
     Expr * attrs, * body;
-    unsigned int prevWith;
+    size_t prevWith;
     ExprWith(const Pos & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { };
     COMMON_METHODS
 };
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index df4fdf032479..97e1a65849e2 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -1,7 +1,7 @@
 %glr-parser
 %pure-parser
 %locations
-%error-verbose
+%define parse.error verbose
 %defines
 /* %no-lines */
 %parse-param { void * scanner }
@@ -31,12 +31,10 @@ namespace nix {
         Path basePath;
         Symbol path;
         string error;
-        bool atEnd;
         Symbol sLetBody;
         ParseData(EvalState & state)
             : state(state)
             , symbols(state.symbols)
-            , atEnd(false)
             , sLetBody(symbols.create("<let-body>"))
             { };
     };
@@ -156,8 +154,8 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
        whitespace-only final lines are not taken into account.  (So
        the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */
     bool atStartOfLine = true; /* = seen only whitespace in the current line */
-    unsigned int minIndent = 1000000;
-    unsigned int curIndent = 0;
+    size_t minIndent = 1000000;
+    size_t curIndent = 0;
     for (auto & i : es) {
         ExprIndStr * e = dynamic_cast<ExprIndStr *>(i);
         if (!e) {
@@ -168,7 +166,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
             }
             continue;
         }
-        for (unsigned int j = 0; j < e->s.size(); ++j) {
+        for (size_t j = 0; j < e->s.size(); ++j) {
             if (atStartOfLine) {
                 if (e->s[j] == ' ')
                     curIndent++;
@@ -190,8 +188,8 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
     /* Strip spaces from each line. */
     vector<Expr *> * es2 = new vector<Expr *>;
     atStartOfLine = true;
-    unsigned int curDropped = 0;
-    unsigned int n = es.size();
+    size_t curDropped = 0;
+    size_t n = es.size();
     for (vector<Expr *>::iterator i = es.begin(); i != es.end(); ++i, --n) {
         ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i);
         if (!e) {
@@ -202,7 +200,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
         }
 
         string s2;
-        for (unsigned int j = 0; j < e->s.size(); ++j) {
+        for (size_t j = 0; j < e->s.size(); ++j) {
             if (atStartOfLine) {
                 if (e->s[j] == ' ') {
                     if (curDropped++ >= minIndent)
@@ -295,11 +293,11 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 %token IND_STRING_OPEN IND_STRING_CLOSE
 %token ELLIPSIS
 
-%nonassoc IMPL
+%right IMPL
 %left OR
 %left AND
 %nonassoc EQ NEQ
-%left '<' '>' LEQ GEQ
+%nonassoc '<' '>' LEQ GEQ
 %right UPDATE
 %left NOT
 %left '+' '-'
@@ -561,12 +559,7 @@ Expr * EvalState::parse(const char * text,
     int res = yyparse(scanner, &data);
     yylex_destroy(scanner);
 
-    if (res) {
-      if (data.atEnd)
-        throw IncompleteParseError(data.error);
-      else
-        throw ParseError(data.error);
-    }
+    if (res) throw ParseError(data.error);
 
     data.result->bindVars(staticEnv);
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 9e7651da725f..06f577f36fce 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -73,7 +73,7 @@ void EvalState::realiseContext(const PathSet & context)
 
     if (drvs.empty()) return;
 
-    if (!settings.enableImportFromDerivation)
+    if (!evalSettings.enableImportFromDerivation)
         throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin()));
 
     /* For performance, prefetch all substitute info. */
@@ -271,7 +271,18 @@ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Valu
 static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0]);
-    mkBool(v, args[0]->type == tLambda);
+    bool res;
+    switch (args[0]->type) {
+        case tLambda:
+        case tPrimOp:
+        case tPrimOpApp:
+            res = true;
+            break;
+        default:
+            res = false;
+            break;
+    }
+    mkBool(v, res);
 }
 
 
@@ -304,6 +315,12 @@ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Valu
     mkBool(v, args[0]->type == tBool);
 }
 
+/* Determine whether the argument is a path. */
+static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceValue(*args[0]);
+    mkBool(v, args[0]->type == tPath);
+}
 
 struct CompareValues
 {
@@ -453,7 +470,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val
 static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string name = state.forceStringNoCtx(*args[0], pos);
-    mkString(v, settings.restrictEval || settings.pureEval ? "" : getEnv(name));
+    mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name));
 }
 
 
@@ -544,7 +561,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     PathSet context;
 
-    std::experimental::optional<std::string> outputHash;
+    std::optional<std::string> outputHash;
     std::string outputHashAlgo;
     bool outputHashRecursive = false;
 
@@ -676,21 +693,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             }
         }
 
-        /* See prim_unsafeDiscardOutputDependency. */
-        else if (path.at(0) == '~')
-            drv.inputSrcs.insert(string(path, 1));
-
         /* Handle derivation outputs of the form ‘!<name>!<path>’. */
         else if (path.at(0) == '!') {
             std::pair<string, string> ctx = decodeContext(path);
             drv.inputDrvs[ctx.first].insert(ctx.second);
         }
 
-        /* Handle derivation contexts returned by
-           ‘builtins.storePath’. */
-        else if (isDerivation(path))
-            drv.inputDrvs[path] = state.store->queryDerivationOutputNames(path);
-
         /* Otherwise it's a source file. */
         else
             drv.inputSrcs.insert(path);
@@ -713,16 +721,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         if (outputs.size() != 1 || *(outputs.begin()) != "out")
             throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName);
 
-        HashType ht = parseHashType(outputHashAlgo);
-        if (ht == htUnknown)
-            throw EvalError(format("unknown hash algorithm '%1%', at %2%") % outputHashAlgo % posDrvName);
+        HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
         Hash h(*outputHash, ht);
-        outputHash = h.to_string(Base16, false);
-        if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo;
 
         Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
         if (!jsonObject) drv.env["out"] = outPath;
-        drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, *outputHash);
+        drv.outputs["out"] = DerivationOutput(outPath,
+            (outputHashRecursive ? "r:" : "") + printHashType(h.type),
+            h.to_string(Base16, false));
     }
 
     else {
@@ -855,7 +861,7 @@ static void prim_baseNameOf(EvalState & state, const Pos & pos, Value * * args,
 static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     PathSet context;
-    Path dir = dirOf(state.coerceToPath(pos, *args[0], context));
+    Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false));
     if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context);
 }
 
@@ -917,6 +923,20 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
     mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str());
 }
 
+/* Return the cryptographic hash of a file in base-16. */
+static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    string type = state.forceStringNoCtx(*args[0], pos);
+    HashType ht = parseHashType(type);
+    if (ht == htUnknown)
+      throw Error(format("unknown hash type '%1%', at %2%") % type % pos);
+
+    PathSet context; // discarded
+    Path p = state.coerceToPath(pos, *args[1], context);
+
+    mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context);
+}
+
 /* Read a directory (without . or ..) */
 static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -995,13 +1015,8 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
     PathSet refs;
 
     for (auto path : context) {
-        if (path.at(0) == '=') path = string(path, 1);
-        if (isDerivation(path)) {
-            /* See prim_unsafeDiscardOutputDependency. */
-            if (path.at(0) != '~')
-                throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
-            path = string(path, 1);
-        }
+        if (path.at(0) != '/')
+            throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
         refs.insert(path);
     }
 
@@ -1020,7 +1035,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
     Value * filterFun, bool recursive, const Hash & expectedHash, Value & v)
 {
-    const auto path = settings.pureEval && expectedHash ?
+    const auto path = evalSettings.pureEval && expectedHash ?
         path_ :
         state.checkSourcePath(path_);
     PathFilter filter = filterFun ? ([&](const Path & path) {
@@ -1345,6 +1360,24 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
 }
 
 
+/* Apply a function to every element of an attribute set. */
+static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceAttrs(*args[1], pos);
+
+    state.mkAttrs(v, args[1]->attrs->size());
+
+    for (auto & i : *args[1]->attrs) {
+        Value * vName = state.allocValue();
+        Value * vFun2 = state.allocValue();
+        mkString(*vName, i.name);
+        mkApp(*vFun2, *args[0], *vName);
+        mkApp(*state.allocAttr(v, i.name), *vFun2, *i.value);
+    }
+}
+
+
+
 /*************************************************************
  * Lists
  *************************************************************/
@@ -1399,7 +1432,6 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
 /* Apply a function to every element of a list. */
 static void prim_map(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceFunction(*args[0], pos);
     state.forceList(*args[1], pos);
 
     state.mkList(v, args[1]->listSize());
@@ -1478,19 +1510,20 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args,
     state.forceFunction(*args[0], pos);
     state.forceList(*args[2], pos);
 
-    Value * vCur = args[1];
+    if (args[2]->listSize()) {
+        Value * vCur = args[1];
 
-    if (args[2]->listSize())
         for (unsigned int n = 0; n < args[2]->listSize(); ++n) {
             Value vTmp;
             state.callFunction(*args[0], *vCur, vTmp, pos);
             vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue();
             state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos);
         }
-    else
-        v = *vCur;
-
-    state.forceValue(v);
+        state.forceValue(v);
+    } else {
+        state.forceValue(*args[1]);
+        v = *args[1];
+    }
 }
 
 
@@ -1527,7 +1560,6 @@ static void prim_all(EvalState & state, const Pos & pos, Value * * args, Value &
 
 static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceFunction(*args[0], pos);
     auto len = state.forceInt(*args[1], pos);
 
     if (len < 0)
@@ -1616,6 +1648,35 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V
 }
 
 
+/* concatMap = f: list: concatLists (map f list); */
+/* C++-version is to avoid allocating `mkApp', call `f' eagerly */
+static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceFunction(*args[0], pos);
+    state.forceList(*args[1], pos);
+    auto nrLists = args[1]->listSize();
+
+    Value lists[nrLists];
+    size_t len = 0;
+
+    for (unsigned int n = 0; n < nrLists; ++n) {
+        Value * vElem = args[1]->listElems()[n];
+        state.callFunction(*args[0], *vElem, lists[n], pos);
+        state.forceList(lists[n], pos);
+        len += lists[n].listSize();
+    }
+
+    state.mkList(v, len);
+    auto out = v.listElems();
+    for (unsigned int n = 0, pos = 0; n < nrLists; ++n) {
+        auto l = lists[n].listSize();
+        if (l)
+            memcpy(out + pos, lists[n].listElems(), l * sizeof(Value *));
+        pos += l;
+    }
+}
+
+
 /*************************************************************
  * Integer arithmetic
  *************************************************************/
@@ -1623,6 +1684,8 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V
 
 static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
     if (args[0]->type == tFloat || args[1]->type == tFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos));
     else
@@ -1632,6 +1695,8 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value &
 
 static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
     if (args[0]->type == tFloat || args[1]->type == tFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos));
     else
@@ -1641,6 +1706,8 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value &
 
 static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
     if (args[0]->type == tFloat || args[1]->type == tFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos));
     else
@@ -1650,6 +1717,9 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value &
 
 static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
+
     NixFloat f2 = state.forceFloat(*args[1], pos);
     if (f2 == 0) throw EvalError(format("division by zero, at %1%") % pos);
 
@@ -1665,6 +1735,20 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
     }
 }
 
+static void prim_bitAnd(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    mkInt(v, state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos));
+}
+
+static void prim_bitOr(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    mkInt(v, state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos));
+}
+
+static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    mkInt(v, state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos));
+}
 
 static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -1716,41 +1800,6 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args
 }
 
 
-static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    PathSet context;
-    string s = state.coerceToString(pos, *args[0], context);
-    mkString(v, s, PathSet());
-}
-
-
-static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    PathSet context;
-    state.forceString(*args[0], context, pos);
-    mkBool(v, !context.empty());
-}
-
-
-/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
-   builder without causing the derivation to be built (for instance,
-   in the derivation that builds NARs in nix-push, when doing
-   source-only deployment).  This primop marks the string context so
-   that builtins.derivation adds the path to drv.inputSrcs rather than
-   drv.inputDrvs. */
-static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    PathSet context;
-    string s = state.coerceToString(pos, *args[0], context);
-
-    PathSet context2;
-    for (auto & p : context)
-        context2.insert(p.at(0) == '=' ? "~" + string(p, 1) : p);
-
-    mkString(v, s, context2);
-}
-
-
 /* Return the cryptographic hash of a string in base-16. */
 static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -2031,7 +2080,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
 
     state.checkURI(url);
 
-    if (settings.pureEval && !expectedHash)
+    if (evalSettings.pureEval && !expectedHash)
         throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
 
     Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash);
@@ -2099,12 +2148,12 @@ void EvalState::createBaseEnv()
         addConstant(name, v);
     };
 
-    if (!settings.pureEval) {
+    if (!evalSettings.pureEval) {
         mkInt(v, time(0));
         addConstant("__currentTime", v);
     }
 
-    if (!settings.pureEval) {
+    if (!evalSettings.pureEval) {
         mkString(v, settings.thisSystem);
         addConstant("__currentSystem", v);
     }
@@ -2129,7 +2178,7 @@ void EvalState::createBaseEnv()
     mkApp(v, *vScopedImport, *v2);
     forceValue(v);
     addConstant("import", v);
-    if (settings.enableNativeCode) {
+    if (evalSettings.enableNativeCode) {
         addPrimOp("__importNative", 2, prim_importNative);
         addPrimOp("__exec", 1, prim_exec);
     }
@@ -2140,6 +2189,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__isInt", 1, prim_isInt);
     addPrimOp("__isFloat", 1, prim_isFloat);
     addPrimOp("__isBool", 1, prim_isBool);
+    addPrimOp("__isPath", 1, prim_isPath);
     addPrimOp("__genericClosure", 1, prim_genericClosure);
     addPrimOp("abort", 1, prim_abort);
     addPrimOp("__addErrorContext", 2, prim_addErrorContext);
@@ -2156,7 +2206,7 @@ void EvalState::createBaseEnv()
 
     // Paths
     addPrimOp("__toPath", 1, prim_toPath);
-    if (settings.pureEval)
+    if (evalSettings.pureEval)
         addPurityError("__storePath");
     else
         addPrimOp("__storePath", 1, prim_storePath);
@@ -2166,6 +2216,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__readFile", 1, prim_readFile);
     addPrimOp("__readDir", 1, prim_readDir);
     addPrimOp("__findFile", 2, prim_findFile);
+    addPrimOp("__hashFile", 2, prim_hashFile);
 
     // Creating files
     addPrimOp("__toXML", 1, prim_toXML);
@@ -2187,6 +2238,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__intersectAttrs", 2, prim_intersectAttrs);
     addPrimOp("__catAttrs", 2, prim_catAttrs);
     addPrimOp("__functionArgs", 1, prim_functionArgs);
+    addPrimOp("__mapAttrs", 2, prim_mapAttrs);
 
     // Lists
     addPrimOp("__isList", 1, prim_isList);
@@ -2204,21 +2256,22 @@ void EvalState::createBaseEnv()
     addPrimOp("__genList", 2, prim_genList);
     addPrimOp("__sort", 2, prim_sort);
     addPrimOp("__partition", 2, prim_partition);
+    addPrimOp("__concatMap", 2, prim_concatMap);
 
     // Integer arithmetic
     addPrimOp("__add", 2, prim_add);
     addPrimOp("__sub", 2, prim_sub);
     addPrimOp("__mul", 2, prim_mul);
     addPrimOp("__div", 2, prim_div);
+    addPrimOp("__bitAnd", 2, prim_bitAnd);
+    addPrimOp("__bitOr", 2, prim_bitOr);
+    addPrimOp("__bitXor", 2, prim_bitXor);
     addPrimOp("__lessThan", 2, prim_lessThan);
 
     // String manipulation
     addPrimOp("toString", 1, prim_toString);
     addPrimOp("__substring", 3, prim_substring);
     addPrimOp("__stringLength", 1, prim_stringLength);
-    addPrimOp("__hasContext", 1, prim_hasContext);
-    addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
-    addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
     addPrimOp("__hashString", 2, prim_hashString);
     addPrimOp("__match", 2, prim_match);
     addPrimOp("__split", 2, prim_split);
diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc
new file mode 100644
index 000000000000..2d79739ea047
--- /dev/null
+++ b/src/libexpr/primops/context.cc
@@ -0,0 +1,187 @@
+#include "primops.hh"
+#include "eval-inline.hh"
+#include "derivations.hh"
+
+namespace nix {
+
+static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    string s = state.coerceToString(pos, *args[0], context);
+    mkString(v, s, PathSet());
+}
+
+static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
+
+
+static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    state.forceString(*args[0], context, pos);
+    mkBool(v, !context.empty());
+}
+
+static RegisterPrimOp r2("__hasContext", 1, prim_hasContext);
+
+
+/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
+   builder without causing the derivation to be built (for instance,
+   in the derivation that builds NARs in nix-push, when doing
+   source-only deployment).  This primop marks the string context so
+   that builtins.derivation adds the path to drv.inputSrcs rather than
+   drv.inputDrvs. */
+static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    string s = state.coerceToString(pos, *args[0], context);
+
+    PathSet context2;
+    for (auto & p : context)
+        context2.insert(p.at(0) == '=' ? string(p, 1) : p);
+
+    mkString(v, s, context2);
+}
+
+static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
+
+
+/* Extract the context of a string as a structured Nix value.
+
+   The context is represented as an attribute set whose keys are the
+   paths in the context set and whose values are attribute sets with
+   the following keys:
+     path: True if the relevant path is in the context as a plain store
+           path (i.e. the kind of context you get when interpolating
+           a Nix path (e.g. ./.) into a string). False if missing.
+     allOutputs: True if the relevant path is a derivation and it is
+                  in the context as a drv file with all of its outputs
+                  (i.e. the kind of context you get when referencing
+                  .drvPath of some derivation). False if missing.
+     outputs: If a non-empty list, the relevant path is a derivation
+              and the provided outputs are referenced in the context
+              (i.e. the kind of context you get when referencing
+              .outPath of some derivation). Empty list if missing.
+   Note that for a given path any combination of the above attributes
+   may be present.
+*/
+static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    struct ContextInfo {
+        bool path = false;
+        bool allOutputs = false;
+        Strings outputs;
+    };
+    PathSet context;
+    state.forceString(*args[0], context, pos);
+    auto contextInfos = std::map<Path, ContextInfo>();
+    for (const auto & p : context) {
+        Path drv;
+        string output;
+        const Path * path = &p;
+        if (p.at(0) == '=') {
+            drv = string(p, 1);
+            path = &drv;
+        } else if (p.at(0) == '!') {
+            std::pair<string, string> ctx = decodeContext(p);
+            drv = ctx.first;
+            output = ctx.second;
+            path = &drv;
+        }
+        auto isPath = drv.empty();
+        auto isAllOutputs = (!drv.empty()) && output.empty();
+
+        auto iter = contextInfos.find(*path);
+        if (iter == contextInfos.end()) {
+            contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}});
+        } else {
+            if (isPath)
+                iter->second.path = true;
+            else if (isAllOutputs)
+                iter->second.allOutputs = true;
+            else
+                iter->second.outputs.emplace_back(std::move(output));
+        }
+    }
+
+    state.mkAttrs(v, contextInfos.size());
+
+    auto sPath = state.symbols.create("path");
+    auto sAllOutputs = state.symbols.create("allOutputs");
+    for (const auto & info : contextInfos) {
+        auto & infoVal = *state.allocAttr(v, state.symbols.create(info.first));
+        state.mkAttrs(infoVal, 3);
+        if (info.second.path)
+            mkBool(*state.allocAttr(infoVal, sPath), true);
+        if (info.second.allOutputs)
+            mkBool(*state.allocAttr(infoVal, sAllOutputs), true);
+        if (!info.second.outputs.empty()) {
+            auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs);
+            state.mkList(outputsVal, info.second.outputs.size());
+            size_t i = 0;
+            for (const auto & output : info.second.outputs) {
+                mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output);
+            }
+        }
+        infoVal.attrs->sort();
+    }
+    v.attrs->sort();
+}
+
+static RegisterPrimOp r4("__getContext", 1, prim_getContext);
+
+
+/* Append the given context to a given string.
+
+   See the commentary above unsafeGetContext for details of the
+   context representation.
+*/
+static void prim_appendContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    auto orig = state.forceString(*args[0], context, pos);
+
+    state.forceAttrs(*args[1], pos);
+
+    auto sPath = state.symbols.create("path");
+    auto sAllOutputs = state.symbols.create("allOutputs");
+    for (auto & i : *args[1]->attrs) {
+        if (!state.store->isStorePath(i.name))
+            throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos);
+        if (!settings.readOnlyMode)
+            state.store->ensurePath(i.name);
+        state.forceAttrs(*i.value, *i.pos);
+        auto iter = i.value->attrs->find(sPath);
+        if (iter != i.value->attrs->end()) {
+            if (state.forceBool(*iter->value, *iter->pos))
+                context.insert(i.name);
+        }
+
+        iter = i.value->attrs->find(sAllOutputs);
+        if (iter != i.value->attrs->end()) {
+            if (state.forceBool(*iter->value, *iter->pos)) {
+                if (!isDerivation(i.name)) {
+                    throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+                }
+                context.insert("=" + string(i.name));
+            }
+        }
+
+        iter = i.value->attrs->find(state.sOutputs);
+        if (iter != i.value->attrs->end()) {
+            state.forceList(*iter->value, *iter->pos);
+            if (iter->value->listSize() && !isDerivation(i.name)) {
+                throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+            }
+            for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
+                auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);
+                context.insert("!" + name + "!" + string(i.name));
+            }
+        }
+    }
+
+    mkString(v, orig, context);
+}
+
+static RegisterPrimOp r5("__appendContext", 2, prim_appendContext);
+
+}
diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 8bb74dad639e..aaf02c856d4f 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -3,6 +3,7 @@
 #include "download.hh"
 #include "store-api.hh"
 #include "pathlocks.hh"
+#include "hash.hh"
 
 #include <sys/time.h>
 
@@ -25,10 +26,10 @@ struct GitInfo
 std::regex revRegex("^[0-9a-fA-F]{40}$");
 
 GitInfo exportGit(ref<Store> store, const std::string & uri,
-    std::experimental::optional<std::string> ref, std::string rev,
+    std::optional<std::string> ref, std::string rev,
     const std::string & name)
 {
-    if (settings.pureEval && rev == "")
+    if (evalSettings.pureEval && rev == "")
         throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
 
     if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
@@ -84,15 +85,16 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
     if (rev != "" && !std::regex_match(rev, revRegex))
         throw Error("invalid Git revision '%s'", rev);
 
-    Path cacheDir = getCacheDir() + "/nix/git";
+    deletePath(getCacheDir() + "/nix/git");
+
+    Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
 
     if (!pathExists(cacheDir)) {
+        createDirs(dirOf(cacheDir));
         runProgram("git", true, { "init", "--bare", cacheDir });
     }
 
-    std::string localRef = hashString(htSHA256, fmt("%s-%s", uri, *ref)).to_string(Base32, false);
-
-    Path localRefFile = cacheDir + "/refs/heads/" + localRef;
+    Path localRefFile = cacheDir + "/refs/heads/" + *ref;
 
     bool doFetch;
     time_t now = time(0);
@@ -122,7 +124,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
         // FIXME: git stderr messes up our progress indicator, so
         // we're using --quiet for now. Should process its stderr.
-        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, *ref + ":" + localRef });
+        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });
 
         struct timeval times[2];
         times[0].tv_sec = now;
@@ -188,7 +190,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     std::string url;
-    std::experimental::optional<std::string> ref;
+    std::optional<std::string> ref;
     std::string rev;
     std::string name = "source";
     PathSet context;
@@ -219,8 +221,6 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
 
-    if (!isUri(url)) url = absPath(url);
-
     // FIXME: git externals probably can be used to bypass the URI
     // whitelist. Ah well.
     state.checkURI(url);
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index a75c5fc2ddff..66f49f374321 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -27,7 +27,7 @@ std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
 HgInfo exportMercurial(ref<Store> store, const std::string & uri,
     std::string rev, const std::string & name)
 {
-    if (settings.pureEval && rev == "")
+    if (evalSettings.pureEval && rev == "")
         throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
 
     if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {
@@ -93,7 +93,22 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
             Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));
 
             if (pathExists(cacheDir)) {
-                runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
+                try {
+                    runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
+                }
+                catch (ExecError & e){
+                    string transJournal = cacheDir + "/.hg/store/journal";
+                    /* hg throws "abandoned transaction" error only if this file exists */
+                    if (pathExists(transJournal))
+                    {
+                        runProgram("hg", true, { "recover", "-R", cacheDir });
+                        runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
+                    }
+                    else 
+                    {
+                        throw ExecError(e.status, fmt("program hg '%1%' ", statusToString(e.status)));
+                    }
+                }
             } else {
                 createDirs(dirOf(cacheDir));
                 runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir });
@@ -184,8 +199,6 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
 
-    if (!isUri(url)) url = absPath(url);
-
     // FIXME: git externals probably can be used to bypass the URI
     // whitelist. Ah well.
     state.checkURI(url);
diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc
new file mode 100644
index 000000000000..4128de05d0cf
--- /dev/null
+++ b/src/libexpr/primops/fromTOML.cc
@@ -0,0 +1,77 @@
+#include "primops.hh"
+#include "eval-inline.hh"
+
+#include "cpptoml/cpptoml.h"
+
+namespace nix {
+
+static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    using namespace cpptoml;
+
+    auto toml = state.forceStringNoCtx(*args[0], pos);
+
+    std::istringstream tomlStream(toml);
+
+    std::function<void(Value &, std::shared_ptr<base>)> visit;
+
+    visit = [&](Value & v, std::shared_ptr<base> t) {
+
+        if (auto t2 = t->as_table()) {
+
+            size_t size = 0;
+            for (auto & i : *t2) { (void) i; size++; }
+
+            state.mkAttrs(v, size);
+
+            for (auto & i : *t2) {
+                auto & v2 = *state.allocAttr(v, state.symbols.create(i.first));
+
+                if (auto i2 = i.second->as_table_array()) {
+                    size_t size2 = i2->get().size();
+                    state.mkList(v2, size2);
+                    for (size_t j = 0; j < size2; ++j)
+                        visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]);
+                }
+                else
+                    visit(v2, i.second);
+            }
+
+            v.attrs->sort();
+        }
+
+        else if (auto t2 = t->as_array()) {
+            size_t size = t2->get().size();
+
+            state.mkList(v, size);
+
+            for (size_t i = 0; i < size; ++i)
+                visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]);
+        }
+
+        else if (t->is_value()) {
+            if (auto val = t->as<int64_t>())
+                mkInt(v, val->get());
+            else if (auto val = t->as<NixFloat>())
+                mkFloat(v, val->get());
+            else if (auto val = t->as<bool>())
+                mkBool(v, val->get());
+            else if (auto val = t->as<std::string>())
+                mkString(v, val->get());
+            else
+                throw EvalError("unsupported value type in TOML");
+        }
+
+        else abort();
+    };
+
+    try {
+        visit(v, parser(tomlStream).parse());
+    } catch (std::runtime_error & e) {
+        throw EvalError("while parsing a TOML string at %s: %s", pos, e.what());
+    }
+}
+
+static RegisterPrimOp r("fromTOML", 1, prim_fromTOML);
+
+}
diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh
index c2ee49dd32fb..91faea122ce1 100644
--- a/src/libexpr/symbol-table.hh
+++ b/src/libexpr/symbol-table.hh
@@ -69,12 +69,19 @@ public:
         return Symbol(&*res.first);
     }
 
-    unsigned int size() const
+    size_t size() const
     {
         return symbols.size();
     }
 
     size_t totalSize() const;
+
+    template<typename T>
+    void dump(T callback)
+    {
+        for (auto & s : symbols)
+            callback(s);
+    }
 };
 
 }
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 9df516f062ef..e1ec87d3b84c 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -43,8 +43,8 @@ class XMLWriter;
 class JSONPlaceholder;
 
 
-typedef long NixInt;
-typedef float NixFloat;
+typedef int64_t NixInt;
+typedef double NixFloat;
 
 /* External values must descend from ExternalValueBase, so that
  * type-agnostic nix functions (e.g. showType) can be implemented
@@ -128,7 +128,7 @@ struct Value
         const char * path;
         Bindings * attrs;
         struct {
-            unsigned int size;
+            size_t size;
             Value * * elems;
         } bigList;
         Value * smallList[2];
@@ -166,7 +166,7 @@ struct Value
         return type == tList1 || type == tList2 ? smallList : bigList.elems;
     }
 
-    unsigned int listSize() const
+    size_t listSize() const
     {
         return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
     }
generated by cgit-pink 1.4.1 (git 2.44.2) at 2025-01-28T07·50+0000