about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/build-remote/build-remote.cc30
-rw-r--r--src/build-remote/local.mk9
-rw-r--r--src/cpptoml/LICENSE18
-rw-r--r--src/cpptoml/cpptoml.h3457
-rw-r--r--src/libexpr/attr-set.cc24
-rw-r--r--src/libexpr/eval-inline.hh14
-rw-r--r--src/libexpr/eval.cc355
-rw-r--r--src/libexpr/eval.hh61
-rw-r--r--src/libexpr/get-drvs.cc2
-rw-r--r--src/libexpr/get-drvs.hh4
-rw-r--r--src/libexpr/json-to-value.cc2
-rw-r--r--src/libexpr/lexer.l49
-rw-r--r--src/libexpr/local.mk2
-rw-r--r--src/libexpr/nix-expr.pc.in2
-rw-r--r--src/libexpr/nixexpr.hh3
-rw-r--r--src/libexpr/parser.y27
-rw-r--r--src/libexpr/primops.cc211
-rw-r--r--src/libexpr/primops/context.cc187
-rw-r--r--src/libexpr/primops/fetchGit.cc20
-rw-r--r--src/libexpr/primops/fetchMercurial.cc21
-rw-r--r--src/libexpr/primops/fromTOML.cc77
-rw-r--r--src/libexpr/symbol-table.hh9
-rw-r--r--src/libexpr/value.hh8
-rw-r--r--src/libmain/common-args.cc4
-rw-r--r--src/libmain/local.mk2
-rw-r--r--src/libmain/nix-main.pc.in2
-rw-r--r--src/libmain/shared.cc2
-rw-r--r--src/libmain/stack.cc4
-rw-r--r--src/libstore/binary-cache-store.cc82
-rw-r--r--src/libstore/binary-cache-store.hh49
-rw-r--r--src/libstore/build.cc617
-rw-r--r--src/libstore/builtins/fetchurl.cc70
-rw-r--r--src/libstore/derivations.cc24
-rw-r--r--src/libstore/derivations.hh4
-rw-r--r--src/libstore/download.cc320
-rw-r--r--src/libstore/download.hh20
-rw-r--r--src/libstore/gc.cc147
-rw-r--r--src/libstore/globals.cc55
-rw-r--r--src/libstore/globals.hh92
-rw-r--r--src/libstore/http-binary-cache-store.cc79
-rw-r--r--src/libstore/legacy-ssh-store.cc116
-rw-r--r--src/libstore/local-binary-cache-store.cc18
-rw-r--r--src/libstore/local-store.cc37
-rw-r--r--src/libstore/local-store.hh13
-rw-r--r--src/libstore/local.mk4
-rw-r--r--src/libstore/machines.cc9
-rw-r--r--src/libstore/misc.cc17
-rw-r--r--src/libstore/nar-info-disk-cache.cc35
-rw-r--r--src/libstore/nix-store.pc.in2
-rw-r--r--src/libstore/optimise-store.cc3
-rw-r--r--src/libstore/parsed-derivations.cc111
-rw-r--r--src/libstore/parsed-derivations.hh35
-rw-r--r--src/libstore/profiles.cc23
-rw-r--r--src/libstore/profiles.hh2
-rw-r--r--src/libstore/references.cc6
-rw-r--r--src/libstore/remote-store.cc255
-rw-r--r--src/libstore/remote-store.hh22
-rw-r--r--src/libstore/s3-binary-cache-store.cc180
-rw-r--r--src/libstore/s3.hh4
-rw-r--r--src/libstore/serve-protocol.hh3
-rw-r--r--src/libstore/sqlite.cc3
-rw-r--r--src/libstore/ssh.cc22
-rw-r--r--src/libstore/ssh.hh1
-rw-r--r--src/libstore/store-api.cc160
-rw-r--r--src/libstore/store-api.hh61
-rw-r--r--src/libstore/worker-protocol.hh2
-rw-r--r--src/libutil/archive.cc30
-rw-r--r--src/libutil/archive.hh4
-rw-r--r--src/libutil/compression.cc604
-rw-r--r--src/libutil/compression.hh12
-rw-r--r--src/libutil/config.cc88
-rw-r--r--src/libutil/config.hh81
-rw-r--r--src/libutil/hash.cc36
-rw-r--r--src/libutil/hash.hh7
-rw-r--r--src/libutil/json.cc1
-rw-r--r--src/libutil/local.mk4
-rw-r--r--src/libutil/lru-cache.hh4
-rw-r--r--src/libutil/pool.hh6
-rw-r--r--src/libutil/serialise.cc44
-rw-r--r--src/libutil/serialise.hh21
-rw-r--r--src/libutil/sync.hh4
-rw-r--r--src/libutil/util.cc58
-rw-r--r--src/libutil/util.hh73
-rw-r--r--src/libutil/xml-writer.cc4
-rw-r--r--src/libutil/xml-writer.hh2
-rw-r--r--src/linenoise/ConvertUTF.cpp542
-rwxr-xr-xsrc/linenoise/ConvertUTF.h162
-rw-r--r--src/linenoise/LICENSE66
-rw-r--r--src/linenoise/linenoise.cpp3450
-rw-r--r--src/linenoise/linenoise.h73
-rw-r--r--src/linenoise/wcwidth.cpp315
-rw-r--r--src/nix-build/local.mk9
-rwxr-xr-xsrc/nix-build/nix-build.cc75
-rw-r--r--src/nix-channel/local.mk7
-rwxr-xr-xsrc/nix-channel/nix-channel.cc20
-rw-r--r--src/nix-collect-garbage/local.mk7
-rw-r--r--src/nix-collect-garbage/nix-collect-garbage.cc15
-rw-r--r--src/nix-copy-closure/local.mk7
-rwxr-xr-xsrc/nix-copy-closure/nix-copy-closure.cc13
-rw-r--r--src/nix-daemon/local.mk13
-rw-r--r--src/nix-daemon/nix-daemon.cc64
-rw-r--r--src/nix-env/local.mk7
-rw-r--r--src/nix-env/nix-env.cc46
-rw-r--r--src/nix-instantiate/local.mk7
-rw-r--r--src/nix-instantiate/nix-instantiate.cc36
-rw-r--r--src/nix-prefetch-url/local.mk7
-rw-r--r--src/nix-prefetch-url/nix-prefetch-url.cc75
-rw-r--r--src/nix-store/dotgraph.cc3
-rw-r--r--src/nix-store/graphml.cc90
-rw-r--r--src/nix-store/graphml.hh (renamed from src/nix-store/xmlgraph.hh)2
-rw-r--r--src/nix-store/local.mk9
-rw-r--r--src/nix-store/nix-store.cc82
-rw-r--r--src/nix-store/xmlgraph.cc66
-rw-r--r--src/nix/add-to-store.cc2
-rw-r--r--src/nix/copy.cc8
-rw-r--r--src/nix/doctor.cc124
-rw-r--r--src/nix/edit.cc4
-rw-r--r--src/nix/hash.cc23
-rw-r--r--src/nix/installables.cc26
-rw-r--r--src/nix/local.mk22
-rw-r--r--src/nix/ls.cc2
-rw-r--r--src/nix/main.cc24
-rw-r--r--src/nix/path-info.cc36
-rw-r--r--src/nix/progress-bar.cc54
-rw-r--r--src/nix/progress-bar.hh2
-rw-r--r--src/nix/repl.cc177
-rw-r--r--src/nix/run.cc26
-rw-r--r--src/nix/search.cc88
-rw-r--r--src/nix/show-config.cc8
-rw-r--r--src/nix/upgrade-nix.cc53
-rw-r--r--src/nix/verify.cc2
-rw-r--r--src/nix/why-depends.cc2
-rw-r--r--src/nlohmann/json.hpp10830
-rw-r--r--src/resolve-system-dependencies/local.mk2
134 files changed, 15631 insertions, 9544 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 9cd01bb61bf5..279ae62f69cc 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -17,6 +17,7 @@
 #include "store-api.hh"
 #include "derivations.hh"
 #include "local-store.hh"
+#include "legacy.hh"
 
 using namespace nix;
 using std::cin;
@@ -37,11 +38,15 @@ static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot)
     return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
 }
 
-int main (int argc, char * * argv)
-{
-    return handleExceptions(argv[0], [&]() {
-        initNix();
+static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) {
+    for (auto & feature : requiredFeatures)
+        if (!settings.systemFeatures.get().count(feature)) return false;
+    return true;
+}
 
+static int _main(int argc, char * * argv)
+{
+    {
         logger = makeJSONLogger(*logger);
 
         /* Ensure we don't get any SSH passphrase or host key popups. */
@@ -80,7 +85,7 @@ int main (int argc, char * * argv)
 
         if (machines.empty()) {
             std::cerr << "# decline-permanently\n";
-            return;
+            return 0;
         }
 
         string drvPath;
@@ -90,15 +95,18 @@ int main (int argc, char * * argv)
 
             try {
                 auto s = readString(source);
-                if (s != "try") return;
-            } catch (EndOfFile &) { return; }
+                if (s != "try") return 0;
+            } catch (EndOfFile &) { return 0; }
 
             auto amWilling = readInt(source);
             auto neededSystem = readString(source);
             source >> drvPath;
             auto requiredFeatures = readStrings<std::set<std::string>>(source);
 
-            auto canBuildLocally = amWilling && (neededSystem == settings.thisSystem);
+             auto canBuildLocally = amWilling
+                 &&  (  neededSystem == settings.thisSystem
+                     || settings.extraPlatforms.get().count(neededSystem) > 0)
+                 &&  allSupportedLocally(requiredFeatures);
 
             /* Error ignored here, will be caught later */
             mkdir(currentLoad.c_str(), 0777);
@@ -251,6 +259,8 @@ connected:
             copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute);
         }
 
-        return;
-    });
+        return 0;
+    }
 }
+
+static RegisterLegacyCommand s1("build-remote", _main);
diff --git a/src/build-remote/local.mk b/src/build-remote/local.mk
deleted file mode 100644
index 64368a43ff73..000000000000
--- a/src/build-remote/local.mk
+++ /dev/null
@@ -1,9 +0,0 @@
-programs += build-remote
-
-build-remote_DIR := $(d)
-
-build-remote_INSTALL_DIR := $(libexecdir)/nix
-
-build-remote_LIBS = libmain libutil libformat libstore
-
-build-remote_SOURCES := $(d)/build-remote.cc
diff --git a/src/cpptoml/LICENSE b/src/cpptoml/LICENSE
new file mode 100644
index 000000000000..8802c4fa5a36
--- /dev/null
+++ b/src/cpptoml/LICENSE
@@ -0,0 +1,18 @@
+Copyright (c) 2014 Chase Geigle
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/src/cpptoml/cpptoml.h b/src/cpptoml/cpptoml.h
new file mode 100644
index 000000000000..07de010b1520
--- /dev/null
+++ b/src/cpptoml/cpptoml.h
@@ -0,0 +1,3457 @@
+/**
+ * @file cpptoml.h
+ * @author Chase Geigle
+ * @date May 2013
+ */
+
+#ifndef _CPPTOML_H_
+#define _CPPTOML_H_
+
+#include <algorithm>
+#include <cassert>
+#include <clocale>
+#include <cstdint>
+#include <cstring>
+#include <fstream>
+#include <iomanip>
+#include <map>
+#include <memory>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#if __cplusplus > 201103L
+#define CPPTOML_DEPRECATED(reason) [[deprecated(reason)]]
+#elif defined(__clang__)
+#define CPPTOML_DEPRECATED(reason) __attribute__((deprecated(reason)))
+#elif defined(__GNUG__)
+#define CPPTOML_DEPRECATED(reason) __attribute__((deprecated))
+#elif defined(_MSC_VER)
+#if _MSC_VER < 1910
+#define CPPTOML_DEPRECATED(reason) __declspec(deprecated)
+#else
+#define CPPTOML_DEPRECATED(reason) [[deprecated(reason)]]
+#endif
+#endif
+
+namespace cpptoml
+{
+class writer; // forward declaration
+class base;   // forward declaration
+#if defined(CPPTOML_USE_MAP)
+// a std::map will ensure that entries a sorted, albeit at a slight
+// performance penalty relative to the (default) unordered_map
+using string_to_base_map = std::map<std::string, std::shared_ptr<base>>;
+#else
+// by default an unordered_map is used for best performance as the
+// toml specification does not require entries to be sorted
+using string_to_base_map
+    = std::unordered_map<std::string, std::shared_ptr<base>>;
+#endif
+
+// if defined, `base` will retain type information in form of an enum class
+// such that static_cast can be used instead of dynamic_cast
+// #define CPPTOML_NO_RTTI
+
+template <class T>
+class option
+{
+  public:
+    option() : empty_{true}
+    {
+        // nothing
+    }
+
+    option(T value) : empty_{false}, value_(std::move(value))
+    {
+        // nothing
+    }
+
+    explicit operator bool() const
+    {
+        return !empty_;
+    }
+
+    const T& operator*() const
+    {
+        return value_;
+    }
+
+    const T* operator->() const
+    {
+        return &value_;
+    }
+
+    const T& value_or(const T& alternative) const
+    {
+        if (!empty_)
+            return value_;
+        return alternative;
+    }
+
+  private:
+    bool empty_;
+    T value_;
+};
+
+struct local_date
+{
+    int year = 0;
+    int month = 0;
+    int day = 0;
+};
+
+struct local_time
+{
+    int hour = 0;
+    int minute = 0;
+    int second = 0;
+    int microsecond = 0;
+};
+
+struct zone_offset
+{
+    int hour_offset = 0;
+    int minute_offset = 0;
+};
+
+struct local_datetime : local_date, local_time
+{
+};
+
+struct offset_datetime : local_datetime, zone_offset
+{
+    static inline struct offset_datetime from_zoned(const struct tm& t)
+    {
+        offset_datetime dt;
+        dt.year = t.tm_year + 1900;
+        dt.month = t.tm_mon + 1;
+        dt.day = t.tm_mday;
+        dt.hour = t.tm_hour;
+        dt.minute = t.tm_min;
+        dt.second = t.tm_sec;
+
+        char buf[16];
+        strftime(buf, 16, "%z", &t);
+
+        int offset = std::stoi(buf);
+        dt.hour_offset = offset / 100;
+        dt.minute_offset = offset % 100;
+        return dt;
+    }
+
+    CPPTOML_DEPRECATED("from_local has been renamed to from_zoned")
+    static inline struct offset_datetime from_local(const struct tm& t)
+    {
+        return from_zoned(t);
+    }
+
+    static inline struct offset_datetime from_utc(const struct tm& t)
+    {
+        offset_datetime dt;
+        dt.year = t.tm_year + 1900;
+        dt.month = t.tm_mon + 1;
+        dt.day = t.tm_mday;
+        dt.hour = t.tm_hour;
+        dt.minute = t.tm_min;
+        dt.second = t.tm_sec;
+        return dt;
+    }
+};
+
+CPPTOML_DEPRECATED("datetime has been renamed to offset_datetime")
+typedef offset_datetime datetime;
+
+class fill_guard
+{
+  public:
+    fill_guard(std::ostream& os) : os_(os), fill_{os.fill()}
+    {
+        // nothing
+    }
+
+    ~fill_guard()
+    {
+        os_.fill(fill_);
+    }
+
+  private:
+    std::ostream& os_;
+    std::ostream::char_type fill_;
+};
+
+inline std::ostream& operator<<(std::ostream& os, const local_date& dt)
+{
+    fill_guard g{os};
+    os.fill('0');
+
+    using std::setw;
+    os << setw(4) << dt.year << "-" << setw(2) << dt.month << "-" << setw(2)
+       << dt.day;
+
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const local_time& ltime)
+{
+    fill_guard g{os};
+    os.fill('0');
+
+    using std::setw;
+    os << setw(2) << ltime.hour << ":" << setw(2) << ltime.minute << ":"
+       << setw(2) << ltime.second;
+
+    if (ltime.microsecond > 0)
+    {
+        os << ".";
+        int power = 100000;
+        for (int curr_us = ltime.microsecond; curr_us; power /= 10)
+        {
+            auto num = curr_us / power;
+            os << num;
+            curr_us -= num * power;
+        }
+    }
+
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const zone_offset& zo)
+{
+    fill_guard g{os};
+    os.fill('0');
+
+    using std::setw;
+
+    if (zo.hour_offset != 0 || zo.minute_offset != 0)
+    {
+        if (zo.hour_offset > 0)
+        {
+            os << "+";
+        }
+        else
+        {
+            os << "-";
+        }
+        os << setw(2) << std::abs(zo.hour_offset) << ":" << setw(2)
+           << std::abs(zo.minute_offset);
+    }
+    else
+    {
+        os << "Z";
+    }
+
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const local_datetime& dt)
+{
+    return os << static_cast<const local_date&>(dt) << "T"
+              << static_cast<const local_time&>(dt);
+}
+
+inline std::ostream& operator<<(std::ostream& os, const offset_datetime& dt)
+{
+    return os << static_cast<const local_datetime&>(dt)
+              << static_cast<const zone_offset&>(dt);
+}
+
+template <class T, class... Ts>
+struct is_one_of;
+
+template <class T, class V>
+struct is_one_of<T, V> : std::is_same<T, V>
+{
+};
+
+template <class T, class V, class... Ts>
+struct is_one_of<T, V, Ts...>
+{
+    const static bool value
+        = std::is_same<T, V>::value || is_one_of<T, Ts...>::value;
+};
+
+template <class T>
+class value;
+
+template <class T>
+struct valid_value
+    : is_one_of<T, std::string, int64_t, double, bool, local_date, local_time,
+                local_datetime, offset_datetime>
+{
+};
+
+template <class T, class Enable = void>
+struct value_traits;
+
+template <class T>
+struct valid_value_or_string_convertible
+{
+
+    const static bool value = valid_value<typename std::decay<T>::type>::value
+                              || std::is_convertible<T, std::string>::value;
+};
+
+template <class T>
+struct value_traits<T, typename std::
+                           enable_if<valid_value_or_string_convertible<T>::
+                                         value>::type>
+{
+    using value_type = typename std::
+        conditional<valid_value<typename std::decay<T>::type>::value,
+                    typename std::decay<T>::type, std::string>::type;
+
+    using type = value<value_type>;
+
+    static value_type construct(T&& val)
+    {
+        return value_type(val);
+    }
+};
+
+template <class T>
+struct value_traits<T,
+                    typename std::
+                        enable_if<!valid_value_or_string_convertible<T>::value
+                                  && std::is_floating_point<
+                                         typename std::decay<T>::type>::value>::
+                            type>
+{
+    using value_type = typename std::decay<T>::type;
+
+    using type = value<double>;
+
+    static value_type construct(T&& val)
+    {
+        return value_type(val);
+    }
+};
+
+template <class T>
+struct value_traits<T,
+                    typename std::
+                        enable_if<!valid_value_or_string_convertible<T>::value
+                                  && std::is_signed<typename std::decay<T>::
+                                                        type>::value>::type>
+{
+    using value_type = int64_t;
+
+    using type = value<int64_t>;
+
+    static value_type construct(T&& val)
+    {
+        if (val < (std::numeric_limits<int64_t>::min)())
+            throw std::underflow_error{"constructed value cannot be "
+                                       "represented by a 64-bit signed "
+                                       "integer"};
+
+        if (val > (std::numeric_limits<int64_t>::max)())
+            throw std::overflow_error{"constructed value cannot be represented "
+                                      "by a 64-bit signed integer"};
+
+        return static_cast<int64_t>(val);
+    }
+};
+
+template <class T>
+struct value_traits<T,
+                    typename std::
+                        enable_if<!valid_value_or_string_convertible<T>::value
+                                  && std::is_unsigned<typename std::decay<T>::
+                                                          type>::value>::type>
+{
+    using value_type = int64_t;
+
+    using type = value<int64_t>;
+
+    static value_type construct(T&& val)
+    {
+        if (val > static_cast<uint64_t>((std::numeric_limits<int64_t>::max)()))
+            throw std::overflow_error{"constructed value cannot be represented "
+                                      "by a 64-bit signed integer"};
+
+        return static_cast<int64_t>(val);
+    }
+};
+
+class array;
+class table;
+class table_array;
+
+template <class T>
+struct array_of_trait
+{
+    using return_type = option<std::vector<T>>;
+};
+
+template <>
+struct array_of_trait<array>
+{
+    using return_type = option<std::vector<std::shared_ptr<array>>>;
+};
+
+template <class T>
+inline std::shared_ptr<typename value_traits<T>::type> make_value(T&& val);
+inline std::shared_ptr<array> make_array();
+template <class T>
+inline std::shared_ptr<T> make_element();
+inline std::shared_ptr<table> make_table();
+inline std::shared_ptr<table_array> make_table_array();
+
+#if defined(CPPTOML_NO_RTTI)
+/// Base type used to store underlying data type explicitly if RTTI is disabled
+enum class base_type
+{
+    NONE,
+    STRING,
+    LOCAL_TIME,
+    LOCAL_DATE,
+    LOCAL_DATETIME,
+    OFFSET_DATETIME,
+    INT,
+    FLOAT,
+    BOOL,
+    TABLE,
+    ARRAY,
+    TABLE_ARRAY
+};
+
+/// Type traits class to convert C++ types to enum member
+template <class T>
+struct base_type_traits;
+
+template <>
+struct base_type_traits<std::string>
+{
+    static const base_type type = base_type::STRING;
+};
+
+template <>
+struct base_type_traits<local_time>
+{
+    static const base_type type = base_type::LOCAL_TIME;
+};
+
+template <>
+struct base_type_traits<local_date>
+{
+    static const base_type type = base_type::LOCAL_DATE;
+};
+
+template <>
+struct base_type_traits<local_datetime>
+{
+    static const base_type type = base_type::LOCAL_DATETIME;
+};
+
+template <>
+struct base_type_traits<offset_datetime>
+{
+    static const base_type type = base_type::OFFSET_DATETIME;
+};
+
+template <>
+struct base_type_traits<int64_t>
+{
+    static const base_type type = base_type::INT;
+};
+
+template <>
+struct base_type_traits<double>
+{
+    static const base_type type = base_type::FLOAT;
+};
+
+template <>
+struct base_type_traits<bool>
+{
+    static const base_type type = base_type::BOOL;
+};
+
+template <>
+struct base_type_traits<table>
+{
+    static const base_type type = base_type::TABLE;
+};
+
+template <>
+struct base_type_traits<array>
+{
+    static const base_type type = base_type::ARRAY;
+};
+
+template <>
+struct base_type_traits<table_array>
+{
+    static const base_type type = base_type::TABLE_ARRAY;
+};
+#endif
+
+/**
+ * A generic base TOML value used for type erasure.
+ */
+class base : public std::enable_shared_from_this<base>
+{
+  public:
+    virtual ~base() = default;
+
+    virtual std::shared_ptr<base> clone() const = 0;
+
+    /**
+     * Determines if the given TOML element is a value.
+     */
+    virtual bool is_value() const
+    {
+        return false;
+    }
+
+    /**
+     * Determines if the given TOML element is a table.
+     */
+    virtual bool is_table() const
+    {
+        return false;
+    }
+
+    /**
+     * Converts the TOML element into a table.
+     */
+    std::shared_ptr<table> as_table()
+    {
+        if (is_table())
+            return std::static_pointer_cast<table>(shared_from_this());
+        return nullptr;
+    }
+    /**
+     * Determines if the TOML element is an array of "leaf" elements.
+     */
+    virtual bool is_array() const
+    {
+        return false;
+    }
+
+    /**
+     * Converts the TOML element to an array.
+     */
+    std::shared_ptr<array> as_array()
+    {
+        if (is_array())
+            return std::static_pointer_cast<array>(shared_from_this());
+        return nullptr;
+    }
+
+    /**
+     * Determines if the given TOML element is an array of tables.
+     */
+    virtual bool is_table_array() const
+    {
+        return false;
+    }
+
+    /**
+     * Converts the TOML element into a table array.
+     */
+    std::shared_ptr<table_array> as_table_array()
+    {
+        if (is_table_array())
+            return std::static_pointer_cast<table_array>(shared_from_this());
+        return nullptr;
+    }
+
+    /**
+     * Attempts to coerce the TOML element into a concrete TOML value
+     * of type T.
+     */
+    template <class T>
+    std::shared_ptr<value<T>> as();
+
+    template <class T>
+    std::shared_ptr<const value<T>> as() const;
+
+    template <class Visitor, class... Args>
+    void accept(Visitor&& visitor, Args&&... args) const;
+
+#if defined(CPPTOML_NO_RTTI)
+    base_type type() const
+    {
+      return type_;
+    }
+
+  protected:
+    base(const base_type t) : type_(t)
+    {
+        // nothing
+    }
+
+  private:
+    const base_type type_ = base_type::NONE;
+
+#else
+  protected:
+    base()
+    {
+        // nothing
+    }
+#endif
+};
+
+/**
+ * A concrete TOML value representing the "leaves" of the "tree".
+ */
+template <class T>
+class value : public base
+{
+    struct make_shared_enabler
+    {
+        // nothing; this is a private key accessible only to friends
+    };
+
+    template <class U>
+    friend std::shared_ptr<typename value_traits<U>::type>
+    cpptoml::make_value(U&& val);
+
+  public:
+    static_assert(valid_value<T>::value, "invalid value type");
+
+    std::shared_ptr<base> clone() const override;
+
+    value(const make_shared_enabler&, const T& val) : value(val)
+    {
+        // nothing; note that users cannot actually invoke this function
+        // because they lack access to the make_shared_enabler.
+    }
+
+    bool is_value() const override
+    {
+        return true;
+    }
+
+    /**
+     * Gets the data associated with this value.
+     */
+    T& get()
+    {
+        return data_;
+    }
+
+    /**
+     * Gets the data associated with this value. Const version.
+     */
+    const T& get() const
+    {
+        return data_;
+    }
+
+  private:
+    T data_;
+
+    /**
+     * Constructs a value from the given data.
+     */
+#if defined(CPPTOML_NO_RTTI)
+    value(const T& val) : base(base_type_traits<T>::type), data_(val)
+    {
+    }
+#else
+    value(const T& val) : data_(val)
+    {
+    }
+#endif
+
+    value(const value& val) = delete;
+    value& operator=(const value& val) = delete;
+};
+
+template <class T>
+std::shared_ptr<typename value_traits<T>::type> make_value(T&& val)
+{
+    using value_type = typename value_traits<T>::type;
+    using enabler = typename value_type::make_shared_enabler;
+    return std::make_shared<value_type>(
+        enabler{}, value_traits<T>::construct(std::forward<T>(val)));
+}
+
+template <class T>
+inline std::shared_ptr<value<T>> base::as()
+{
+#if defined(CPPTOML_NO_RTTI)
+    if (type() == base_type_traits<T>::type)
+        return std::static_pointer_cast<value<T>>(shared_from_this());
+    else
+        return nullptr;
+#else
+    return std::dynamic_pointer_cast<value<T>>(shared_from_this());
+#endif
+}
+
+// special case value<double> to allow getting an integer parameter as a
+// double value
+template <>
+inline std::shared_ptr<value<double>> base::as()
+{
+#if defined(CPPTOML_NO_RTTI)
+    if (type() == base_type::FLOAT)
+        return std::static_pointer_cast<value<double>>(shared_from_this());
+
+    if (type() == base_type::INT)
+    {
+        auto v = std::static_pointer_cast<value<int64_t>>(shared_from_this());
+        return make_value<double>(static_cast<double>(v->get()));;
+    }
+#else
+    if (auto v = std::dynamic_pointer_cast<value<double>>(shared_from_this()))
+        return v;
+
+    if (auto v = std::dynamic_pointer_cast<value<int64_t>>(shared_from_this()))
+        return make_value<double>(static_cast<double>(v->get()));
+#endif
+
+    return nullptr;
+}
+
+template <class T>
+inline std::shared_ptr<const value<T>> base::as() const
+{
+#if defined(CPPTOML_NO_RTTI)
+    if (type() == base_type_traits<T>::type)
+        return std::static_pointer_cast<const value<T>>(shared_from_this());
+    else
+        return nullptr;
+#else
+    return std::dynamic_pointer_cast<const value<T>>(shared_from_this());
+#endif
+}
+
+// special case value<double> to allow getting an integer parameter as a
+// double value
+template <>
+inline std::shared_ptr<const value<double>> base::as() const
+{
+#if defined(CPPTOML_NO_RTTI)
+    if (type() == base_type::FLOAT)
+        return std::static_pointer_cast<const value<double>>(shared_from_this());
+
+    if (type() == base_type::INT)
+    {
+        auto v = as<int64_t>();
+        // the below has to be a non-const value<double> due to a bug in
+        // libc++: https://llvm.org/bugs/show_bug.cgi?id=18843
+        return make_value<double>(static_cast<double>(v->get()));
+    }
+#else
+    if (auto v
+        = std::dynamic_pointer_cast<const value<double>>(shared_from_this()))
+        return v;
+
+    if (auto v = as<int64_t>())
+    {
+        // the below has to be a non-const value<double> due to a bug in
+        // libc++: https://llvm.org/bugs/show_bug.cgi?id=18843
+        return make_value<double>(static_cast<double>(v->get()));
+    }
+#endif
+
+    return nullptr;
+}
+
+/**
+ * Exception class for array insertion errors.
+ */
+class array_exception : public std::runtime_error
+{
+  public:
+    array_exception(const std::string& err) : std::runtime_error{err}
+    {
+    }
+};
+
+class array : public base
+{
+  public:
+    friend std::shared_ptr<array> make_array();
+
+    std::shared_ptr<base> clone() const override;
+
+    virtual bool is_array() const override
+    {
+        return true;
+    }
+
+    using size_type = std::size_t;
+
+    /**
+     * arrays can be iterated over
+     */
+    using iterator = std::vector<std::shared_ptr<base>>::iterator;
+
+    /**
+     * arrays can be iterated over.  Const version.
+     */
+    using const_iterator = std::vector<std::shared_ptr<base>>::const_iterator;
+
+    iterator begin()
+    {
+        return values_.begin();
+    }
+
+    const_iterator begin() const
+    {
+        return values_.begin();
+    }
+
+    iterator end()
+    {
+        return values_.end();
+    }
+
+    const_iterator end() const
+    {
+        return values_.end();
+    }
+
+    /**
+     * Obtains the array (vector) of base values.
+     */
+    std::vector<std::shared_ptr<base>>& get()
+    {
+        return values_;
+    }
+
+    /**
+     * Obtains the array (vector) of base values. Const version.
+     */
+    const std::vector<std::shared_ptr<base>>& get() const
+    {
+        return values_;
+    }
+
+    std::shared_ptr<base> at(size_t idx) const
+    {
+        return values_.at(idx);
+    }
+
+    /**
+     * Obtains an array of value<T>s. Note that elements may be
+     * nullptr if they cannot be converted to a value<T>.
+     */
+    template <class T>
+    std::vector<std::shared_ptr<value<T>>> array_of() const
+    {
+        std::vector<std::shared_ptr<value<T>>> result(values_.size());
+
+        std::transform(values_.begin(), values_.end(), result.begin(),
+                       [&](std::shared_ptr<base> v) { return v->as<T>(); });
+
+        return result;
+    }
+
+    /**
+     * Obtains a option<vector<T>>. The option will be empty if the array
+     * contains values that are not of type T.
+     */
+    template <class T>
+    inline typename array_of_trait<T>::return_type get_array_of() const
+    {
+        std::vector<T> result;
+        result.reserve(values_.size());
+
+        for (const auto& val : values_)
+        {
+            if (auto v = val->as<T>())
+                result.push_back(v->get());
+            else
+                return {};
+        }
+
+        return {std::move(result)};
+    }
+
+    /**
+     * Obtains an array of arrays. Note that elements may be nullptr
+     * if they cannot be converted to a array.
+     */
+    std::vector<std::shared_ptr<array>> nested_array() const
+    {
+        std::vector<std::shared_ptr<array>> result(values_.size());
+
+        std::transform(values_.begin(), values_.end(), result.begin(),
+                       [&](std::shared_ptr<base> v) -> std::shared_ptr<array> {
+                           if (v->is_array())
+                               return std::static_pointer_cast<array>(v);
+                           return std::shared_ptr<array>{};
+                       });
+
+        return result;
+    }
+
+    /**
+     * Add a value to the end of the array
+     */
+    template <class T>
+    void push_back(const std::shared_ptr<value<T>>& val)
+    {
+        if (values_.empty() || values_[0]->as<T>())
+        {
+            values_.push_back(val);
+        }
+        else
+        {
+            throw array_exception{"Arrays must be homogenous."};
+        }
+    }
+
+    /**
+     * Add an array to the end of the array
+     */
+    void push_back(const std::shared_ptr<array>& val)
+    {
+        if (values_.empty() || values_[0]->is_array())
+        {
+            values_.push_back(val);
+        }
+        else
+        {
+            throw array_exception{"Arrays must be homogenous."};
+        }
+    }
+
+    /**
+     * Convenience function for adding a simple element to the end
+     * of the array.
+     */
+    template <class T>
+    void push_back(T&& val, typename value_traits<T>::type* = 0)
+    {
+        push_back(make_value(std::forward<T>(val)));
+    }
+
+    /**
+     * Insert a value into the array
+     */
+    template <class T>
+    iterator insert(iterator position, const std::shared_ptr<value<T>>& value)
+    {
+        if (values_.empty() || values_[0]->as<T>())
+        {
+            return values_.insert(position, value);
+        }
+        else
+        {
+            throw array_exception{"Arrays must be homogenous."};
+        }
+    }
+
+    /**
+     * Insert an array into the array
+     */
+    iterator insert(iterator position, const std::shared_ptr<array>& value)
+    {
+        if (values_.empty() || values_[0]->is_array())
+        {
+            return values_.insert(position, value);
+        }
+        else
+        {
+            throw array_exception{"Arrays must be homogenous."};
+        }
+    }
+
+    /**
+     * Convenience function for inserting a simple element in the array
+     */
+    template <class T>
+    iterator insert(iterator position, T&& val,
+                    typename value_traits<T>::type* = 0)
+    {
+        return insert(position, make_value(std::forward<T>(val)));
+    }
+
+    /**
+     * Erase an element from the array
+     */
+    iterator erase(iterator position)
+    {
+        return values_.erase(position);
+    }
+
+    /**
+     * Clear the array
+     */
+    void clear()
+    {
+        values_.clear();
+    }
+
+    /**
+     * Reserve space for n values.
+     */
+    void reserve(size_type n)
+    {
+        values_.reserve(n);
+    }
+
+  private:
+#if defined(CPPTOML_NO_RTTI)
+    array() : base(base_type::ARRAY)
+    {
+        // empty
+    }
+#else
+    array() = default;
+#endif
+
+    template <class InputIterator>
+    array(InputIterator begin, InputIterator end) : values_{begin, end}
+    {
+        // nothing
+    }
+
+    array(const array& obj) = delete;
+    array& operator=(const array& obj) = delete;
+
+    std::vector<std::shared_ptr<base>> values_;
+};
+
+inline std::shared_ptr<array> make_array()
+{
+    struct make_shared_enabler : public array
+    {
+        make_shared_enabler()
+        {
+            // nothing
+        }
+    };
+
+    return std::make_shared<make_shared_enabler>();
+}
+
+template <>
+inline std::shared_ptr<array> make_element<array>()
+{
+    return make_array();
+}
+
+/**
+ * Obtains a option<vector<T>>. The option will be empty if the array
+ * contains values that are not of type T.
+ */
+template <>
+inline typename array_of_trait<array>::return_type
+array::get_array_of<array>() const
+{
+    std::vector<std::shared_ptr<array>> result;
+    result.reserve(values_.size());
+
+    for (const auto& val : values_)
+    {
+        if (auto v = val->as_array())
+            result.push_back(v);
+        else
+            return {};
+    }
+
+    return {std::move(result)};
+}
+
+class table;
+
+class table_array : public base
+{
+    friend class table;
+    friend std::shared_ptr<table_array> make_table_array();
+
+  public:
+    std::shared_ptr<base> clone() const override;
+
+    using size_type = std::size_t;
+
+    /**
+     * arrays can be iterated over
+     */
+    using iterator = std::vector<std::shared_ptr<table>>::iterator;
+
+    /**
+     * arrays can be iterated over.  Const version.
+     */
+    using const_iterator = std::vector<std::shared_ptr<table>>::const_iterator;
+
+    iterator begin()
+    {
+        return array_.begin();
+    }
+
+    const_iterator begin() const
+    {
+        return array_.begin();
+    }
+
+    iterator end()
+    {
+        return array_.end();
+    }
+
+    const_iterator end() const
+    {
+        return array_.end();
+    }
+
+    virtual bool is_table_array() const override
+    {
+        return true;
+    }
+
+    std::vector<std::shared_ptr<table>>& get()
+    {
+        return array_;
+    }
+
+    const std::vector<std::shared_ptr<table>>& get() const
+    {
+        return array_;
+    }
+
+    /**
+     * Add a table to the end of the array
+     */
+    void push_back(const std::shared_ptr<table>& val)
+    {
+        array_.push_back(val);
+    }
+
+    /**
+     * Insert a table into the array
+     */
+    iterator insert(iterator position, const std::shared_ptr<table>& value)
+    {
+        return array_.insert(position, value);
+    }
+
+    /**
+     * Erase an element from the array
+     */
+    iterator erase(iterator position)
+    {
+        return array_.erase(position);
+    }
+
+    /**
+     * Clear the array
+     */
+    void clear()
+    {
+        array_.clear();
+    }
+
+    /**
+     * Reserve space for n tables.
+     */
+    void reserve(size_type n)
+    {
+        array_.reserve(n);
+    }
+
+  private:
+#if defined(CPPTOML_NO_RTTI)
+    table_array() : base(base_type::TABLE_ARRAY)
+    {
+        // nothing
+    }
+#else
+    table_array()
+    {
+        // nothing
+    }
+#endif
+
+    table_array(const table_array& obj) = delete;
+    table_array& operator=(const table_array& rhs) = delete;
+
+    std::vector<std::shared_ptr<table>> array_;
+};
+
+inline std::shared_ptr<table_array> make_table_array()
+{
+    struct make_shared_enabler : public table_array
+    {
+        make_shared_enabler()
+        {
+            // nothing
+        }
+    };
+
+    return std::make_shared<make_shared_enabler>();
+}
+
+template <>
+inline std::shared_ptr<table_array> make_element<table_array>()
+{
+    return make_table_array();
+}
+
+// The below are overloads for fetching specific value types out of a value
+// where special casting behavior (like bounds checking) is desired
+
+template <class T>
+typename std::enable_if<!std::is_floating_point<T>::value
+                            && std::is_signed<T>::value,
+                        option<T>>::type
+get_impl(const std::shared_ptr<base>& elem)
+{
+    if (auto v = elem->as<int64_t>())
+    {
+        if (v->get() < (std::numeric_limits<T>::min)())
+            throw std::underflow_error{
+                "T cannot represent the value requested in get"};
+
+        if (v->get() > (std::numeric_limits<T>::max)())
+            throw std::overflow_error{
+                "T cannot represent the value requested in get"};
+
+        return {static_cast<T>(v->get())};
+    }
+    else
+    {
+        return {};
+    }
+}
+
+template <class T>
+typename std::enable_if<!std::is_same<T, bool>::value
+                            && std::is_unsigned<T>::value,
+                        option<T>>::type
+get_impl(const std::shared_ptr<base>& elem)
+{
+    if (auto v = elem->as<int64_t>())
+    {
+        if (v->get() < 0)
+            throw std::underflow_error{"T cannot store negative value in get"};
+
+        if (static_cast<uint64_t>(v->get()) > (std::numeric_limits<T>::max)())
+            throw std::overflow_error{
+                "T cannot represent the value requested in get"};
+
+        return {static_cast<T>(v->get())};
+    }
+    else
+    {
+        return {};
+    }
+}
+
+template <class T>
+typename std::enable_if<!std::is_integral<T>::value
+                            || std::is_same<T, bool>::value,
+                        option<T>>::type
+get_impl(const std::shared_ptr<base>& elem)
+{
+    if (auto v = elem->as<T>())
+    {
+        return {v->get()};
+    }
+    else
+    {
+        return {};
+    }
+}
+
+/**
+ * Represents a TOML keytable.
+ */
+class table : public base
+{
+  public:
+    friend class table_array;
+    friend std::shared_ptr<table> make_table();
+
+    std::shared_ptr<base> clone() const override;
+
+    /**
+     * tables can be iterated over.
+     */
+    using iterator = string_to_base_map::iterator;
+
+    /**
+     * tables can be iterated over. Const version.
+     */
+    using const_iterator = string_to_base_map::const_iterator;
+
+    iterator begin()
+    {
+        return map_.begin();
+    }
+
+    const_iterator begin() const
+    {
+        return map_.begin();
+    }
+
+    iterator end()
+    {
+        return map_.end();
+    }
+
+    const_iterator end() const
+    {
+        return map_.end();
+    }
+
+    bool is_table() const override
+    {
+        return true;
+    }
+
+    bool empty() const
+    {
+        return map_.empty();
+    }
+
+    /**
+     * Determines if this key table contains the given key.
+     */
+    bool contains(const std::string& key) const
+    {
+        return map_.find(key) != map_.end();
+    }
+
+    /**
+     * Determines if this key table contains the given key. Will
+     * resolve "qualified keys". Qualified keys are the full access
+     * path separated with dots like "grandparent.parent.child".
+     */
+    bool contains_qualified(const std::string& key) const
+    {
+        return resolve_qualified(key);
+    }
+
+    /**
+     * Obtains the base for a given key.
+     * @throw std::out_of_range if the key does not exist
+     */
+    std::shared_ptr<base> get(const std::string& key) const
+    {
+        return map_.at(key);
+    }
+
+    /**
+     * Obtains the base for a given key. Will resolve "qualified
+     * keys". Qualified keys are the full access path separated with
+     * dots like "grandparent.parent.child".
+     *
+     * @throw std::out_of_range if the key does not exist
+     */
+    std::shared_ptr<base> get_qualified(const std::string& key) const
+    {
+        std::shared_ptr<base> p;
+        resolve_qualified(key, &p);
+        return p;
+    }
+
+    /**
+     * Obtains a table for a given key, if possible.
+     */
+    std::shared_ptr<table> get_table(const std::string& key) const
+    {
+        if (contains(key) && get(key)->is_table())
+            return std::static_pointer_cast<table>(get(key));
+        return nullptr;
+    }
+
+    /**
+     * Obtains a table for a given key, if possible. Will resolve
+     * "qualified keys".
+     */
+    std::shared_ptr<table> get_table_qualified(const std::string& key) const
+    {
+        if (contains_qualified(key) && get_qualified(key)->is_table())
+            return std::static_pointer_cast<table>(get_qualified(key));
+        return nullptr;
+    }
+
+    /**
+     * Obtains an array for a given key.
+     */
+    std::shared_ptr<array> get_array(const std::string& key) const
+    {
+        if (!contains(key))
+            return nullptr;
+        return get(key)->as_array();
+    }
+
+    /**
+     * Obtains an array for a given key. Will resolve "qualified keys".
+     */
+    std::shared_ptr<array> get_array_qualified(const std::string& key) const
+    {
+        if (!contains_qualified(key))
+            return nullptr;
+        return get_qualified(key)->as_array();
+    }
+
+    /**
+     * Obtains a table_array for a given key, if possible.
+     */
+    std::shared_ptr<table_array> get_table_array(const std::string& key) const
+    {
+        if (!contains(key))
+            return nullptr;
+        return get(key)->as_table_array();
+    }
+
+    /**
+     * Obtains a table_array for a given key, if possible. Will resolve
+     * "qualified keys".
+     */
+    std::shared_ptr<table_array>
+    get_table_array_qualified(const std::string& key) const
+    {
+        if (!contains_qualified(key))
+            return nullptr;
+        return get_qualified(key)->as_table_array();
+    }
+
+    /**
+     * Helper function that attempts to get a value corresponding
+     * to the template parameter from a given key.
+     */
+    template <class T>
+    option<T> get_as(const std::string& key) const
+    {
+        try
+        {
+            return get_impl<T>(get(key));
+        }
+        catch (const std::out_of_range&)
+        {
+            return {};
+        }
+    }
+
+    /**
+     * Helper function that attempts to get a value corresponding
+     * to the template parameter from a given key. Will resolve "qualified
+     * keys".
+     */
+    template <class T>
+    option<T> get_qualified_as(const std::string& key) const
+    {
+        try
+        {
+            return get_impl<T>(get_qualified(key));
+        }
+        catch (const std::out_of_range&)
+        {
+            return {};
+        }
+    }
+
+    /**
+     * Helper function that attempts to get an array of values of a given
+     * type corresponding to the template parameter for a given key.
+     *
+     * If the key doesn't exist, doesn't exist as an array type, or one or
+     * more keys inside the array type are not of type T, an empty option
+     * is returned. Otherwise, an option containing a vector of the values
+     * is returned.
+     */
+    template <class T>
+    inline typename array_of_trait<T>::return_type
+    get_array_of(const std::string& key) const
+    {
+        if (auto v = get_array(key))
+        {
+            std::vector<T> result;
+            result.reserve(v->get().size());
+
+            for (const auto& b : v->get())
+            {
+                if (auto val = b->as<T>())
+                    result.push_back(val->get());
+                else
+                    return {};
+            }
+            return {std::move(result)};
+        }
+
+        return {};
+    }
+
+    /**
+     * Helper function that attempts to get an array of values of a given
+     * type corresponding to the template parameter for a given key. Will
+     * resolve "qualified keys".
+     *
+     * If the key doesn't exist, doesn't exist as an array type, or one or
+     * more keys inside the array type are not of type T, an empty option
+     * is returned. Otherwise, an option containing a vector of the values
+     * is returned.
+     */
+    template <class T>
+    inline typename array_of_trait<T>::return_type
+    get_qualified_array_of(const std::string& key) const
+    {
+        if (auto v = get_array_qualified(key))
+        {
+            std::vector<T> result;
+            result.reserve(v->get().size());
+
+            for (const auto& b : v->get())
+            {
+                if (auto val = b->as<T>())
+                    result.push_back(val->get());
+                else
+                    return {};
+            }
+            return {std::move(result)};
+        }
+
+        return {};
+    }
+
+    /**
+     * Adds an element to the keytable.
+     */
+    void insert(const std::string& key, const std::shared_ptr<base>& value)
+    {
+        map_[key] = value;
+    }
+
+    /**
+     * Convenience shorthand for adding a simple element to the
+     * keytable.
+     */
+    template <class T>
+    void insert(const std::string& key, T&& val,
+                typename value_traits<T>::type* = 0)
+    {
+        insert(key, make_value(std::forward<T>(val)));
+    }
+
+    /**
+     * Removes an element from the table.
+     */
+    void erase(const std::string& key)
+    {
+        map_.erase(key);
+    }
+
+  private:
+#if defined(CPPTOML_NO_RTTI)
+    table() : base(base_type::TABLE)
+    {
+        // nothing
+    }
+#else
+    table()
+    {
+        // nothing
+    }
+#endif
+
+    table(const table& obj) = delete;
+    table& operator=(const table& rhs) = delete;
+
+    std::vector<std::string> split(const std::string& value,
+                                   char separator) const
+    {
+        std::vector<std::string> result;
+        std::string::size_type p = 0;
+        std::string::size_type q;
+        while ((q = value.find(separator, p)) != std::string::npos)
+        {
+            result.emplace_back(value, p, q - p);
+            p = q + 1;
+        }
+        result.emplace_back(value, p);
+        return result;
+    }
+
+    // If output parameter p is specified, fill it with the pointer to the
+    // specified entry and throw std::out_of_range if it couldn't be found.
+    //
+    // Otherwise, just return true if the entry could be found or false
+    // otherwise and do not throw.
+    bool resolve_qualified(const std::string& key,
+                           std::shared_ptr<base>* p = nullptr) const
+    {
+        auto parts = split(key, '.');
+        auto last_key = parts.back();
+        parts.pop_back();
+
+        auto cur_table = this;
+        for (const auto& part : parts)
+        {
+            cur_table = cur_table->get_table(part).get();
+            if (!cur_table)
+            {
+                if (!p)
+                    return false;
+
+                throw std::out_of_range{key + " is not a valid key"};
+            }
+        }
+
+        if (!p)
+            return cur_table->map_.count(last_key) != 0;
+
+        *p = cur_table->map_.at(last_key);
+        return true;
+    }
+
+    string_to_base_map map_;
+};
+
+/**
+ * Helper function that attempts to get an array of arrays for a given
+ * key.
+ *
+ * If the key doesn't exist, doesn't exist as an array type, or one or
+ * more keys inside the array type are not of type T, an empty option
+ * is returned. Otherwise, an option containing a vector of the values
+ * is returned.
+ */
+template <>
+inline typename array_of_trait<array>::return_type
+table::get_array_of<array>(const std::string& key) const
+{
+    if (auto v = get_array(key))
+    {
+        std::vector<std::shared_ptr<array>> result;
+        result.reserve(v->get().size());
+
+        for (const auto& b : v->get())
+        {
+            if (auto val = b->as_array())
+                result.push_back(val);
+            else
+                return {};
+        }
+
+        return {std::move(result)};
+    }
+
+    return {};
+}
+
+/**
+ * Helper function that attempts to get an array of arrays for a given
+ * key. Will resolve "qualified keys".
+ *
+ * If the key doesn't exist, doesn't exist as an array type, or one or
+ * more keys inside the array type are not of type T, an empty option
+ * is returned. Otherwise, an option containing a vector of the values
+ * is returned.
+ */
+template <>
+inline typename array_of_trait<array>::return_type
+table::get_qualified_array_of<array>(const std::string& key) const
+{
+    if (auto v = get_array_qualified(key))
+    {
+        std::vector<std::shared_ptr<array>> result;
+        result.reserve(v->get().size());
+
+        for (const auto& b : v->get())
+        {
+            if (auto val = b->as_array())
+                result.push_back(val);
+            else
+                return {};
+        }
+
+        return {std::move(result)};
+    }
+
+    return {};
+}
+
+std::shared_ptr<table> make_table()
+{
+    struct make_shared_enabler : public table
+    {
+        make_shared_enabler()
+        {
+            // nothing
+        }
+    };
+
+    return std::make_shared<make_shared_enabler>();
+}
+
+template <>
+inline std::shared_ptr<table> make_element<table>()
+{
+    return make_table();
+}
+
+template <class T>
+std::shared_ptr<base> value<T>::clone() const
+{
+    return make_value(data_);
+}
+
+inline std::shared_ptr<base> array::clone() const
+{
+    auto result = make_array();
+    result->reserve(values_.size());
+    for (const auto& ptr : values_)
+        result->values_.push_back(ptr->clone());
+    return result;
+}
+
+inline std::shared_ptr<base> table_array::clone() const
+{
+    auto result = make_table_array();
+    result->reserve(array_.size());
+    for (const auto& ptr : array_)
+        result->array_.push_back(ptr->clone()->as_table());
+    return result;
+}
+
+inline std::shared_ptr<base> table::clone() const
+{
+    auto result = make_table();
+    for (const auto& pr : map_)
+        result->insert(pr.first, pr.second->clone());
+    return result;
+}
+
+/**
+ * Exception class for all TOML parsing errors.
+ */
+class parse_exception : public std::runtime_error
+{
+  public:
+    parse_exception(const std::string& err) : std::runtime_error{err}
+    {
+    }
+
+    parse_exception(const std::string& err, std::size_t line_number)
+        : std::runtime_error{err + " at line " + std::to_string(line_number)}
+    {
+    }
+};
+
+inline bool is_number(char c)
+{
+    return c >= '0' && c <= '9';
+}
+
+/**
+ * Helper object for consuming expected characters.
+ */
+template <class OnError>
+class consumer
+{
+  public:
+    consumer(std::string::iterator& it, const std::string::iterator& end,
+             OnError&& on_error)
+        : it_(it), end_(end), on_error_(std::forward<OnError>(on_error))
+    {
+        // nothing
+    }
+
+    void operator()(char c)
+    {
+        if (it_ == end_ || *it_ != c)
+            on_error_();
+        ++it_;
+    }
+
+    template <std::size_t N>
+    void operator()(const char (&str)[N])
+    {
+        std::for_each(std::begin(str), std::end(str) - 1,
+                      [&](char c) { (*this)(c); });
+    }
+
+    int eat_digits(int len)
+    {
+        int val = 0;
+        for (int i = 0; i < len; ++i)
+        {
+            if (!is_number(*it_) || it_ == end_)
+                on_error_();
+            val = 10 * val + (*it_++ - '0');
+        }
+        return val;
+    }
+
+    void error()
+    {
+        on_error_();
+    }
+
+  private:
+    std::string::iterator& it_;
+    const std::string::iterator& end_;
+    OnError on_error_;
+};
+
+template <class OnError>
+consumer<OnError> make_consumer(std::string::iterator& it,
+                                const std::string::iterator& end,
+                                OnError&& on_error)
+{
+    return consumer<OnError>(it, end, std::forward<OnError>(on_error));
+}
+
+// replacement for std::getline to handle incorrectly line-ended files
+// https://stackoverflow.com/questions/6089231/getting-std-ifstream-to-handle-lf-cr-and-crlf
+namespace detail
+{
+inline std::istream& getline(std::istream& input, std::string& line)
+{
+    line.clear();
+
+    std::istream::sentry sentry{input, true};
+    auto sb = input.rdbuf();
+
+    while (true)
+    {
+        auto c = sb->sbumpc();
+        if (c == '\r')
+        {
+            if (sb->sgetc() == '\n')
+                c = sb->sbumpc();
+        }
+
+        if (c == '\n')
+            return input;
+
+        if (c == std::istream::traits_type::eof())
+        {
+            if (line.empty())
+                input.setstate(std::ios::eofbit);
+            return input;
+        }
+
+        line.push_back(static_cast<char>(c));
+    }
+}
+}
+
+/**
+ * The parser class.
+ */
+class parser
+{
+  public:
+    /**
+     * Parsers are constructed from streams.
+     */
+    parser(std::istream& stream) : input_(stream)
+    {
+        // nothing
+    }
+
+    parser& operator=(const parser& parser) = delete;
+
+    /**
+     * Parses the stream this parser was created on until EOF.
+     * @throw parse_exception if there are errors in parsing
+     */
+    std::shared_ptr<table> parse()
+    {
+        std::shared_ptr<table> root = make_table();
+
+        table* curr_table = root.get();
+
+        while (detail::getline(input_, line_))
+        {
+            line_number_++;
+            auto it = line_.begin();
+            auto end = line_.end();
+            consume_whitespace(it, end);
+            if (it == end || *it == '#')
+                continue;
+            if (*it == '[')
+            {
+                curr_table = root.get();
+                parse_table(it, end, curr_table);
+            }
+            else
+            {
+                parse_key_value(it, end, curr_table);
+                consume_whitespace(it, end);
+                eol_or_comment(it, end);
+            }
+        }
+        return root;
+    }
+
+  private:
+#if defined _MSC_VER
+    __declspec(noreturn)
+#elif defined __GNUC__
+    __attribute__((noreturn))
+#endif
+        void throw_parse_exception(const std::string& err)
+    {
+        throw parse_exception{err, line_number_};
+    }
+
+    void parse_table(std::string::iterator& it,
+                     const std::string::iterator& end, table*& curr_table)
+    {
+        // remove the beginning keytable marker
+        ++it;
+        if (it == end)
+            throw_parse_exception("Unexpected end of table");
+        if (*it == '[')
+            parse_table_array(it, end, curr_table);
+        else
+            parse_single_table(it, end, curr_table);
+    }
+
+    void parse_single_table(std::string::iterator& it,
+                            const std::string::iterator& end,
+                            table*& curr_table)
+    {
+        if (it == end || *it == ']')
+            throw_parse_exception("Table name cannot be empty");
+
+        std::string full_table_name;
+        bool inserted = false;
+        while (it != end && *it != ']')
+        {
+            auto part = parse_key(it, end,
+                                  [](char c) { return c == '.' || c == ']'; });
+
+            if (part.empty())
+                throw_parse_exception("Empty component of table name");
+
+            if (!full_table_name.empty())
+                full_table_name += ".";
+            full_table_name += part;
+
+            if (curr_table->contains(part))
+            {
+                auto b = curr_table->get(part);
+                if (b->is_table())
+                    curr_table = static_cast<table*>(b.get());
+                else if (b->is_table_array())
+                    curr_table = std::static_pointer_cast<table_array>(b)
+                                     ->get()
+                                     .back()
+                                     .get();
+                else
+                    throw_parse_exception("Key " + full_table_name
+                                          + "already exists as a value");
+            }
+            else
+            {
+                inserted = true;
+                curr_table->insert(part, make_table());
+                curr_table = static_cast<table*>(curr_table->get(part).get());
+            }
+            consume_whitespace(it, end);
+            if (it != end && *it == '.')
+                ++it;
+            consume_whitespace(it, end);
+        }
+
+        if (it == end)
+            throw_parse_exception(
+                "Unterminated table declaration; did you forget a ']'?");
+
+        // table already existed
+        if (!inserted)
+        {
+            auto is_value
+                = [](const std::pair<const std::string&,
+                                     const std::shared_ptr<base>&>& p) {
+                      return p.second->is_value();
+                  };
+
+            // if there are any values, we can't add values to this table
+            // since it has already been defined. If there aren't any
+            // values, then it was implicitly created by something like
+            // [a.b]
+            if (curr_table->empty() || std::any_of(curr_table->begin(),
+                                                   curr_table->end(), is_value))
+            {
+                throw_parse_exception("Redefinition of table "
+                                      + full_table_name);
+            }
+        }
+
+        ++it;
+        consume_whitespace(it, end);
+        eol_or_comment(it, end);
+    }
+
+    void parse_table_array(std::string::iterator& it,
+                           const std::string::iterator& end, table*& curr_table)
+    {
+        ++it;
+        if (it == end || *it == ']')
+            throw_parse_exception("Table array name cannot be empty");
+
+        std::string full_ta_name;
+        while (it != end && *it != ']')
+        {
+            auto part = parse_key(it, end,
+                                  [](char c) { return c == '.' || c == ']'; });
+
+            if (part.empty())
+                throw_parse_exception("Empty component of table array name");
+
+            if (!full_ta_name.empty())
+                full_ta_name += ".";
+            full_ta_name += part;
+
+            consume_whitespace(it, end);
+            if (it != end && *it == '.')
+                ++it;
+            consume_whitespace(it, end);
+
+            if (curr_table->contains(part))
+            {
+                auto b = curr_table->get(part);
+
+                // if this is the end of the table array name, add an
+                // element to the table array that we just looked up
+                if (it != end && *it == ']')
+                {
+                    if (!b->is_table_array())
+                        throw_parse_exception("Key " + full_ta_name
+                                              + " is not a table array");
+                    auto v = b->as_table_array();
+                    v->get().push_back(make_table());
+                    curr_table = v->get().back().get();
+                }
+                // otherwise, just keep traversing down the key name
+                else
+                {
+                    if (b->is_table())
+                        curr_table = static_cast<table*>(b.get());
+                    else if (b->is_table_array())
+                        curr_table = std::static_pointer_cast<table_array>(b)
+                                         ->get()
+                                         .back()
+                                         .get();
+                    else
+                        throw_parse_exception("Key " + full_ta_name
+                                              + " already exists as a value");
+                }
+            }
+            else
+            {
+                // if this is the end of the table array name, add a new
+                // table array and a new table inside that array for us to
+                // add keys to next
+                if (it != end && *it == ']')
+                {
+                    curr_table->insert(part, make_table_array());
+                    auto arr = std::static_pointer_cast<table_array>(
+                        curr_table->get(part));
+                    arr->get().push_back(make_table());
+                    curr_table = arr->get().back().get();
+                }
+                // otherwise, create the implicitly defined table and move
+                // down to it
+                else
+                {
+                    curr_table->insert(part, make_table());
+                    curr_table
+                        = static_cast<table*>(curr_table->get(part).get());
+                }
+            }
+        }
+
+        // consume the last "]]"
+        if (it == end)
+            throw_parse_exception("Unterminated table array name");
+        ++it;
+        if (it == end)
+            throw_parse_exception("Unterminated table array name");
+        ++it;
+
+        consume_whitespace(it, end);
+        eol_or_comment(it, end);
+    }
+
+    void parse_key_value(std::string::iterator& it, std::string::iterator& end,
+                         table* curr_table)
+    {
+        auto key = parse_key(it, end, [](char c) { return c == '='; });
+        if (curr_table->contains(key))
+            throw_parse_exception("Key " + key + " already present");
+        if (it == end || *it != '=')
+            throw_parse_exception("Value must follow after a '='");
+        ++it;
+        consume_whitespace(it, end);
+        curr_table->insert(key, parse_value(it, end));
+        consume_whitespace(it, end);
+    }
+
+    template <class Function>
+    std::string parse_key(std::string::iterator& it,
+                          const std::string::iterator& end, Function&& fun)
+    {
+        consume_whitespace(it, end);
+        if (*it == '"')
+        {
+            return parse_quoted_key(it, end);
+        }
+        else
+        {
+            auto bke = std::find_if(it, end, std::forward<Function>(fun));
+            return parse_bare_key(it, bke);
+        }
+    }
+
+    std::string parse_bare_key(std::string::iterator& it,
+                               const std::string::iterator& end)
+    {
+        if (it == end)
+        {
+            throw_parse_exception("Bare key missing name");
+        }
+
+        auto key_end = end;
+        --key_end;
+        consume_backwards_whitespace(key_end, it);
+        ++key_end;
+        std::string key{it, key_end};
+
+        if (std::find(it, key_end, '#') != key_end)
+        {
+            throw_parse_exception("Bare key " + key + " cannot contain #");
+        }
+
+        if (std::find_if(it, key_end,
+                         [](char c) { return c == ' ' || c == '\t'; })
+            != key_end)
+        {
+            throw_parse_exception("Bare key " + key
+                                  + " cannot contain whitespace");
+        }
+
+        if (std::find_if(it, key_end,
+                         [](char c) { return c == '[' || c == ']'; })
+            != key_end)
+        {
+            throw_parse_exception("Bare key " + key
+                                  + " cannot contain '[' or ']'");
+        }
+
+        it = end;
+        return key;
+    }
+
+    std::string parse_quoted_key(std::string::iterator& it,
+                                 const std::string::iterator& end)
+    {
+        return string_literal(it, end, '"');
+    }
+
+    enum class parse_type
+    {
+        STRING = 1,
+        LOCAL_TIME,
+        LOCAL_DATE,
+        LOCAL_DATETIME,
+        OFFSET_DATETIME,
+        INT,
+        FLOAT,
+        BOOL,
+        ARRAY,
+        INLINE_TABLE
+    };
+
+    std::shared_ptr<base> parse_value(std::string::iterator& it,
+                                      std::string::iterator& end)
+    {
+        parse_type type = determine_value_type(it, end);
+        switch (type)
+        {
+            case parse_type::STRING:
+                return parse_string(it, end);
+            case parse_type::LOCAL_TIME:
+                return parse_time(it, end);
+            case parse_type::LOCAL_DATE:
+            case parse_type::LOCAL_DATETIME:
+            case parse_type::OFFSET_DATETIME:
+                return parse_date(it, end);
+            case parse_type::INT:
+            case parse_type::FLOAT:
+                return parse_number(it, end);
+            case parse_type::BOOL:
+                return parse_bool(it, end);
+            case parse_type::ARRAY:
+                return parse_array(it, end);
+            case parse_type::INLINE_TABLE:
+                return parse_inline_table(it, end);
+            default:
+                throw_parse_exception("Failed to parse value");
+        }
+    }
+
+    parse_type determine_value_type(const std::string::iterator& it,
+                                    const std::string::iterator& end)
+    {
+        if(it == end)
+        {
+            throw_parse_exception("Failed to parse value type");
+        }
+        if (*it == '"' || *it == '\'')
+        {
+            return parse_type::STRING;
+        }
+        else if (is_time(it, end))
+        {
+            return parse_type::LOCAL_TIME;
+        }
+        else if (auto dtype = date_type(it, end))
+        {
+            return *dtype;
+        }
+        else if (is_number(*it) || *it == '-' || *it == '+')
+        {
+            return determine_number_type(it, end);
+        }
+        else if (*it == 't' || *it == 'f')
+        {
+            return parse_type::BOOL;
+        }
+        else if (*it == '[')
+        {
+            return parse_type::ARRAY;
+        }
+        else if (*it == '{')
+        {
+            return parse_type::INLINE_TABLE;
+        }
+        throw_parse_exception("Failed to parse value type");
+    }
+
+    parse_type determine_number_type(const std::string::iterator& it,
+                                     const std::string::iterator& end)
+    {
+        // determine if we are an integer or a float
+        auto check_it = it;
+        if (*check_it == '-' || *check_it == '+')
+            ++check_it;
+        while (check_it != end && is_number(*check_it))
+            ++check_it;
+        if (check_it != end && *check_it == '.')
+        {
+            ++check_it;
+            while (check_it != end && is_number(*check_it))
+                ++check_it;
+            return parse_type::FLOAT;
+        }
+        else
+        {
+            return parse_type::INT;
+        }
+    }
+
+    std::shared_ptr<value<std::string>> parse_string(std::string::iterator& it,
+                                                     std::string::iterator& end)
+    {
+        auto delim = *it;
+        assert(delim == '"' || delim == '\'');
+
+        // end is non-const here because we have to be able to potentially
+        // parse multiple lines in a string, not just one
+        auto check_it = it;
+        ++check_it;
+        if (check_it != end && *check_it == delim)
+        {
+            ++check_it;
+            if (check_it != end && *check_it == delim)
+            {
+                it = ++check_it;
+                return parse_multiline_string(it, end, delim);
+            }
+        }
+        return make_value<std::string>(string_literal(it, end, delim));
+    }
+
+    std::shared_ptr<value<std::string>>
+    parse_multiline_string(std::string::iterator& it,
+                           std::string::iterator& end, char delim)
+    {
+        std::stringstream ss;
+
+        auto is_ws = [](char c) { return c == ' ' || c == '\t'; };
+
+        bool consuming = false;
+        std::shared_ptr<value<std::string>> ret;
+
+        auto handle_line
+            = [&](std::string::iterator& local_it,
+                  std::string::iterator& local_end) {
+                  if (consuming)
+                  {
+                      local_it = std::find_if_not(local_it, local_end, is_ws);
+
+                      // whole line is whitespace
+                      if (local_it == local_end)
+                          return;
+                  }
+
+                  consuming = false;
+
+                  while (local_it != local_end)
+                  {
+                      // handle escaped characters
+                      if (delim == '"' && *local_it == '\\')
+                      {
+                          auto check = local_it;
+                          // check if this is an actual escape sequence or a
+                          // whitespace escaping backslash
+                          ++check;
+                          consume_whitespace(check, local_end);
+                          if (check == local_end)
+                          {
+                              consuming = true;
+                              break;
+                          }
+
+                          ss << parse_escape_code(local_it, local_end);
+                          continue;
+                      }
+
+                      // if we can end the string
+                      if (std::distance(local_it, local_end) >= 3)
+                      {
+                          auto check = local_it;
+                          // check for """
+                          if (*check++ == delim && *check++ == delim
+                              && *check++ == delim)
+                          {
+                              local_it = check;
+                              ret = make_value<std::string>(ss.str());
+                              break;
+                          }
+                      }
+
+                      ss << *local_it++;
+                  }
+              };
+
+        // handle the remainder of the current line
+        handle_line(it, end);
+        if (ret)
+            return ret;
+
+        // start eating lines
+        while (detail::getline(input_, line_))
+        {
+            ++line_number_;
+
+            it = line_.begin();
+            end = line_.end();
+
+            handle_line(it, end);
+
+            if (ret)
+                return ret;
+
+            if (!consuming)
+                ss << std::endl;
+        }
+
+        throw_parse_exception("Unterminated multi-line basic string");
+    }
+
+    std::string string_literal(std::string::iterator& it,
+                               const std::string::iterator& end, char delim)
+    {
+        ++it;
+        std::string val;
+        while (it != end)
+        {
+            // handle escaped characters
+            if (delim == '"' && *it == '\\')
+            {
+                val += parse_escape_code(it, end);
+            }
+            else if (*it == delim)
+            {
+                ++it;
+                consume_whitespace(it, end);
+                return val;
+            }
+            else
+            {
+                val += *it++;
+            }
+        }
+        throw_parse_exception("Unterminated string literal");
+    }
+
+    std::string parse_escape_code(std::string::iterator& it,
+                                  const std::string::iterator& end)
+    {
+        ++it;
+        if (it == end)
+            throw_parse_exception("Invalid escape sequence");
+        char value;
+        if (*it == 'b')
+        {
+            value = '\b';
+        }
+        else if (*it == 't')
+        {
+            value = '\t';
+        }
+        else if (*it == 'n')
+        {
+            value = '\n';
+        }
+        else if (*it == 'f')
+        {
+            value = '\f';
+        }
+        else if (*it == 'r')
+        {
+            value = '\r';
+        }
+        else if (*it == '"')
+        {
+            value = '"';
+        }
+        else if (*it == '\\')
+        {
+            value = '\\';
+        }
+        else if (*it == 'u' || *it == 'U')
+        {
+            return parse_unicode(it, end);
+        }
+        else
+        {
+            throw_parse_exception("Invalid escape sequence");
+        }
+        ++it;
+        return std::string(1, value);
+    }
+
+    std::string parse_unicode(std::string::iterator& it,
+                              const std::string::iterator& end)
+    {
+        bool large = *it++ == 'U';
+        auto codepoint = parse_hex(it, end, large ? 0x10000000 : 0x1000);
+
+        if ((codepoint > 0xd7ff && codepoint < 0xe000) || codepoint > 0x10ffff)
+        {
+            throw_parse_exception(
+                "Unicode escape sequence is not a Unicode scalar value");
+        }
+
+        std::string result;
+        // See Table 3-6 of the Unicode standard
+        if (codepoint <= 0x7f)
+        {
+            // 1-byte codepoints: 00000000 0xxxxxxx
+            // repr: 0xxxxxxx
+            result += static_cast<char>(codepoint & 0x7f);
+        }
+        else if (codepoint <= 0x7ff)
+        {
+            // 2-byte codepoints: 00000yyy yyxxxxxx
+            // repr: 110yyyyy 10xxxxxx
+            //
+            // 0x1f = 00011111
+            // 0xc0 = 11000000
+            //
+            result += static_cast<char>(0xc0 | ((codepoint >> 6) & 0x1f));
+            //
+            // 0x80 = 10000000
+            // 0x3f = 00111111
+            //
+            result += static_cast<char>(0x80 | (codepoint & 0x3f));
+        }
+        else if (codepoint <= 0xffff)
+        {
+            // 3-byte codepoints: zzzzyyyy yyxxxxxx
+            // repr: 1110zzzz 10yyyyyy 10xxxxxx
+            //
+            // 0xe0 = 11100000
+            // 0x0f = 00001111
+            //
+            result += static_cast<char>(0xe0 | ((codepoint >> 12) & 0x0f));
+            result += static_cast<char>(0x80 | ((codepoint >> 6) & 0x1f));
+            result += static_cast<char>(0x80 | (codepoint & 0x3f));
+        }
+        else
+        {
+            // 4-byte codepoints: 000uuuuu zzzzyyyy yyxxxxxx
+            // repr: 11110uuu 10uuzzzz 10yyyyyy 10xxxxxx
+            //
+            // 0xf0 = 11110000
+            // 0x07 = 00000111
+            //
+            result += static_cast<char>(0xf0 | ((codepoint >> 18) & 0x07));
+            result += static_cast<char>(0x80 | ((codepoint >> 12) & 0x3f));
+            result += static_cast<char>(0x80 | ((codepoint >> 6) & 0x3f));
+            result += static_cast<char>(0x80 | (codepoint & 0x3f));
+        }
+        return result;
+    }
+
+    uint32_t parse_hex(std::string::iterator& it,
+                       const std::string::iterator& end, uint32_t place)
+    {
+        uint32_t value = 0;
+        while (place > 0)
+        {
+            if (it == end)
+                throw_parse_exception("Unexpected end of unicode sequence");
+
+            if (!is_hex(*it))
+                throw_parse_exception("Invalid unicode escape sequence");
+
+            value += place * hex_to_digit(*it++);
+            place /= 16;
+        }
+        return value;
+    }
+
+    bool is_hex(char c)
+    {
+        return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
+    }
+
+    uint32_t hex_to_digit(char c)
+    {
+        if (is_number(c))
+            return static_cast<uint32_t>(c - '0');
+        return 10 + static_cast<uint32_t>(
+                        c - ((c >= 'a' && c <= 'f') ? 'a' : 'A'));
+    }
+
+    std::shared_ptr<base> parse_number(std::string::iterator& it,
+                                       const std::string::iterator& end)
+    {
+        auto check_it = it;
+        auto check_end = find_end_of_number(it, end);
+
+        auto eat_sign = [&]() {
+            if (check_it != end && (*check_it == '-' || *check_it == '+'))
+                ++check_it;
+        };
+
+        eat_sign();
+
+        auto eat_numbers = [&]() {
+            auto beg = check_it;
+            while (check_it != end && is_number(*check_it))
+            {
+                ++check_it;
+                if (check_it != end && *check_it == '_')
+                {
+                    ++check_it;
+                    if (check_it == end || !is_number(*check_it))
+                        throw_parse_exception("Malformed number");
+                }
+            }
+
+            if (check_it == beg)
+                throw_parse_exception("Malformed number");
+        };
+
+        auto check_no_leading_zero = [&]() {
+            if (check_it != end && *check_it == '0' && check_it + 1 != check_end
+                && check_it[1] != '.')
+            {
+                throw_parse_exception("Numbers may not have leading zeros");
+            }
+        };
+
+        check_no_leading_zero();
+        eat_numbers();
+
+        if (check_it != end
+            && (*check_it == '.' || *check_it == 'e' || *check_it == 'E'))
+        {
+            bool is_exp = *check_it == 'e' || *check_it == 'E';
+
+            ++check_it;
+            if (check_it == end)
+                throw_parse_exception("Floats must have trailing digits");
+
+            auto eat_exp = [&]() {
+                eat_sign();
+                check_no_leading_zero();
+                eat_numbers();
+            };
+
+            if (is_exp)
+                eat_exp();
+            else
+                eat_numbers();
+
+            if (!is_exp && check_it != end
+                && (*check_it == 'e' || *check_it == 'E'))
+            {
+                ++check_it;
+                eat_exp();
+            }
+
+            return parse_float(it, check_it);
+        }
+        else
+        {
+            return parse_int(it, check_it);
+        }
+    }
+
+    std::shared_ptr<value<int64_t>> parse_int(std::string::iterator& it,
+                                              const std::string::iterator& end)
+    {
+        std::string v{it, end};
+        v.erase(std::remove(v.begin(), v.end(), '_'), v.end());
+        it = end;
+        try
+        {
+            return make_value<int64_t>(std::stoll(v));
+        }
+        catch (const std::invalid_argument& ex)
+        {
+            throw_parse_exception("Malformed number (invalid argument: "
+                                  + std::string{ex.what()} + ")");
+        }
+        catch (const std::out_of_range& ex)
+        {
+            throw_parse_exception("Malformed number (out of range: "
+                                  + std::string{ex.what()} + ")");
+        }
+    }
+
+    std::shared_ptr<value<double>> parse_float(std::string::iterator& it,
+                                               const std::string::iterator& end)
+    {
+        std::string v{it, end};
+        v.erase(std::remove(v.begin(), v.end(), '_'), v.end());
+        it = end;
+        char decimal_point = std::localeconv()->decimal_point[0];
+        std::replace(v.begin(), v.end(), '.', decimal_point);
+        try
+        {
+            return make_value<double>(std::stod(v));
+        }
+        catch (const std::invalid_argument& ex)
+        {
+            throw_parse_exception("Malformed number (invalid argument: "
+                                  + std::string{ex.what()} + ")");
+        }
+        catch (const std::out_of_range& ex)
+        {
+            throw_parse_exception("Malformed number (out of range: "
+                                  + std::string{ex.what()} + ")");
+        }
+    }
+
+    std::shared_ptr<value<bool>> parse_bool(std::string::iterator& it,
+                                            const std::string::iterator& end)
+    {
+        auto eat = make_consumer(it, end, [this]() {
+            throw_parse_exception("Attempted to parse invalid boolean value");
+        });
+
+        if (*it == 't')
+        {
+            eat("true");
+            return make_value<bool>(true);
+        }
+        else if (*it == 'f')
+        {
+            eat("false");
+            return make_value<bool>(false);
+        }
+
+        eat.error();
+        return nullptr;
+    }
+
+    std::string::iterator find_end_of_number(std::string::iterator it,
+                                             std::string::iterator end)
+    {
+        return std::find_if(it, end, [](char c) {
+            return !is_number(c) && c != '_' && c != '.' && c != 'e' && c != 'E'
+                   && c != '-' && c != '+';
+        });
+    }
+
+    std::string::iterator find_end_of_date(std::string::iterator it,
+                                           std::string::iterator end)
+    {
+        return std::find_if(it, end, [](char c) {
+            return !is_number(c) && c != 'T' && c != 'Z' && c != ':' && c != '-'
+                   && c != '+' && c != '.';
+        });
+    }
+
+    std::string::iterator find_end_of_time(std::string::iterator it,
+                                           std::string::iterator end)
+    {
+        return std::find_if(it, end, [](char c) {
+            return !is_number(c) && c != ':' && c != '.';
+        });
+    }
+
+    local_time read_time(std::string::iterator& it,
+                         const std::string::iterator& end)
+    {
+        auto time_end = find_end_of_time(it, end);
+
+        auto eat = make_consumer(
+            it, time_end, [&]() { throw_parse_exception("Malformed time"); });
+
+        local_time ltime;
+
+        ltime.hour = eat.eat_digits(2);
+        eat(':');
+        ltime.minute = eat.eat_digits(2);
+        eat(':');
+        ltime.second = eat.eat_digits(2);
+
+        int power = 100000;
+        if (it != time_end && *it == '.')
+        {
+            ++it;
+            while (it != time_end && is_number(*it))
+            {
+                ltime.microsecond += power * (*it++ - '0');
+                power /= 10;
+            }
+        }
+
+        if (it != time_end)
+            throw_parse_exception("Malformed time");
+
+        return ltime;
+    }
+
+    std::shared_ptr<value<local_time>>
+    parse_time(std::string::iterator& it, const std::string::iterator& end)
+    {
+        return make_value(read_time(it, end));
+    }
+
+    std::shared_ptr<base> parse_date(std::string::iterator& it,
+                                     const std::string::iterator& end)
+    {
+        auto date_end = find_end_of_date(it, end);
+
+        auto eat = make_consumer(
+            it, date_end, [&]() { throw_parse_exception("Malformed date"); });
+
+        local_date ldate;
+        ldate.year = eat.eat_digits(4);
+        eat('-');
+        ldate.month = eat.eat_digits(2);
+        eat('-');
+        ldate.day = eat.eat_digits(2);
+
+        if (it == date_end)
+            return make_value(ldate);
+
+        eat('T');
+
+        local_datetime ldt;
+        static_cast<local_date&>(ldt) = ldate;
+        static_cast<local_time&>(ldt) = read_time(it, date_end);
+
+        if (it == date_end)
+            return make_value(ldt);
+
+        offset_datetime dt;
+        static_cast<local_datetime&>(dt) = ldt;
+
+        int hoff = 0;
+        int moff = 0;
+        if (*it == '+' || *it == '-')
+        {
+            auto plus = *it == '+';
+            ++it;
+
+            hoff = eat.eat_digits(2);
+            dt.hour_offset = (plus) ? hoff : -hoff;
+            eat(':');
+            moff = eat.eat_digits(2);
+            dt.minute_offset = (plus) ? moff : -moff;
+        }
+        else if (*it == 'Z')
+        {
+            ++it;
+        }
+
+        if (it != date_end)
+            throw_parse_exception("Malformed date");
+
+        return make_value(dt);
+    }
+
+    std::shared_ptr<base> parse_array(std::string::iterator& it,
+                                      std::string::iterator& end)
+    {
+        // this gets ugly because of the "homogeneity" restriction:
+        // arrays can either be of only one type, or contain arrays
+        // (each of those arrays could be of different types, though)
+        //
+        // because of the latter portion, we don't really have a choice
+        // but to represent them as arrays of base values...
+        ++it;
+
+        // ugh---have to read the first value to determine array type...
+        skip_whitespace_and_comments(it, end);
+
+        // edge case---empty array
+        if (*it == ']')
+        {
+            ++it;
+            return make_array();
+        }
+
+        auto val_end = std::find_if(
+            it, end, [](char c) { return c == ',' || c == ']' || c == '#'; });
+        parse_type type = determine_value_type(it, val_end);
+        switch (type)
+        {
+            case parse_type::STRING:
+                return parse_value_array<std::string>(it, end);
+            case parse_type::LOCAL_TIME:
+                return parse_value_array<local_time>(it, end);
+            case parse_type::LOCAL_DATE:
+                return parse_value_array<local_date>(it, end);
+            case parse_type::LOCAL_DATETIME:
+                return parse_value_array<local_datetime>(it, end);
+            case parse_type::OFFSET_DATETIME:
+                return parse_value_array<offset_datetime>(it, end);
+            case parse_type::INT:
+                return parse_value_array<int64_t>(it, end);
+            case parse_type::FLOAT:
+                return parse_value_array<double>(it, end);
+            case parse_type::BOOL:
+                return parse_value_array<bool>(it, end);
+            case parse_type::ARRAY:
+                return parse_object_array<array>(&parser::parse_array, '[', it,
+                                                 end);
+            case parse_type::INLINE_TABLE:
+                return parse_object_array<table_array>(
+                    &parser::parse_inline_table, '{', it, end);
+            default:
+                throw_parse_exception("Unable to parse array");
+        }
+    }
+
+    template <class Value>
+    std::shared_ptr<array> parse_value_array(std::string::iterator& it,
+                                             std::string::iterator& end)
+    {
+        auto arr = make_array();
+        while (it != end && *it != ']')
+        {
+            auto value = parse_value(it, end);
+            if (auto v = value->as<Value>())
+                arr->get().push_back(value);
+            else
+                throw_parse_exception("Arrays must be homogeneous");
+            skip_whitespace_and_comments(it, end);
+            if (*it != ',')
+                break;
+            ++it;
+            skip_whitespace_and_comments(it, end);
+        }
+        if (it != end)
+            ++it;
+        return arr;
+    }
+
+    template <class Object, class Function>
+    std::shared_ptr<Object> parse_object_array(Function&& fun, char delim,
+                                               std::string::iterator& it,
+                                               std::string::iterator& end)
+    {
+        auto arr = make_element<Object>();
+
+        while (it != end && *it != ']')
+        {
+            if (*it != delim)
+                throw_parse_exception("Unexpected character in array");
+
+            arr->get().push_back(((*this).*fun)(it, end));
+            skip_whitespace_and_comments(it, end);
+
+            if (*it != ',')
+                break;
+
+            ++it;
+            skip_whitespace_and_comments(it, end);
+        }
+
+        if (it == end || *it != ']')
+            throw_parse_exception("Unterminated array");
+
+        ++it;
+        return arr;
+    }
+
+    std::shared_ptr<table> parse_inline_table(std::string::iterator& it,
+                                              std::string::iterator& end)
+    {
+        auto tbl = make_table();
+        do
+        {
+            ++it;
+            if (it == end)
+                throw_parse_exception("Unterminated inline table");
+
+            consume_whitespace(it, end);
+            parse_key_value(it, end, tbl.get());
+            consume_whitespace(it, end);
+        } while (*it == ',');
+
+        if (it == end || *it != '}')
+            throw_parse_exception("Unterminated inline table");
+
+        ++it;
+        consume_whitespace(it, end);
+
+        return tbl;
+    }
+
+    void skip_whitespace_and_comments(std::string::iterator& start,
+                                      std::string::iterator& end)
+    {
+        consume_whitespace(start, end);
+        while (start == end || *start == '#')
+        {
+            if (!detail::getline(input_, line_))
+                throw_parse_exception("Unclosed array");
+            line_number_++;
+            start = line_.begin();
+            end = line_.end();
+            consume_whitespace(start, end);
+        }
+    }
+
+    void consume_whitespace(std::string::iterator& it,
+                            const std::string::iterator& end)
+    {
+        while (it != end && (*it == ' ' || *it == '\t'))
+            ++it;
+    }
+
+    void consume_backwards_whitespace(std::string::iterator& back,
+                                      const std::string::iterator& front)
+    {
+        while (back != front && (*back == ' ' || *back == '\t'))
+            --back;
+    }
+
+    void eol_or_comment(const std::string::iterator& it,
+                        const std::string::iterator& end)
+    {
+        if (it != end && *it != '#')
+            throw_parse_exception("Unidentified trailing character '"
+                                  + std::string{*it}
+                                  + "'---did you forget a '#'?");
+    }
+
+    bool is_time(const std::string::iterator& it,
+                 const std::string::iterator& end)
+    {
+        auto time_end = find_end_of_time(it, end);
+        auto len = std::distance(it, time_end);
+
+        if (len < 8)
+            return false;
+
+        if (it[2] != ':' || it[5] != ':')
+            return false;
+
+        if (len > 8)
+            return it[8] == '.' && len > 9;
+
+        return true;
+    }
+
+    option<parse_type> date_type(const std::string::iterator& it,
+                                 const std::string::iterator& end)
+    {
+        auto date_end = find_end_of_date(it, end);
+        auto len = std::distance(it, date_end);
+
+        if (len < 10)
+            return {};
+
+        if (it[4] != '-' || it[7] != '-')
+            return {};
+
+        if (len >= 19 && it[10] == 'T' && is_time(it + 11, date_end))
+        {
+            // datetime type
+            auto time_end = find_end_of_time(it + 11, date_end);
+            if (time_end == date_end)
+                return {parse_type::LOCAL_DATETIME};
+            else
+                return {parse_type::OFFSET_DATETIME};
+        }
+        else if (len == 10)
+        {
+            // just a regular date
+            return {parse_type::LOCAL_DATE};
+        }
+
+        return {};
+    }
+
+    std::istream& input_;
+    std::string line_;
+    std::size_t line_number_ = 0;
+};
+
+/**
+ * Utility function to parse a file as a TOML file. Returns the root table.
+ * Throws a parse_exception if the file cannot be opened.
+ */
+inline std::shared_ptr<table> parse_file(const std::string& filename)
+{
+#if defined(BOOST_NOWIDE_FSTREAM_INCLUDED_HPP)
+    boost::nowide::ifstream file{filename.c_str()};
+#elif defined(NOWIDE_FSTREAM_INCLUDED_HPP)
+    nowide::ifstream file{filename.c_str()};
+#else
+    std::ifstream file{filename};
+#endif
+    if (!file.is_open())
+        throw parse_exception{filename + " could not be opened for parsing"};
+    parser p{file};
+    return p.parse();
+}
+
+template <class... Ts>
+struct value_accept;
+
+template <>
+struct value_accept<>
+{
+    template <class Visitor, class... Args>
+    static void accept(const base&, Visitor&&, Args&&...)
+    {
+        // nothing
+    }
+};
+
+template <class T, class... Ts>
+struct value_accept<T, Ts...>
+{
+    template <class Visitor, class... Args>
+    static void accept(const base& b, Visitor&& visitor, Args&&... args)
+    {
+        if (auto v = b.as<T>())
+        {
+            visitor.visit(*v, std::forward<Args>(args)...);
+        }
+        else
+        {
+            value_accept<Ts...>::accept(b, std::forward<Visitor>(visitor),
+                                        std::forward<Args>(args)...);
+        }
+    }
+};
+
+/**
+ * base implementation of accept() that calls visitor.visit() on the concrete
+ * class.
+ */
+template <class Visitor, class... Args>
+void base::accept(Visitor&& visitor, Args&&... args) const
+{
+    if (is_value())
+    {
+        using value_acceptor
+            = value_accept<std::string, int64_t, double, bool, local_date,
+                           local_time, local_datetime, offset_datetime>;
+        value_acceptor::accept(*this, std::forward<Visitor>(visitor),
+                               std::forward<Args>(args)...);
+    }
+    else if (is_table())
+    {
+        visitor.visit(static_cast<const table&>(*this),
+                      std::forward<Args>(args)...);
+    }
+    else if (is_array())
+    {
+        visitor.visit(static_cast<const array&>(*this),
+                      std::forward<Args>(args)...);
+    }
+    else if (is_table_array())
+    {
+        visitor.visit(static_cast<const table_array&>(*this),
+                      std::forward<Args>(args)...);
+    }
+}
+
+/**
+ * Writer that can be passed to accept() functions of cpptoml objects and
+ * will output valid TOML to a stream.
+ */
+class toml_writer
+{
+  public:
+    /**
+     * Construct a toml_writer that will write to the given stream
+     */
+    toml_writer(std::ostream& s, const std::string& indent_space = "\t")
+        : stream_(s), indent_(indent_space), has_naked_endline_(false)
+    {
+        // nothing
+    }
+
+  public:
+    /**
+     * Output a base value of the TOML tree.
+     */
+    template <class T>
+    void visit(const value<T>& v, bool = false)
+    {
+        write(v);
+    }
+
+    /**
+     * Output a table element of the TOML tree
+     */
+    void visit(const table& t, bool in_array = false)
+    {
+        write_table_header(in_array);
+        std::vector<std::string> values;
+        std::vector<std::string> tables;
+
+        for (const auto& i : t)
+        {
+            if (i.second->is_table() || i.second->is_table_array())
+            {
+                tables.push_back(i.first);
+            }
+            else
+            {
+                values.push_back(i.first);
+            }
+        }
+
+        for (unsigned int i = 0; i < values.size(); ++i)
+        {
+            path_.push_back(values[i]);
+
+            if (i > 0)
+                endline();
+
+            write_table_item_header(*t.get(values[i]));
+            t.get(values[i])->accept(*this, false);
+            path_.pop_back();
+        }
+
+        for (unsigned int i = 0; i < tables.size(); ++i)
+        {
+            path_.push_back(tables[i]);
+
+            if (values.size() > 0 || i > 0)
+                endline();
+
+            write_table_item_header(*t.get(tables[i]));
+            t.get(tables[i])->accept(*this, false);
+            path_.pop_back();
+        }
+
+        endline();
+    }
+
+    /**
+     * Output an array element of the TOML tree
+     */
+    void visit(const array& a, bool = false)
+    {
+        write("[");
+
+        for (unsigned int i = 0; i < a.get().size(); ++i)
+        {
+            if (i > 0)
+                write(", ");
+
+            if (a.get()[i]->is_array())
+            {
+                a.get()[i]->as_array()->accept(*this, true);
+            }
+            else
+            {
+                a.get()[i]->accept(*this, true);
+            }
+        }
+
+        write("]");
+    }
+
+    /**
+     * Output a table_array element of the TOML tree
+     */
+    void visit(const table_array& t, bool = false)
+    {
+        for (unsigned int j = 0; j < t.get().size(); ++j)
+        {
+            if (j > 0)
+                endline();
+
+            t.get()[j]->accept(*this, true);
+        }
+
+        endline();
+    }
+
+    /**
+     * Escape a string for output.
+     */
+    static std::string escape_string(const std::string& str)
+    {
+        std::string res;
+        for (auto it = str.begin(); it != str.end(); ++it)
+        {
+            if (*it == '\b')
+            {
+                res += "\\b";
+            }
+            else if (*it == '\t')
+            {
+                res += "\\t";
+            }
+            else if (*it == '\n')
+            {
+                res += "\\n";
+            }
+            else if (*it == '\f')
+            {
+                res += "\\f";
+            }
+            else if (*it == '\r')
+            {
+                res += "\\r";
+            }
+            else if (*it == '"')
+            {
+                res += "\\\"";
+            }
+            else if (*it == '\\')
+            {
+                res += "\\\\";
+            }
+            else if ((const uint32_t)*it <= 0x001f)
+            {
+                res += "\\u";
+                std::stringstream ss;
+                ss << std::hex << static_cast<uint32_t>(*it);
+                res += ss.str();
+            }
+            else
+            {
+                res += *it;
+            }
+        }
+        return res;
+    }
+
+  protected:
+    /**
+     * Write out a string.
+     */
+    void write(const value<std::string>& v)
+    {
+        write("\"");
+        write(escape_string(v.get()));
+        write("\"");
+    }
+
+    /**
+     * Write out a double.
+     */
+    void write(const value<double>& v)
+    {
+        std::ios::fmtflags flags{stream_.flags()};
+
+        stream_ << std::showpoint;
+        write(v.get());
+
+        stream_.flags(flags);
+    }
+
+    /**
+     * Write out an integer, local_date, local_time, local_datetime, or
+     * offset_datetime.
+     */
+    template <class T>
+    typename std::enable_if<is_one_of<T, int64_t, local_date, local_time,
+                                      local_datetime,
+                                      offset_datetime>::value>::type
+    write(const value<T>& v)
+    {
+        write(v.get());
+    }
+
+    /**
+     * Write out a boolean.
+     */
+    void write(const value<bool>& v)
+    {
+        write((v.get() ? "true" : "false"));
+    }
+
+    /**
+     * Write out the header of a table.
+     */
+    void write_table_header(bool in_array = false)
+    {
+        if (!path_.empty())
+        {
+            indent();
+
+            write("[");
+
+            if (in_array)
+            {
+                write("[");
+            }
+
+            for (unsigned int i = 0; i < path_.size(); ++i)
+            {
+                if (i > 0)
+                {
+                    write(".");
+                }
+
+                if (path_[i].find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcde"
+                                               "fghijklmnopqrstuvwxyz0123456789"
+                                               "_-")
+                    == std::string::npos)
+                {
+                    write(path_[i]);
+                }
+                else
+                {
+                    write("\"");
+                    write(escape_string(path_[i]));
+                    write("\"");
+                }
+            }
+
+            if (in_array)
+            {
+                write("]");
+            }
+
+            write("]");
+            endline();
+        }
+    }
+
+    /**
+     * Write out the identifier for an item in a table.
+     */
+    void write_table_item_header(const base& b)
+    {
+        if (!b.is_table() && !b.is_table_array())
+        {
+            indent();
+
+            if (path_.back().find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcde"
+                                               "fghijklmnopqrstuvwxyz0123456789"
+                                               "_-")
+                == std::string::npos)
+            {
+                write(path_.back());
+            }
+            else
+            {
+                write("\"");
+                write(escape_string(path_.back()));
+                write("\"");
+            }
+
+            write(" = ");
+        }
+    }
+
+  private:
+    /**
+     * Indent the proper number of tabs given the size of
+     * the path.
+     */
+    void indent()
+    {
+        for (std::size_t i = 1; i < path_.size(); ++i)
+            write(indent_);
+    }
+
+    /**
+     * Write a value out to the stream.
+     */
+    template <class T>
+    void write(const T& v)
+    {
+        stream_ << v;
+        has_naked_endline_ = false;
+    }
+
+    /**
+     * Write an endline out to the stream
+     */
+    void endline()
+    {
+        if (!has_naked_endline_)
+        {
+            stream_ << "\n";
+            has_naked_endline_ = true;
+        }
+    }
+
+  private:
+    std::ostream& stream_;
+    const std::string indent_;
+    std::vector<std::string> path_;
+    bool has_naked_endline_;
+};
+
+inline std::ostream& operator<<(std::ostream& stream, const base& b)
+{
+    toml_writer writer{stream};
+    b.accept(writer);
+    return stream;
+}
+
+template <class T>
+std::ostream& operator<<(std::ostream& stream, const value<T>& v)
+{
+    toml_writer writer{stream};
+    v.accept(writer);
+    return stream;
+}
+
+inline std::ostream& operator<<(std::ostream& stream, const table& t)
+{
+    toml_writer writer{stream};
+    t.accept(writer);
+    return stream;
+}
+
+inline std::ostream& operator<<(std::ostream& stream, const table_array& t)
+{
+    toml_writer writer{stream};
+    t.accept(writer);
+    return stream;
+}
+
+inline std::ostream& operator<<(std::ostream& stream, const array& a)
+{
+    toml_writer writer{stream};
+    a.accept(writer);
+    return stream;
+}
+}
+#endif
diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc
index b284daa3c2f7..0785897d2513 100644
--- a/src/libexpr/attr-set.cc
+++ b/src/libexpr/attr-set.cc
@@ -1,5 +1,5 @@
 #include "attr-set.hh"
-#include "eval.hh"
+#include "eval-inline.hh"
 
 #include <algorithm>
 
@@ -7,30 +7,18 @@
 namespace nix {
 
 
-/* Note: Various places expect the allocated memory to be zeroed. */
-static void * allocBytes(size_t n)
-{
-    void * p;
-#if HAVE_BOEHMGC
-    p = GC_malloc(n);
-#else
-    p = calloc(n, 1);
-#endif
-    if (!p) throw std::bad_alloc();
-    return p;
-}
-
-
 /* Allocate a new array of attributes for an attribute set with a specific
    capacity. The space is implicitly reserved after the Bindings
    structure. */
-Bindings * EvalState::allocBindings(Bindings::size_t capacity)
+Bindings * EvalState::allocBindings(size_t capacity)
 {
-    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings(capacity);
+    if (capacity > std::numeric_limits<Bindings::size_t>::max())
+        throw Error("attribute set of size %d is too big", capacity);
+    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity);
 }
 
 
-void EvalState::mkAttrs(Value & v, unsigned int capacity)
+void EvalState::mkAttrs(Value & v, size_t capacity)
 {
     if (capacity == 0) {
         v = vEmptySet;
diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index 8cc50e561354..c27116e3b448 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -78,4 +78,18 @@ inline void EvalState::forceList(Value & v, const Pos & pos)
         throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
 }
 
+/* Note: Various places expect the allocated memory to be zeroed. */
+inline void * allocBytes(size_t n)
+{
+    void * p;
+#if HAVE_BOEHMGC
+    p = GC_MALLOC(n);
+#else
+    p = calloc(n, 1);
+#endif
+    if (!p) throw std::bad_alloc();
+    return p;
+}
+
+
 }
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 37b977736e28..d8e10d9f20e1 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -6,27 +6,26 @@
 #include "globals.hh"
 #include "eval-inline.hh"
 #include "download.hh"
+#include "json.hh"
 
 #include <algorithm>
 #include <cstring>
 #include <unistd.h>
 #include <sys/time.h>
 #include <sys/resource.h>
+#include <iostream>
+#include <fstream>
+
+#include <sys/time.h>
+#include <sys/resource.h>
 
 #if HAVE_BOEHMGC
 
 #include <gc/gc.h>
 #include <gc/gc_cpp.h>
 
-#define NEW new (UseGC)
-
-#else
-
-#define NEW new
-
 #endif
 
-
 namespace nix {
 
 
@@ -34,7 +33,7 @@ static char * dupString(const char * s)
 {
     char * t;
 #if HAVE_BOEHMGC
-    t = GC_strdup(s);
+    t = GC_STRDUP(s);
 #else
     t = strdup(s);
 #endif
@@ -43,20 +42,6 @@ static char * dupString(const char * s)
 }
 
 
-/* Note: Various places expect the allocated memory to be zeroed. */
-static void * allocBytes(size_t n)
-{
-    void * p;
-#if HAVE_BOEHMGC
-    p = GC_malloc(n);
-#else
-    p = calloc(n, 1);
-#endif
-    if (!p) throw std::bad_alloc();
-    return p;
-}
-
-
 static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
 {
     checkInterrupt();
@@ -145,6 +130,16 @@ std::ostream & operator << (std::ostream & str, const Value & v)
 }
 
 
+const Value *getPrimOp(const Value &v) {
+    const Value * primOp = &v;
+    while (primOp->type == tPrimOpApp) {
+        primOp = primOp->primOpApp.left;
+    }
+    assert(primOp->type == tPrimOp);
+    return primOp;
+}
+
+
 string showType(const Value & v)
 {
     switch (v.type) {
@@ -159,8 +154,10 @@ string showType(const Value & v)
         case tApp: return "a function application";
         case tLambda: return "a function";
         case tBlackhole: return "a black hole";
-        case tPrimOp: return "a built-in function";
-        case tPrimOpApp: return "a partially applied built-in function";
+        case tPrimOp:
+            return fmt("the built-in function '%s'", string(v.primOp->name));
+        case tPrimOpApp:
+            return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
         case tExternal: return v.external->showType();
         case tFloat: return "a float";
     }
@@ -199,8 +196,15 @@ void initGC()
 
 #if HAVE_BOEHMGC
     /* Initialise the Boehm garbage collector. */
+
+    /* Don't look for interior pointers. This reduces the odds of
+       misdetection a bit. */
     GC_set_all_interior_pointers(0);
 
+    /* We don't have any roots in data segments, so don't scan from
+       there. */
+    GC_set_no_dls(1);
+
     GC_INIT();
 
     GC_set_oom_fn(oomHandler);
@@ -307,20 +311,32 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
     assert(gcInitialised);
 
+    static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
+
     /* Initialise the Nix expression search path. */
-    if (!settings.pureEval) {
+    if (!evalSettings.pureEval) {
         Strings paths = parseNixPath(getEnv("NIX_PATH", ""));
         for (auto & i : _searchPath) addToSearchPath(i);
         for (auto & i : paths) addToSearchPath(i);
     }
     addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true));
 
-    if (settings.restrictEval || settings.pureEval) {
+    if (evalSettings.restrictEval || evalSettings.pureEval) {
         allowedPaths = PathSet();
+
         for (auto & i : searchPath) {
             auto r = resolveSearchPathElem(i);
             if (!r.first) continue;
-            allowedPaths->insert(r.second);
+
+            auto path = r.second;
+
+            if (store->isInStore(r.second)) {
+                PathSet closure;
+                store->computeFSClosure(store->toStorePath(r.second), closure);
+                for (auto & path : closure)
+                    allowedPaths->insert(path);
+            } else
+                allowedPaths->insert(r.second);
         }
     }
 
@@ -334,7 +350,6 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
 EvalState::~EvalState()
 {
-    fileEvalCache.clear();
 }
 
 
@@ -342,25 +357,37 @@ Path EvalState::checkSourcePath(const Path & path_)
 {
     if (!allowedPaths) return path_;
 
+    auto i = resolvedPaths.find(path_);
+    if (i != resolvedPaths.end())
+        return i->second;
+
     bool found = false;
 
+    /* First canonicalize the path without symlinks, so we make sure an
+     * attacker can't append ../../... to a path that would be in allowedPaths
+     * and thus leak symlink targets.
+     */
+    Path abspath = canonPath(path_);
+
     for (auto & i : *allowedPaths) {
-        if (isDirOrInDir(path_, i)) {
+        if (isDirOrInDir(abspath, i)) {
             found = true;
             break;
         }
     }
 
     if (!found)
-        throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path_);
+        throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", abspath);
 
     /* Resolve symlinks. */
-    debug(format("checking access to '%s'") % path_);
-    Path path = canonPath(path_, true);
+    debug(format("checking access to '%s'") % abspath);
+    Path path = canonPath(abspath, true);
 
     for (auto & i : *allowedPaths) {
-        if (isDirOrInDir(path, i))
+        if (isDirOrInDir(path, i)) {
+            resolvedPaths[path_] = path;
             return path;
+        }
     }
 
     throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path);
@@ -369,13 +396,13 @@ Path EvalState::checkSourcePath(const Path & path_)
 
 void EvalState::checkURI(const std::string & uri)
 {
-    if (!settings.restrictEval) return;
+    if (!evalSettings.restrictEval) return;
 
     /* 'uri' should be equal to a prefix, or in a subdirectory of a
        prefix. Thus, the prefix https://github.co does not permit
        access to https://github.com. Note: this allows 'http://' and
        'https://' as prefixes for any http/https URI. */
-    for (auto & prefix : settings.allowedUris.get())
+    for (auto & prefix : evalSettings.allowedUris.get())
         if (uri == prefix ||
             (uri.size() > prefix.size()
             && prefix.size() > 0
@@ -422,7 +449,7 @@ Value * EvalState::addConstant(const string & name, Value & v)
 
 
 Value * EvalState::addPrimOp(const string & name,
-    unsigned int arity, PrimOpFun primOp)
+    size_t arity, PrimOpFun primOp)
 {
     if (arity == 0) {
         Value v;
@@ -433,7 +460,7 @@ Value * EvalState::addPrimOp(const string & name,
     string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
     Symbol sym = symbols.create(name2);
     v->type = tPrimOp;
-    v->primOp = NEW PrimOp(primOp, arity, sym);
+    v->primOp = new PrimOp(primOp, arity, sym);
     staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
     baseEnv.values[baseEnvDispl++] = v;
     baseEnv.values[0]->attrs->push_back(Attr(sym, v));
@@ -528,7 +555,7 @@ Value & mkString(Value & v, const string & s, const PathSet & context)
 {
     mkString(v, s.c_str());
     if (!context.empty()) {
-        unsigned int n = 0;
+        size_t n = 0;
         v.string.context = (const char * *)
             allocBytes((context.size() + 1) * sizeof(char *));
         for (auto & i : context)
@@ -547,17 +574,17 @@ void mkPath(Value & v, const char * s)
 
 inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 {
-    for (unsigned int l = var.level; l; --l, env = env->up) ;
+    for (size_t l = var.level; l; --l, env = env->up) ;
 
     if (!var.fromWith) return env->values[var.displ];
 
     while (1) {
-        if (!env->haveWithAttrs) {
+        if (env->type == Env::HasWithExpr) {
             if (noEval) return 0;
             Value * v = allocValue();
             evalAttrs(*env->up, (Expr *) env->values[0], *v);
             env->values[0] = v;
-            env->haveWithAttrs = true;
+            env->type = Env::HasWithAttrs;
         }
         Bindings::iterator j = env->values[0]->attrs->find(var.name);
         if (j != env->values[0]->attrs->end()) {
@@ -566,26 +593,37 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
         }
         if (!env->prevWith)
             throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, var.pos);
-        for (unsigned int l = env->prevWith; l; --l, env = env->up) ;
+        for (size_t l = env->prevWith; l; --l, env = env->up) ;
     }
 }
 
 
+std::atomic<uint64_t> nrValuesFreed{0};
+
+void finalizeValue(void * obj, void * data)
+{
+    nrValuesFreed++;
+}
+
 Value * EvalState::allocValue()
 {
     nrValues++;
-    return (Value *) allocBytes(sizeof(Value));
+    auto v = (Value *) allocBytes(sizeof(Value));
+    //GC_register_finalizer_no_order(v, finalizeValue, nullptr, nullptr, nullptr);
+    return v;
 }
 
 
-Env & EvalState::allocEnv(unsigned int size)
+Env & EvalState::allocEnv(size_t size)
 {
-    assert(size <= std::numeric_limits<decltype(Env::size)>::max());
+    if (size > std::numeric_limits<decltype(Env::size)>::max())
+        throw Error("environment size %d is too big", size);
 
     nrEnvs++;
     nrValuesInEnvs += size;
     Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
-    env->size = size;
+    env->size = (decltype(Env::size)) size;
+    env->type = Env::Plain;
 
     /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
 
@@ -593,7 +631,7 @@ Env & EvalState::allocEnv(unsigned int size)
 }
 
 
-void EvalState::mkList(Value & v, unsigned int size)
+void EvalState::mkList(Value & v, size_t size)
 {
     clearValue(v);
     if (size == 1)
@@ -628,7 +666,7 @@ void EvalState::mkThunk_(Value & v, Expr * expr)
 
 void EvalState::mkPos(Value & v, Pos * pos)
 {
-    if (pos) {
+    if (pos && pos->file.set()) {
         mkAttrs(v, 3);
         mkString(*allocAttr(v, sFile), pos->file);
         mkInt(*allocAttr(v, sLine), pos->line);
@@ -705,7 +743,17 @@ void EvalState::evalFile(const Path & path_, Value & v)
     }
 
     printTalkative("evaluating file '%1%'", path2);
-    Expr * e = parseExprFromFile(checkSourcePath(path2));
+    Expr * e = nullptr;
+
+    auto j = fileParseCache.find(path2);
+    if (j != fileParseCache.end())
+        e = j->second;
+
+    if (!e)
+        e = parseExprFromFile(checkSourcePath(path2));
+
+    fileParseCache[path2] = e;
+
     try {
         eval(e, v);
     } catch (Error & e) {
@@ -721,6 +769,7 @@ void EvalState::evalFile(const Path & path_, Value & v)
 void EvalState::resetFileCache()
 {
     fileEvalCache.clear();
+    fileParseCache.clear();
 }
 
 
@@ -805,7 +854,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         /* The recursive attributes are evaluated in the new
            environment, while the inherited attributes are evaluated
            in the original environment. */
-        unsigned int displ = 0;
+        size_t displ = 0;
         for (auto & i : attrs) {
             Value * vAttr;
             if (hasOverrides && !i.second.inherited) {
@@ -879,7 +928,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
     /* The recursive attributes are evaluated in the new environment,
        while the inherited attributes are evaluated in the original
        environment. */
-    unsigned int displ = 0;
+    size_t displ = 0;
     for (auto & i : attrs->attrs)
         env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2);
 
@@ -890,7 +939,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
 void ExprList::eval(EvalState & state, Env & env, Value & v)
 {
     state.mkList(v, elems.size());
-    for (unsigned int n = 0; n < elems.size(); ++n)
+    for (size_t n = 0; n < elems.size(); ++n)
         v.listElems()[n] = elems[n]->maybeThunk(state, env);
 }
 
@@ -1012,22 +1061,22 @@ void ExprApp::eval(EvalState & state, Env & env, Value & v)
 void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
 {
     /* Figure out the number of arguments still needed. */
-    unsigned int argsDone = 0;
+    size_t argsDone = 0;
     Value * primOp = &fun;
     while (primOp->type == tPrimOpApp) {
         argsDone++;
         primOp = primOp->primOpApp.left;
     }
     assert(primOp->type == tPrimOp);
-    unsigned int arity = primOp->primOp->arity;
-    unsigned int argsLeft = arity - argsDone;
+    auto arity = primOp->primOp->arity;
+    auto argsLeft = arity - argsDone;
 
     if (argsLeft == 1) {
         /* We have all the arguments, so call the primop. */
 
         /* Put all the arguments in an array. */
         Value * vArgs[arity];
-        unsigned int n = arity - 1;
+        auto n = arity - 1;
         vArgs[n--] = &arg;
         for (Value * arg = &fun; arg->type == tPrimOpApp; arg = arg->primOpApp.left)
             vArgs[n--] = arg->primOpApp.right;
@@ -1048,6 +1097,8 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
 
 void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos)
 {
+    forceValue(fun, pos);
+
     if (fun.type == tPrimOp || fun.type == tPrimOpApp) {
         callPrimOp(fun, arg, v, pos);
         return;
@@ -1063,10 +1114,8 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
         auto & fun2 = *allocValue();
         fun2 = fun;
         /* !!! Should we use the attr pos here? */
-        forceValue(*found->value, pos);
         Value v2;
         callFunction(*found->value, fun2, v2, pos);
-        forceValue(v2, pos);
         return callFunction(v2, arg, v, pos);
       }
     }
@@ -1076,13 +1125,13 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
 
     ExprLambda & lambda(*fun.lambda.fun);
 
-    unsigned int size =
+    auto size =
         (lambda.arg.empty() ? 0 : 1) +
         (lambda.matchAttrs ? lambda.formals->formals.size() : 0);
     Env & env2(allocEnv(size));
     env2.up = fun.lambda.env;
 
-    unsigned int displ = 0;
+    size_t displ = 0;
 
     if (!lambda.matchAttrs)
         env2.values[displ++] = &arg;
@@ -1096,7 +1145,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
         /* For each formal argument, get the actual argument.  If
            there is no matching actual argument but the formal
            argument has a default, use the default. */
-        unsigned int attrsUsed = 0;
+        size_t attrsUsed = 0;
         for (auto & i : lambda.formals->formals) {
             Bindings::iterator j = arg.attrs->find(i.name);
             if (j == arg.attrs->end()) {
@@ -1153,7 +1202,6 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
     if (fun.type == tAttrs) {
         auto found = fun.attrs->find(sFunctor);
         if (found != fun.attrs->end()) {
-            forceValue(*found->value);
             Value * v = allocValue();
             callFunction(*found->value, fun, *v, noPos);
             forceValue(*v);
@@ -1188,7 +1236,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
     Env & env2(state.allocEnv(1));
     env2.up = &env;
     env2.prevWith = prevWith;
-    env2.haveWithAttrs = false;
+    env2.type = Env::HasWithExpr;
     env2.values[0] = (Value *) attrs;
 
     body->eval(state, env2, v);
@@ -1294,15 +1342,15 @@ void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v)
 }
 
 
-void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists, const Pos & pos)
+void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos)
 {
     nrListConcats++;
 
     Value * nonEmpty = 0;
-    unsigned int len = 0;
-    for (unsigned int n = 0; n < nrLists; ++n) {
+    size_t len = 0;
+    for (size_t n = 0; n < nrLists; ++n) {
         forceList(*lists[n], pos);
-        unsigned int l = lists[n]->listSize();
+        auto l = lists[n]->listSize();
         len += l;
         if (l) nonEmpty = lists[n];
     }
@@ -1314,8 +1362,8 @@ void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists, co
 
     mkList(v, len);
     auto out = v.listElems();
-    for (unsigned int n = 0, pos = 0; n < nrLists; ++n) {
-        unsigned int l = lists[n]->listSize();
+    for (size_t n = 0, pos = 0; n < nrLists; ++n) {
+        auto l = lists[n]->listSize();
         if (l)
             memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *));
         pos += l;
@@ -1410,7 +1458,7 @@ void EvalState::forceValueDeep(Value & v)
         }
 
         else if (v.isList()) {
-            for (unsigned int n = 0; n < v.listSize(); ++n)
+            for (size_t n = 0; n < v.listSize(); ++n)
                 recurse(*v.listElems()[n]);
         }
     };
@@ -1537,7 +1585,6 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
     if (v.type == tAttrs) {
         auto i = v.attrs->find(sToString);
         if (i != v.attrs->end()) {
-            forceValue(*i->value, pos);
             Value v1;
             callFunction(*i->value, v, v1, pos);
             return coerceToString(pos, v1, context, coerceMore, copyToStore);
@@ -1562,7 +1609,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
 
         if (v.isList()) {
             string result;
-            for (unsigned int n = 0; n < v.listSize(); ++n) {
+            for (size_t n = 0; n < v.listSize(); ++n) {
                 result += coerceToString(pos, *v.listElems()[n],
                     context, coerceMore, copyToStore);
                 if (n < v.listSize() - 1
@@ -1649,7 +1696,7 @@ bool EvalState::eqValues(Value & v1, Value & v2)
         case tList2:
         case tListN:
             if (v1.listSize() != v2.listSize()) return false;
-            for (unsigned int n = 0; n < v1.listSize(); ++n)
+            for (size_t n = 0; n < v1.listSize(); ++n)
                 if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false;
             return true;
 
@@ -1691,12 +1738,9 @@ bool EvalState::eqValues(Value & v1, Value & v2)
     }
 }
 
-
 void EvalState::printStats()
 {
     bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0";
-    Verbosity v = showStats ? lvlInfo : lvlDebug;
-    printMsg(v, "evaluation statistics:");
 
     struct rusage buf;
     getrusage(RUSAGE_SELF, &buf);
@@ -1707,62 +1751,107 @@ void EvalState::printStats()
     uint64_t bValues = nrValues * sizeof(Value);
     uint64_t bAttrsets = nrAttrsets * sizeof(Bindings) + nrAttrsInAttrsets * sizeof(Attr);
 
-    printMsg(v, format("  time elapsed: %1%") % cpuTime);
-    printMsg(v, format("  size of a value: %1%") % sizeof(Value));
-    printMsg(v, format("  size of an attr: %1%") % sizeof(Attr));
-    printMsg(v, format("  environments allocated count: %1%") % nrEnvs);
-    printMsg(v, format("  environments allocated bytes: %1%") % bEnvs);
-    printMsg(v, format("  list elements count: %1%") % nrListElems);
-    printMsg(v, format("  list elements bytes: %1%") % bLists);
-    printMsg(v, format("  list concatenations: %1%") % nrListConcats);
-    printMsg(v, format("  values allocated count: %1%") % nrValues);
-    printMsg(v, format("  values allocated bytes: %1%") % bValues);
-    printMsg(v, format("  sets allocated: %1% (%2% bytes)") % nrAttrsets % bAttrsets);
-    printMsg(v, format("  right-biased unions: %1%") % nrOpUpdates);
-    printMsg(v, format("  values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied);
-    printMsg(v, format("  symbols in symbol table: %1%") % symbols.size());
-    printMsg(v, format("  size of symbol table: %1%") % symbols.totalSize());
-    printMsg(v, format("  number of thunks: %1%") % nrThunks);
-    printMsg(v, format("  number of thunks avoided: %1%") % nrAvoided);
-    printMsg(v, format("  number of attr lookups: %1%") % nrLookups);
-    printMsg(v, format("  number of primop calls: %1%") % nrPrimOpCalls);
-    printMsg(v, format("  number of function calls: %1%") % nrFunctionCalls);
-    printMsg(v, format("  total allocations: %1% bytes") % (bEnvs + bLists + bValues + bAttrsets));
-
 #if HAVE_BOEHMGC
     GC_word heapSize, totalBytes;
     GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes);
-    printMsg(v, format("  current Boehm heap size: %1% bytes") % heapSize);
-    printMsg(v, format("  total Boehm heap allocations: %1% bytes") % totalBytes);
+#endif
+    if (showStats) {
+        auto outPath = getEnv("NIX_SHOW_STATS_PATH","-");
+        std::fstream fs;
+        if (outPath != "-")
+            fs.open(outPath, std::fstream::out);
+        JSONObject topObj(outPath == "-" ? std::cerr : fs, true);
+        topObj.attr("cpuTime",cpuTime);
+        {
+            auto envs = topObj.object("envs");
+            envs.attr("number", nrEnvs);
+            envs.attr("elements", nrValuesInEnvs);
+            envs.attr("bytes", bEnvs);
+        }
+        {
+            auto lists = topObj.object("list");
+            lists.attr("elements", nrListElems);
+            lists.attr("bytes", bLists);
+            lists.attr("concats", nrListConcats);
+        }
+        {
+            auto values = topObj.object("values");
+            values.attr("number", nrValues);
+            values.attr("bytes", bValues);
+        }
+        {
+            auto syms = topObj.object("symbols");
+            syms.attr("number", symbols.size());
+            syms.attr("bytes", symbols.totalSize());
+        }
+        {
+            auto sets = topObj.object("sets");
+            sets.attr("number", nrAttrsets);
+            sets.attr("bytes", bAttrsets);
+            sets.attr("elements", nrAttrsInAttrsets);
+        }
+        {
+            auto sizes = topObj.object("sizes");
+            sizes.attr("Env", sizeof(Env));
+            sizes.attr("Value", sizeof(Value));
+            sizes.attr("Bindings", sizeof(Bindings));
+            sizes.attr("Attr", sizeof(Attr));
+        }
+        topObj.attr("nrOpUpdates", nrOpUpdates);
+        topObj.attr("nrOpUpdateValuesCopied", nrOpUpdateValuesCopied);
+        topObj.attr("nrThunks", nrThunks);
+        topObj.attr("nrAvoided", nrAvoided);
+        topObj.attr("nrLookups", nrLookups);
+        topObj.attr("nrPrimOpCalls", nrPrimOpCalls);
+        topObj.attr("nrFunctionCalls", nrFunctionCalls);
+#if HAVE_BOEHMGC
+        {
+            auto gc = topObj.object("gc");
+            gc.attr("heapSize", heapSize);
+            gc.attr("totalBytes", totalBytes);
+        }
 #endif
 
-    if (countCalls) {
-        v = lvlInfo;
-
-        printMsg(v, format("calls to %1% primops:") % primOpCalls.size());
-        typedef std::multimap<unsigned int, Symbol> PrimOpCalls_;
-        PrimOpCalls_ primOpCalls_;
-        for (auto & i : primOpCalls)
-            primOpCalls_.insert(std::pair<unsigned int, Symbol>(i.second, i.first));
-        for (auto i = primOpCalls_.rbegin(); i != primOpCalls_.rend(); ++i)
-            printMsg(v, format("%1$10d %2%") % i->first % i->second);
-
-        printMsg(v, format("calls to %1% functions:") % functionCalls.size());
-        typedef std::multimap<unsigned int, ExprLambda *> FunctionCalls_;
-        FunctionCalls_ functionCalls_;
-        for (auto & i : functionCalls)
-            functionCalls_.insert(std::pair<unsigned int, ExprLambda *>(i.second, i.first));
-        for (auto i = functionCalls_.rbegin(); i != functionCalls_.rend(); ++i)
-            printMsg(v, format("%1$10d %2%") % i->first % i->second->showNamePos());
-
-        printMsg(v, format("evaluations of %1% attributes:") % attrSelects.size());
-        typedef std::multimap<unsigned int, Pos> AttrSelects_;
-        AttrSelects_ attrSelects_;
-        for (auto & i : attrSelects)
-            attrSelects_.insert(std::pair<unsigned int, Pos>(i.second, i.first));
-        for (auto i = attrSelects_.rbegin(); i != attrSelects_.rend(); ++i)
-            printMsg(v, format("%1$10d %2%") % i->first % i->second);
+        if (countCalls) {
+            {
+                auto obj = topObj.object("primops");
+                for (auto & i : primOpCalls)
+                    obj.attr(i.first, i.second);
+            }
+            {
+                auto list = topObj.list("functions");
+                for (auto & i : functionCalls) {
+                    auto obj = list.object();
+                    if (i.first->name.set())
+                        obj.attr("name", (const string &) i.first->name);
+                    else
+                        obj.attr("name", nullptr);
+                    if (i.first->pos) {
+                        obj.attr("file", (const string &) i.first->pos.file);
+                        obj.attr("line", i.first->pos.line);
+                        obj.attr("column", i.first->pos.column);
+                    }
+                    obj.attr("count", i.second);
+                }
+            }
+            {
+                auto list = topObj.list("attributes");
+                for (auto & i : attrSelects) {
+                    auto obj = list.object();
+                    if (i.first) {
+                        obj.attr("file", (const string &) i.first.file);
+                        obj.attr("line", i.first.line);
+                        obj.attr("column", i.first.column);
+                    }
+                    obj.attr("count", i.second);
+                }
+            }
+        }
 
+        if (getEnv("NIX_SHOW_SYMBOLS", "0") != "0") {
+            auto list = topObj.list("symbols");
+            symbols.dump([&](const std::string & s) { list.elem(s); });
+        }
     }
 }
 
@@ -1810,7 +1899,7 @@ size_t valueSize(Value & v)
             if (seen.find(v.listElems()) == seen.end()) {
                 seen.insert(v.listElems());
                 sz += v.listSize() * sizeof(Value *);
-                for (unsigned int n = 0; n < v.listSize(); ++n)
+                for (size_t n = 0; n < v.listSize(); ++n)
                     sz += doValue(*v.listElems()[n]);
             }
             break;
@@ -1846,9 +1935,10 @@ size_t valueSize(Value & v)
 
         size_t sz = sizeof(Env) + sizeof(Value *) * env.size;
 
-        for (unsigned int i = 0; i < env.size; ++i)
-            if (env.values[i])
-                sz += doValue(*env.values[i]);
+        if (env.type != Env::HasWithExpr)
+            for (size_t i = 0; i < env.size; ++i)
+                if (env.values[i])
+                    sz += doValue(*env.values[i]);
 
         if (env.up) sz += doEnv(*env.up);
 
@@ -1877,4 +1967,9 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) {
 }
 
 
+EvalSettings evalSettings;
+
+static GlobalConfig::Register r1(&evalSettings);
+
+
 }
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 9d8799b7906b..a314e01e0a71 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -5,8 +5,10 @@
 #include "nixexpr.hh"
 #include "symbol-table.hh"
 #include "hash.hh"
+#include "config.hh"
 
 #include <map>
+#include <unordered_map>
 
 
 namespace nix {
@@ -34,8 +36,8 @@ struct Env
 {
     Env * up;
     unsigned short size; // used by โ€˜valueSizeโ€™
-    unsigned short prevWith:15; // nr of levels up to next `with' environment
-    unsigned short haveWithAttrs:1;
+    unsigned short prevWith:14; // nr of levels up to next `with' environment
+    enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2;
     Value * values[0];
 };
 
@@ -79,7 +81,7 @@ public:
 
     /* The allowed filesystem paths in restricted or pure evaluation
        mode. */
-    std::experimental::optional<PathSet> allowedPaths;
+    std::optional<PathSet> allowedPaths;
 
     Value vEmptySet;
 
@@ -88,6 +90,14 @@ public:
 private:
     SrcToStore srcToStore;
 
+    /* A cache from path names to parse trees. */
+#if HAVE_BOEHMGC
+    typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *> > > FileParseCache;
+#else
+    typedef std::map<Path, Expr *> FileParseCache;
+#endif
+    FileParseCache fileParseCache;
+
     /* A cache from path names to values. */
 #if HAVE_BOEHMGC
     typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value> > > FileEvalCache;
@@ -100,6 +110,9 @@ private:
 
     std::map<std::string, std::pair<bool, std::string>> searchPathResolved;
 
+    /* Cache used by checkSourcePath(). */
+    std::unordered_map<Path, Path> resolvedPaths;
+
 public:
 
     EvalState(const Strings & _searchPath, ref<Store> store);
@@ -214,7 +227,7 @@ private:
     Value * addConstant(const string & name, Value & v);
 
     Value * addPrimOp(const string & name,
-        unsigned int arity, PrimOpFun primOp);
+        size_t arity, PrimOpFun primOp);
 
 public:
 
@@ -248,18 +261,18 @@ public:
 
     /* Allocation primitives. */
     Value * allocValue();
-    Env & allocEnv(unsigned int size);
+    Env & allocEnv(size_t size);
 
     Value * allocAttr(Value & vAttrs, const Symbol & name);
 
-    Bindings * allocBindings(Bindings::size_t capacity);
+    Bindings * allocBindings(size_t capacity);
 
-    void mkList(Value & v, unsigned int length);
-    void mkAttrs(Value & v, unsigned int capacity);
+    void mkList(Value & v, size_t length);
+    void mkAttrs(Value & v, size_t capacity);
     void mkThunk_(Value & v, Expr * expr);
     void mkPos(Value & v, Pos * pos);
 
-    void concatLists(Value & v, unsigned int nrLists, Value * * lists, const Pos & pos);
+    void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos);
 
     /* Print statistics. */
     void printStats();
@@ -282,15 +295,15 @@ private:
 
     bool countCalls;
 
-    typedef std::map<Symbol, unsigned int> PrimOpCalls;
+    typedef std::map<Symbol, size_t> PrimOpCalls;
     PrimOpCalls primOpCalls;
 
-    typedef std::map<ExprLambda *, unsigned int> FunctionCalls;
+    typedef std::map<ExprLambda *, size_t> FunctionCalls;
     FunctionCalls functionCalls;
 
     void incrFunctionCall(ExprLambda * fun);
 
-    typedef std::map<Pos, unsigned int> AttrSelects;
+    typedef std::map<Pos, size_t> AttrSelects;
     AttrSelects attrSelects;
 
     friend struct ExprOpUpdate;
@@ -303,6 +316,9 @@ private:
 /* Return a string representing the type of the value `v'. */
 string showType(const Value & v);
 
+/* Decode a context string โ€˜!<name>!<path>โ€™ into a pair <path,
+   name>. */
+std::pair<string, string> decodeContext(const string & s);
 
 /* If `path' refers to a directory, then append "/default.nix". */
 Path resolveExprPath(Path path);
@@ -316,4 +332,25 @@ struct InvalidPathError : EvalError
 #endif
 };
 
+struct EvalSettings : Config
+{
+    Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
+        "Whether builtin functions that allow executing native code should be enabled."};
+
+    Setting<bool> restrictEval{this, false, "restrict-eval",
+        "Whether to restrict file system access to paths in $NIX_PATH, "
+        "and network access to the URI prefixes listed in 'allowed-uris'."};
+
+    Setting<bool> pureEval{this, false, "pure-eval",
+        "Whether to restrict file system and network access to files specified by cryptographic hash."};
+
+    Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation",
+        "Whether the evaluator allows importing the result of a derivation."};
+
+    Setting<Strings> allowedUris{this, {}, "allowed-uris",
+        "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
+};
+
+extern EvalSettings evalSettings;
+
 }
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index d38ed2df3b18..21a4d7917fce 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -295,7 +295,7 @@ static bool getDerivation(EvalState & state, Value & v,
 }
 
 
-std::experimental::optional<DrvInfo> getDerivation(EvalState & state, Value & v,
+std::optional<DrvInfo> getDerivation(EvalState & state, Value & v,
     bool ignoreAssertionFailures)
 {
     Done done;
diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh
index 4d9128e3f448..d7860fc6a4bc 100644
--- a/src/libexpr/get-drvs.hh
+++ b/src/libexpr/get-drvs.hh
@@ -44,7 +44,7 @@ public:
     string queryDrvPath() const;
     string queryOutPath() const;
     string queryOutputName() const;
-    /** Return the list of outputs. The "outputs to install" are determined by `mesa.outputsToInstall`. */
+    /** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */
     Outputs queryOutputs(bool onlyOutputsToInstall = false);
 
     StringSet queryMetaNames();
@@ -78,7 +78,7 @@ typedef list<DrvInfo> DrvInfos;
 
 /* If value `v' denotes a derivation, return a DrvInfo object
    describing it. Otherwise return nothing. */
-std::experimental::optional<DrvInfo> getDerivation(EvalState & state,
+std::optional<DrvInfo> getDerivation(EvalState & state,
     Value & v, bool ignoreAssertionFailures);
 
 void getDerivations(EvalState & state, Value & v, const string & pathPrefix,
diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc
index 8b1404595548..3f6017957782 100644
--- a/src/libexpr/json-to-value.cc
+++ b/src/libexpr/json-to-value.cc
@@ -110,7 +110,7 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
             if (number_type == tFloat)
                 mkFloat(v, stod(tmp_number));
             else
-                mkInt(v, stoi(tmp_number));
+                mkInt(v, stol(tmp_number));
         } catch (std::invalid_argument e) {
             throw JSONParseError("invalid JSON number");
         } catch (std::out_of_range e) {
diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index 1e9c29afa133..c34e5c383923 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -6,12 +6,14 @@
 %option nounput noyy_top_state
 
 
+%s DEFAULT
 %x STRING
 %x IND_STRING
-%x INSIDE_DOLLAR_CURLY
 
 
 %{
+#include <boost/lexical_cast.hpp>
+
 #include "nixexpr.hh"
 #include "parser-tab.hh"
 
@@ -97,8 +99,6 @@ URI         [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~
 
 %%
 
-<INITIAL,INSIDE_DOLLAR_CURLY>{
-
 
 if          { return IF; }
 then        { return THEN; }
@@ -124,9 +124,11 @@ or          { return OR_KW; }
 
 {ID}        { yylval->id = strdup(yytext); return ID; }
 {INT}       { errno = 0;
-              yylval->n = strtol(yytext, 0, 10);
-              if (errno != 0)
+              try {
+                  yylval->n = boost::lexical_cast<int64_t>(yytext);
+              } catch (const boost::bad_lexical_cast &) {
                   throw ParseError(format("invalid integer '%1%'") % yytext);
+              }
               return INT;
             }
 {FLOAT}     { errno = 0;
@@ -136,17 +138,19 @@ or          { return OR_KW; }
               return FLOAT;
             }
 
-\$\{        { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }
-}
+\$\{        { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
 
-\}                           { return '}'; }
-<INSIDE_DOLLAR_CURLY>\}      { POP_STATE(); return '}'; }
-\{                           { return '{'; }
-<INSIDE_DOLLAR_CURLY>\{      { PUSH_STATE(INSIDE_DOLLAR_CURLY); return '{'; }
+\}          { /* State INITIAL only exists at the bottom of the stack and is
+                 used as a marker. DEFAULT replaces it everywhere else.
+                 Popping when in INITIAL state causes an empty stack exception,
+                 so don't */
+              if (YYSTATE != INITIAL)
+                POP_STATE();
+              return '}';
+            }
+\{          { PUSH_STATE(DEFAULT); return '{'; }
 
-<INITIAL,INSIDE_DOLLAR_CURLY>\" {
-                PUSH_STATE(STRING); return '"';
-              }
+\"          { PUSH_STATE(STRING); return '"'; }
 <STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" |
 <STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ {
                 /* It is impossible to match strings ending with '$' with one
@@ -155,7 +159,7 @@ or          { return OR_KW; }
                 yylval->e = unescapeStr(data->symbols, yytext, yyleng);
                 return STR;
               }
-<STRING>\$\{  { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }
+<STRING>\$\{  { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
 <STRING>\"    { POP_STATE(); return '"'; }
 <STRING>\$|\\|\$\\ {
                 /* This can only occur when we reach EOF, otherwise the above
@@ -165,7 +169,7 @@ or          { return OR_KW; }
                 return STR;
               }
 
-<INITIAL,INSIDE_DOLLAR_CURLY>\'\'(\ *\n)?     { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
+\'\'(\ *\n)?     { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
 <IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ {
                    yylval->e = new ExprIndStr(yytext);
                    return IND_STR;
@@ -183,14 +187,13 @@ or          { return OR_KW; }
                    yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2);
                    return IND_STR;
                  }
-<IND_STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }
+<IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
 <IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; }
 <IND_STRING>\'   {
                    yylval->e = new ExprIndStr("'");
                    return IND_STR;
                  }
 
-<INITIAL,INSIDE_DOLLAR_CURLY>{
 
 {PATH}      { if (yytext[yyleng-1] == '/')
                   throw ParseError("path '%s' has a trailing slash", yytext);
@@ -209,11 +212,11 @@ or          { return OR_KW; }
 \#[^\r\n]*    /* single-line comments */
 \/\*([^*]|\*+[^*/])*\*+\/  /* long comments */
 
-{ANY}           return yytext[0];
-
-}
-
-<<EOF>> { data->atEnd = true; return 0; }
+{ANY}       {
+              /* Don't return a negative number, as this will cause
+                 Bison to stop parsing without an error. */
+              return (unsigned char) yytext[0];
+            }
 
 %%
 
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index daa3258f0d3c..ccd5293e4e5e 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -6,7 +6,7 @@ libexpr_DIR := $(d)
 
 libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
 
-libexpr_LIBS = libutil libstore libformat
+libexpr_LIBS = libutil libstore
 
 libexpr_LDFLAGS =
 ifneq ($(OS), FreeBSD)
diff --git a/src/libexpr/nix-expr.pc.in b/src/libexpr/nix-expr.pc.in
index 79f3e2f4506e..80f7a492b1a1 100644
--- a/src/libexpr/nix-expr.pc.in
+++ b/src/libexpr/nix-expr.pc.in
@@ -7,4 +7,4 @@ Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Requires: nix-store bdw-gc
 Libs: -L${libdir} -lnixexpr
-Cflags: -I${includedir}/nix -std=c++14
+Cflags: -I${includedir}/nix -std=c++17
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 8c8f39640681..665a42987dc1 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -11,7 +11,6 @@ namespace nix {
 
 MakeError(EvalError, Error)
 MakeError(ParseError, Error)
-MakeError(IncompleteParseError, ParseError)
 MakeError(AssertionError, EvalError)
 MakeError(ThrownError, AssertionError)
 MakeError(Abort, EvalError)
@@ -255,7 +254,7 @@ struct ExprWith : Expr
 {
     Pos pos;
     Expr * attrs, * body;
-    unsigned int prevWith;
+    size_t prevWith;
     ExprWith(const Pos & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { };
     COMMON_METHODS
 };
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index df4fdf032479..97e1a65849e2 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -1,7 +1,7 @@
 %glr-parser
 %pure-parser
 %locations
-%error-verbose
+%define parse.error verbose
 %defines
 /* %no-lines */
 %parse-param { void * scanner }
@@ -31,12 +31,10 @@ namespace nix {
         Path basePath;
         Symbol path;
         string error;
-        bool atEnd;
         Symbol sLetBody;
         ParseData(EvalState & state)
             : state(state)
             , symbols(state.symbols)
-            , atEnd(false)
             , sLetBody(symbols.create("<let-body>"))
             { };
     };
@@ -156,8 +154,8 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
        whitespace-only final lines are not taken into account.  (So
        the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */
     bool atStartOfLine = true; /* = seen only whitespace in the current line */
-    unsigned int minIndent = 1000000;
-    unsigned int curIndent = 0;
+    size_t minIndent = 1000000;
+    size_t curIndent = 0;
     for (auto & i : es) {
         ExprIndStr * e = dynamic_cast<ExprIndStr *>(i);
         if (!e) {
@@ -168,7 +166,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
             }
             continue;
         }
-        for (unsigned int j = 0; j < e->s.size(); ++j) {
+        for (size_t j = 0; j < e->s.size(); ++j) {
             if (atStartOfLine) {
                 if (e->s[j] == ' ')
                     curIndent++;
@@ -190,8 +188,8 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
     /* Strip spaces from each line. */
     vector<Expr *> * es2 = new vector<Expr *>;
     atStartOfLine = true;
-    unsigned int curDropped = 0;
-    unsigned int n = es.size();
+    size_t curDropped = 0;
+    size_t n = es.size();
     for (vector<Expr *>::iterator i = es.begin(); i != es.end(); ++i, --n) {
         ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i);
         if (!e) {
@@ -202,7 +200,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
         }
 
         string s2;
-        for (unsigned int j = 0; j < e->s.size(); ++j) {
+        for (size_t j = 0; j < e->s.size(); ++j) {
             if (atStartOfLine) {
                 if (e->s[j] == ' ') {
                     if (curDropped++ >= minIndent)
@@ -295,11 +293,11 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 %token IND_STRING_OPEN IND_STRING_CLOSE
 %token ELLIPSIS
 
-%nonassoc IMPL
+%right IMPL
 %left OR
 %left AND
 %nonassoc EQ NEQ
-%left '<' '>' LEQ GEQ
+%nonassoc '<' '>' LEQ GEQ
 %right UPDATE
 %left NOT
 %left '+' '-'
@@ -561,12 +559,7 @@ Expr * EvalState::parse(const char * text,
     int res = yyparse(scanner, &data);
     yylex_destroy(scanner);
 
-    if (res) {
-      if (data.atEnd)
-        throw IncompleteParseError(data.error);
-      else
-        throw ParseError(data.error);
-    }
+    if (res) throw ParseError(data.error);
 
     data.result->bindVars(staticEnv);
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 9e7651da725f..06f577f36fce 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -73,7 +73,7 @@ void EvalState::realiseContext(const PathSet & context)
 
     if (drvs.empty()) return;
 
-    if (!settings.enableImportFromDerivation)
+    if (!evalSettings.enableImportFromDerivation)
         throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin()));
 
     /* For performance, prefetch all substitute info. */
@@ -271,7 +271,18 @@ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Valu
 static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0]);
-    mkBool(v, args[0]->type == tLambda);
+    bool res;
+    switch (args[0]->type) {
+        case tLambda:
+        case tPrimOp:
+        case tPrimOpApp:
+            res = true;
+            break;
+        default:
+            res = false;
+            break;
+    }
+    mkBool(v, res);
 }
 
 
@@ -304,6 +315,12 @@ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Valu
     mkBool(v, args[0]->type == tBool);
 }
 
+/* Determine whether the argument is a path. */
+static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceValue(*args[0]);
+    mkBool(v, args[0]->type == tPath);
+}
 
 struct CompareValues
 {
@@ -453,7 +470,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val
 static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string name = state.forceStringNoCtx(*args[0], pos);
-    mkString(v, settings.restrictEval || settings.pureEval ? "" : getEnv(name));
+    mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name));
 }
 
 
@@ -544,7 +561,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     PathSet context;
 
-    std::experimental::optional<std::string> outputHash;
+    std::optional<std::string> outputHash;
     std::string outputHashAlgo;
     bool outputHashRecursive = false;
 
@@ -676,21 +693,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             }
         }
 
-        /* See prim_unsafeDiscardOutputDependency. */
-        else if (path.at(0) == '~')
-            drv.inputSrcs.insert(string(path, 1));
-
         /* Handle derivation outputs of the form โ€˜!<name>!<path>โ€™. */
         else if (path.at(0) == '!') {
             std::pair<string, string> ctx = decodeContext(path);
             drv.inputDrvs[ctx.first].insert(ctx.second);
         }
 
-        /* Handle derivation contexts returned by
-           โ€˜builtins.storePathโ€™. */
-        else if (isDerivation(path))
-            drv.inputDrvs[path] = state.store->queryDerivationOutputNames(path);
-
         /* Otherwise it's a source file. */
         else
             drv.inputSrcs.insert(path);
@@ -713,16 +721,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         if (outputs.size() != 1 || *(outputs.begin()) != "out")
             throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName);
 
-        HashType ht = parseHashType(outputHashAlgo);
-        if (ht == htUnknown)
-            throw EvalError(format("unknown hash algorithm '%1%', at %2%") % outputHashAlgo % posDrvName);
+        HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
         Hash h(*outputHash, ht);
-        outputHash = h.to_string(Base16, false);
-        if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo;
 
         Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
         if (!jsonObject) drv.env["out"] = outPath;
-        drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, *outputHash);
+        drv.outputs["out"] = DerivationOutput(outPath,
+            (outputHashRecursive ? "r:" : "") + printHashType(h.type),
+            h.to_string(Base16, false));
     }
 
     else {
@@ -855,7 +861,7 @@ static void prim_baseNameOf(EvalState & state, const Pos & pos, Value * * args,
 static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     PathSet context;
-    Path dir = dirOf(state.coerceToPath(pos, *args[0], context));
+    Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false));
     if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context);
 }
 
@@ -917,6 +923,20 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
     mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str());
 }
 
+/* Return the cryptographic hash of a file in base-16. */
+static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    string type = state.forceStringNoCtx(*args[0], pos);
+    HashType ht = parseHashType(type);
+    if (ht == htUnknown)
+      throw Error(format("unknown hash type '%1%', at %2%") % type % pos);
+
+    PathSet context; // discarded
+    Path p = state.coerceToPath(pos, *args[1], context);
+
+    mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context);
+}
+
 /* Read a directory (without . or ..) */
 static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -995,13 +1015,8 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
     PathSet refs;
 
     for (auto path : context) {
-        if (path.at(0) == '=') path = string(path, 1);
-        if (isDerivation(path)) {
-            /* See prim_unsafeDiscardOutputDependency. */
-            if (path.at(0) != '~')
-                throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
-            path = string(path, 1);
-        }
+        if (path.at(0) != '/')
+            throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
         refs.insert(path);
     }
 
@@ -1020,7 +1035,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
     Value * filterFun, bool recursive, const Hash & expectedHash, Value & v)
 {
-    const auto path = settings.pureEval && expectedHash ?
+    const auto path = evalSettings.pureEval && expectedHash ?
         path_ :
         state.checkSourcePath(path_);
     PathFilter filter = filterFun ? ([&](const Path & path) {
@@ -1345,6 +1360,24 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
 }
 
 
+/* Apply a function to every element of an attribute set. */
+static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceAttrs(*args[1], pos);
+
+    state.mkAttrs(v, args[1]->attrs->size());
+
+    for (auto & i : *args[1]->attrs) {
+        Value * vName = state.allocValue();
+        Value * vFun2 = state.allocValue();
+        mkString(*vName, i.name);
+        mkApp(*vFun2, *args[0], *vName);
+        mkApp(*state.allocAttr(v, i.name), *vFun2, *i.value);
+    }
+}
+
+
+
 /*************************************************************
  * Lists
  *************************************************************/
@@ -1399,7 +1432,6 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
 /* Apply a function to every element of a list. */
 static void prim_map(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceFunction(*args[0], pos);
     state.forceList(*args[1], pos);
 
     state.mkList(v, args[1]->listSize());
@@ -1478,19 +1510,20 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args,
     state.forceFunction(*args[0], pos);
     state.forceList(*args[2], pos);
 
-    Value * vCur = args[1];
+    if (args[2]->listSize()) {
+        Value * vCur = args[1];
 
-    if (args[2]->listSize())
         for (unsigned int n = 0; n < args[2]->listSize(); ++n) {
             Value vTmp;
             state.callFunction(*args[0], *vCur, vTmp, pos);
             vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue();
             state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos);
         }
-    else
-        v = *vCur;
-
-    state.forceValue(v);
+        state.forceValue(v);
+    } else {
+        state.forceValue(*args[1]);
+        v = *args[1];
+    }
 }
 
 
@@ -1527,7 +1560,6 @@ static void prim_all(EvalState & state, const Pos & pos, Value * * args, Value &
 
 static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceFunction(*args[0], pos);
     auto len = state.forceInt(*args[1], pos);
 
     if (len < 0)
@@ -1616,6 +1648,35 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V
 }
 
 
+/* concatMap = f: list: concatLists (map f list); */
+/* C++-version is to avoid allocating `mkApp', call `f' eagerly */
+static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceFunction(*args[0], pos);
+    state.forceList(*args[1], pos);
+    auto nrLists = args[1]->listSize();
+
+    Value lists[nrLists];
+    size_t len = 0;
+
+    for (unsigned int n = 0; n < nrLists; ++n) {
+        Value * vElem = args[1]->listElems()[n];
+        state.callFunction(*args[0], *vElem, lists[n], pos);
+        state.forceList(lists[n], pos);
+        len += lists[n].listSize();
+    }
+
+    state.mkList(v, len);
+    auto out = v.listElems();
+    for (unsigned int n = 0, pos = 0; n < nrLists; ++n) {
+        auto l = lists[n].listSize();
+        if (l)
+            memcpy(out + pos, lists[n].listElems(), l * sizeof(Value *));
+        pos += l;
+    }
+}
+
+
 /*************************************************************
  * Integer arithmetic
  *************************************************************/
@@ -1623,6 +1684,8 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V
 
 static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
     if (args[0]->type == tFloat || args[1]->type == tFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos));
     else
@@ -1632,6 +1695,8 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value &
 
 static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
     if (args[0]->type == tFloat || args[1]->type == tFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos));
     else
@@ -1641,6 +1706,8 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value &
 
 static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
     if (args[0]->type == tFloat || args[1]->type == tFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos));
     else
@@ -1650,6 +1717,9 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value &
 
 static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
+
     NixFloat f2 = state.forceFloat(*args[1], pos);
     if (f2 == 0) throw EvalError(format("division by zero, at %1%") % pos);
 
@@ -1665,6 +1735,20 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
     }
 }
 
+static void prim_bitAnd(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    mkInt(v, state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos));
+}
+
+static void prim_bitOr(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    mkInt(v, state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos));
+}
+
+static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    mkInt(v, state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos));
+}
 
 static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -1716,41 +1800,6 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args
 }
 
 
-static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    PathSet context;
-    string s = state.coerceToString(pos, *args[0], context);
-    mkString(v, s, PathSet());
-}
-
-
-static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    PathSet context;
-    state.forceString(*args[0], context, pos);
-    mkBool(v, !context.empty());
-}
-
-
-/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
-   builder without causing the derivation to be built (for instance,
-   in the derivation that builds NARs in nix-push, when doing
-   source-only deployment).  This primop marks the string context so
-   that builtins.derivation adds the path to drv.inputSrcs rather than
-   drv.inputDrvs. */
-static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    PathSet context;
-    string s = state.coerceToString(pos, *args[0], context);
-
-    PathSet context2;
-    for (auto & p : context)
-        context2.insert(p.at(0) == '=' ? "~" + string(p, 1) : p);
-
-    mkString(v, s, context2);
-}
-
-
 /* Return the cryptographic hash of a string in base-16. */
 static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -2031,7 +2080,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
 
     state.checkURI(url);
 
-    if (settings.pureEval && !expectedHash)
+    if (evalSettings.pureEval && !expectedHash)
         throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
 
     Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash);
@@ -2099,12 +2148,12 @@ void EvalState::createBaseEnv()
         addConstant(name, v);
     };
 
-    if (!settings.pureEval) {
+    if (!evalSettings.pureEval) {
         mkInt(v, time(0));
         addConstant("__currentTime", v);
     }
 
-    if (!settings.pureEval) {
+    if (!evalSettings.pureEval) {
         mkString(v, settings.thisSystem);
         addConstant("__currentSystem", v);
     }
@@ -2129,7 +2178,7 @@ void EvalState::createBaseEnv()
     mkApp(v, *vScopedImport, *v2);
     forceValue(v);
     addConstant("import", v);
-    if (settings.enableNativeCode) {
+    if (evalSettings.enableNativeCode) {
         addPrimOp("__importNative", 2, prim_importNative);
         addPrimOp("__exec", 1, prim_exec);
     }
@@ -2140,6 +2189,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__isInt", 1, prim_isInt);
     addPrimOp("__isFloat", 1, prim_isFloat);
     addPrimOp("__isBool", 1, prim_isBool);
+    addPrimOp("__isPath", 1, prim_isPath);
     addPrimOp("__genericClosure", 1, prim_genericClosure);
     addPrimOp("abort", 1, prim_abort);
     addPrimOp("__addErrorContext", 2, prim_addErrorContext);
@@ -2156,7 +2206,7 @@ void EvalState::createBaseEnv()
 
     // Paths
     addPrimOp("__toPath", 1, prim_toPath);
-    if (settings.pureEval)
+    if (evalSettings.pureEval)
         addPurityError("__storePath");
     else
         addPrimOp("__storePath", 1, prim_storePath);
@@ -2166,6 +2216,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__readFile", 1, prim_readFile);
     addPrimOp("__readDir", 1, prim_readDir);
     addPrimOp("__findFile", 2, prim_findFile);
+    addPrimOp("__hashFile", 2, prim_hashFile);
 
     // Creating files
     addPrimOp("__toXML", 1, prim_toXML);
@@ -2187,6 +2238,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__intersectAttrs", 2, prim_intersectAttrs);
     addPrimOp("__catAttrs", 2, prim_catAttrs);
     addPrimOp("__functionArgs", 1, prim_functionArgs);
+    addPrimOp("__mapAttrs", 2, prim_mapAttrs);
 
     // Lists
     addPrimOp("__isList", 1, prim_isList);
@@ -2204,21 +2256,22 @@ void EvalState::createBaseEnv()
     addPrimOp("__genList", 2, prim_genList);
     addPrimOp("__sort", 2, prim_sort);
     addPrimOp("__partition", 2, prim_partition);
+    addPrimOp("__concatMap", 2, prim_concatMap);
 
     // Integer arithmetic
     addPrimOp("__add", 2, prim_add);
     addPrimOp("__sub", 2, prim_sub);
     addPrimOp("__mul", 2, prim_mul);
     addPrimOp("__div", 2, prim_div);
+    addPrimOp("__bitAnd", 2, prim_bitAnd);
+    addPrimOp("__bitOr", 2, prim_bitOr);
+    addPrimOp("__bitXor", 2, prim_bitXor);
     addPrimOp("__lessThan", 2, prim_lessThan);
 
     // String manipulation
     addPrimOp("toString", 1, prim_toString);
     addPrimOp("__substring", 3, prim_substring);
     addPrimOp("__stringLength", 1, prim_stringLength);
-    addPrimOp("__hasContext", 1, prim_hasContext);
-    addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
-    addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
     addPrimOp("__hashString", 2, prim_hashString);
     addPrimOp("__match", 2, prim_match);
     addPrimOp("__split", 2, prim_split);
diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc
new file mode 100644
index 000000000000..2d79739ea047
--- /dev/null
+++ b/src/libexpr/primops/context.cc
@@ -0,0 +1,187 @@
+#include "primops.hh"
+#include "eval-inline.hh"
+#include "derivations.hh"
+
+namespace nix {
+
+static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    string s = state.coerceToString(pos, *args[0], context);
+    mkString(v, s, PathSet());
+}
+
+static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
+
+
+static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    state.forceString(*args[0], context, pos);
+    mkBool(v, !context.empty());
+}
+
+static RegisterPrimOp r2("__hasContext", 1, prim_hasContext);
+
+
+/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
+   builder without causing the derivation to be built (for instance,
+   in the derivation that builds NARs in nix-push, when doing
+   source-only deployment).  This primop marks the string context so
+   that builtins.derivation adds the path to drv.inputSrcs rather than
+   drv.inputDrvs. */
+static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    string s = state.coerceToString(pos, *args[0], context);
+
+    PathSet context2;
+    for (auto & p : context)
+        context2.insert(p.at(0) == '=' ? string(p, 1) : p);
+
+    mkString(v, s, context2);
+}
+
+static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
+
+
+/* Extract the context of a string as a structured Nix value.
+
+   The context is represented as an attribute set whose keys are the
+   paths in the context set and whose values are attribute sets with
+   the following keys:
+     path: True if the relevant path is in the context as a plain store
+           path (i.e. the kind of context you get when interpolating
+           a Nix path (e.g. ./.) into a string). False if missing.
+     allOutputs: True if the relevant path is a derivation and it is
+                  in the context as a drv file with all of its outputs
+                  (i.e. the kind of context you get when referencing
+                  .drvPath of some derivation). False if missing.
+     outputs: If a non-empty list, the relevant path is a derivation
+              and the provided outputs are referenced in the context
+              (i.e. the kind of context you get when referencing
+              .outPath of some derivation). Empty list if missing.
+   Note that for a given path any combination of the above attributes
+   may be present.
+*/
+static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    struct ContextInfo {
+        bool path = false;
+        bool allOutputs = false;
+        Strings outputs;
+    };
+    PathSet context;
+    state.forceString(*args[0], context, pos);
+    auto contextInfos = std::map<Path, ContextInfo>();
+    for (const auto & p : context) {
+        Path drv;
+        string output;
+        const Path * path = &p;
+        if (p.at(0) == '=') {
+            drv = string(p, 1);
+            path = &drv;
+        } else if (p.at(0) == '!') {
+            std::pair<string, string> ctx = decodeContext(p);
+            drv = ctx.first;
+            output = ctx.second;
+            path = &drv;
+        }
+        auto isPath = drv.empty();
+        auto isAllOutputs = (!drv.empty()) && output.empty();
+
+        auto iter = contextInfos.find(*path);
+        if (iter == contextInfos.end()) {
+            contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}});
+        } else {
+            if (isPath)
+                iter->second.path = true;
+            else if (isAllOutputs)
+                iter->second.allOutputs = true;
+            else
+                iter->second.outputs.emplace_back(std::move(output));
+        }
+    }
+
+    state.mkAttrs(v, contextInfos.size());
+
+    auto sPath = state.symbols.create("path");
+    auto sAllOutputs = state.symbols.create("allOutputs");
+    for (const auto & info : contextInfos) {
+        auto & infoVal = *state.allocAttr(v, state.symbols.create(info.first));
+        state.mkAttrs(infoVal, 3);
+        if (info.second.path)
+            mkBool(*state.allocAttr(infoVal, sPath), true);
+        if (info.second.allOutputs)
+            mkBool(*state.allocAttr(infoVal, sAllOutputs), true);
+        if (!info.second.outputs.empty()) {
+            auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs);
+            state.mkList(outputsVal, info.second.outputs.size());
+            size_t i = 0;
+            for (const auto & output : info.second.outputs) {
+                mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output);
+            }
+        }
+        infoVal.attrs->sort();
+    }
+    v.attrs->sort();
+}
+
+static RegisterPrimOp r4("__getContext", 1, prim_getContext);
+
+
+/* Append the given context to a given string.
+
+   See the commentary above unsafeGetContext for details of the
+   context representation.
+*/
+static void prim_appendContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    auto orig = state.forceString(*args[0], context, pos);
+
+    state.forceAttrs(*args[1], pos);
+
+    auto sPath = state.symbols.create("path");
+    auto sAllOutputs = state.symbols.create("allOutputs");
+    for (auto & i : *args[1]->attrs) {
+        if (!state.store->isStorePath(i.name))
+            throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos);
+        if (!settings.readOnlyMode)
+            state.store->ensurePath(i.name);
+        state.forceAttrs(*i.value, *i.pos);
+        auto iter = i.value->attrs->find(sPath);
+        if (iter != i.value->attrs->end()) {
+            if (state.forceBool(*iter->value, *iter->pos))
+                context.insert(i.name);
+        }
+
+        iter = i.value->attrs->find(sAllOutputs);
+        if (iter != i.value->attrs->end()) {
+            if (state.forceBool(*iter->value, *iter->pos)) {
+                if (!isDerivation(i.name)) {
+                    throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+                }
+                context.insert("=" + string(i.name));
+            }
+        }
+
+        iter = i.value->attrs->find(state.sOutputs);
+        if (iter != i.value->attrs->end()) {
+            state.forceList(*iter->value, *iter->pos);
+            if (iter->value->listSize() && !isDerivation(i.name)) {
+                throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+            }
+            for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
+                auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);
+                context.insert("!" + name + "!" + string(i.name));
+            }
+        }
+    }
+
+    mkString(v, orig, context);
+}
+
+static RegisterPrimOp r5("__appendContext", 2, prim_appendContext);
+
+}
diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 8bb74dad639e..aaf02c856d4f 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -3,6 +3,7 @@
 #include "download.hh"
 #include "store-api.hh"
 #include "pathlocks.hh"
+#include "hash.hh"
 
 #include <sys/time.h>
 
@@ -25,10 +26,10 @@ struct GitInfo
 std::regex revRegex("^[0-9a-fA-F]{40}$");
 
 GitInfo exportGit(ref<Store> store, const std::string & uri,
-    std::experimental::optional<std::string> ref, std::string rev,
+    std::optional<std::string> ref, std::string rev,
     const std::string & name)
 {
-    if (settings.pureEval && rev == "")
+    if (evalSettings.pureEval && rev == "")
         throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
 
     if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
@@ -84,15 +85,16 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
     if (rev != "" && !std::regex_match(rev, revRegex))
         throw Error("invalid Git revision '%s'", rev);
 
-    Path cacheDir = getCacheDir() + "/nix/git";
+    deletePath(getCacheDir() + "/nix/git");
+
+    Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
 
     if (!pathExists(cacheDir)) {
+        createDirs(dirOf(cacheDir));
         runProgram("git", true, { "init", "--bare", cacheDir });
     }
 
-    std::string localRef = hashString(htSHA256, fmt("%s-%s", uri, *ref)).to_string(Base32, false);
-
-    Path localRefFile = cacheDir + "/refs/heads/" + localRef;
+    Path localRefFile = cacheDir + "/refs/heads/" + *ref;
 
     bool doFetch;
     time_t now = time(0);
@@ -122,7 +124,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
         // FIXME: git stderr messes up our progress indicator, so
         // we're using --quiet for now. Should process its stderr.
-        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, *ref + ":" + localRef });
+        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });
 
         struct timeval times[2];
         times[0].tv_sec = now;
@@ -188,7 +190,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     std::string url;
-    std::experimental::optional<std::string> ref;
+    std::optional<std::string> ref;
     std::string rev;
     std::string name = "source";
     PathSet context;
@@ -219,8 +221,6 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
 
-    if (!isUri(url)) url = absPath(url);
-
     // FIXME: git externals probably can be used to bypass the URI
     // whitelist. Ah well.
     state.checkURI(url);
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index a75c5fc2ddff..66f49f374321 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -27,7 +27,7 @@ std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
 HgInfo exportMercurial(ref<Store> store, const std::string & uri,
     std::string rev, const std::string & name)
 {
-    if (settings.pureEval && rev == "")
+    if (evalSettings.pureEval && rev == "")
         throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
 
     if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {
@@ -93,7 +93,22 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
             Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));
 
             if (pathExists(cacheDir)) {
-                runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
+                try {
+                    runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
+                }
+                catch (ExecError & e){
+                    string transJournal = cacheDir + "/.hg/store/journal";
+                    /* hg throws "abandoned transaction" error only if this file exists */
+                    if (pathExists(transJournal))
+                    {
+                        runProgram("hg", true, { "recover", "-R", cacheDir });
+                        runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
+                    }
+                    else 
+                    {
+                        throw ExecError(e.status, fmt("program hg '%1%' ", statusToString(e.status)));
+                    }
+                }
             } else {
                 createDirs(dirOf(cacheDir));
                 runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir });
@@ -184,8 +199,6 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
 
-    if (!isUri(url)) url = absPath(url);
-
     // FIXME: git externals probably can be used to bypass the URI
     // whitelist. Ah well.
     state.checkURI(url);
diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc
new file mode 100644
index 000000000000..4128de05d0cf
--- /dev/null
+++ b/src/libexpr/primops/fromTOML.cc
@@ -0,0 +1,77 @@
+#include "primops.hh"
+#include "eval-inline.hh"
+
+#include "cpptoml/cpptoml.h"
+
+namespace nix {
+
+static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    using namespace cpptoml;
+
+    auto toml = state.forceStringNoCtx(*args[0], pos);
+
+    std::istringstream tomlStream(toml);
+
+    std::function<void(Value &, std::shared_ptr<base>)> visit;
+
+    visit = [&](Value & v, std::shared_ptr<base> t) {
+
+        if (auto t2 = t->as_table()) {
+
+            size_t size = 0;
+            for (auto & i : *t2) { (void) i; size++; }
+
+            state.mkAttrs(v, size);
+
+            for (auto & i : *t2) {
+                auto & v2 = *state.allocAttr(v, state.symbols.create(i.first));
+
+                if (auto i2 = i.second->as_table_array()) {
+                    size_t size2 = i2->get().size();
+                    state.mkList(v2, size2);
+                    for (size_t j = 0; j < size2; ++j)
+                        visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]);
+                }
+                else
+                    visit(v2, i.second);
+            }
+
+            v.attrs->sort();
+        }
+
+        else if (auto t2 = t->as_array()) {
+            size_t size = t2->get().size();
+
+            state.mkList(v, size);
+
+            for (size_t i = 0; i < size; ++i)
+                visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]);
+        }
+
+        else if (t->is_value()) {
+            if (auto val = t->as<int64_t>())
+                mkInt(v, val->get());
+            else if (auto val = t->as<NixFloat>())
+                mkFloat(v, val->get());
+            else if (auto val = t->as<bool>())
+                mkBool(v, val->get());
+            else if (auto val = t->as<std::string>())
+                mkString(v, val->get());
+            else
+                throw EvalError("unsupported value type in TOML");
+        }
+
+        else abort();
+    };
+
+    try {
+        visit(v, parser(tomlStream).parse());
+    } catch (std::runtime_error & e) {
+        throw EvalError("while parsing a TOML string at %s: %s", pos, e.what());
+    }
+}
+
+static RegisterPrimOp r("fromTOML", 1, prim_fromTOML);
+
+}
diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh
index c2ee49dd32fb..91faea122ce1 100644
--- a/src/libexpr/symbol-table.hh
+++ b/src/libexpr/symbol-table.hh
@@ -69,12 +69,19 @@ public:
         return Symbol(&*res.first);
     }
 
-    unsigned int size() const
+    size_t size() const
     {
         return symbols.size();
     }
 
     size_t totalSize() const;
+
+    template<typename T>
+    void dump(T callback)
+    {
+        for (auto & s : symbols)
+            callback(s);
+    }
 };
 
 }
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 9df516f062ef..e1ec87d3b84c 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -43,8 +43,8 @@ class XMLWriter;
 class JSONPlaceholder;
 
 
-typedef long NixInt;
-typedef float NixFloat;
+typedef int64_t NixInt;
+typedef double NixFloat;
 
 /* External values must descend from ExternalValueBase, so that
  * type-agnostic nix functions (e.g. showType) can be implemented
@@ -128,7 +128,7 @@ struct Value
         const char * path;
         Bindings * attrs;
         struct {
-            unsigned int size;
+            size_t size;
             Value * * elems;
         } bigList;
         Value * smallList[2];
@@ -166,7 +166,7 @@ struct Value
         return type == tList1 || type == tList2 ? smallList : bigList.elems;
     }
 
-    unsigned int listSize() const
+    size_t listSize() const
     {
         return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
     }
diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index bcc05c2cdad6..4c35a4199590 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -29,14 +29,14 @@ MixCommonArgs::MixCommonArgs(const string & programName)
         .arity(2)
         .handler([](std::vector<std::string> ss) {
             try {
-                settings.set(ss[0], ss[1]);
+                globalConfig.set(ss[0], ss[1]);
             } catch (UsageError & e) {
                 warn(e.what());
             }
         });
 
     std::string cat = "config";
-    settings.convertToArgs(*this, cat);
+    globalConfig.convertToArgs(*this, cat);
 
     // Backward compatibility hack: nix-env already had a --system flag.
     if (programName == "nix-env") longFlags.erase("system");
diff --git a/src/libmain/local.mk b/src/libmain/local.mk
index f1fd3eb72424..0c80f5a0a037 100644
--- a/src/libmain/local.mk
+++ b/src/libmain/local.mk
@@ -8,7 +8,7 @@ libmain_SOURCES := $(wildcard $(d)/*.cc)
 
 libmain_LDFLAGS = $(OPENSSL_LIBS)
 
-libmain_LIBS = libstore libutil libformat
+libmain_LIBS = libstore libutil
 
 libmain_ALLOW_UNDEFINED = 1
 
diff --git a/src/libmain/nix-main.pc.in b/src/libmain/nix-main.pc.in
index 38bc85c484eb..37b03dcd42c0 100644
--- a/src/libmain/nix-main.pc.in
+++ b/src/libmain/nix-main.pc.in
@@ -6,4 +6,4 @@ Name: Nix
 Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Libs: -L${libdir} -lnixmain
-Cflags: -I${includedir}/nix -std=c++14
+Cflags: -I${includedir}/nix -std=c++17
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 91a4eaf922a6..4ed34e54dc55 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -109,7 +109,7 @@ void initNix()
     opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks());
     CRYPTO_set_locking_callback(opensslLockCallback);
 
-    settings.loadConfFile();
+    loadConfFile();
 
     startSignalHandlerThread();
 
diff --git a/src/libmain/stack.cc b/src/libmain/stack.cc
index cc0eea68fca3..e6224de7d28f 100644
--- a/src/libmain/stack.cc
+++ b/src/libmain/stack.cc
@@ -30,7 +30,7 @@ static void sigsegvHandler(int signo, siginfo_t * info, void * ctx)
         if (diff < 0) diff = -diff;
         if (diff < 4096) {
             char msg[] = "error: stack overflow (possible infinite recursion)\n";
-            [[gnu::unused]] int res = write(2, msg, strlen(msg));
+            [[gnu::unused]] auto res = write(2, msg, strlen(msg));
             _exit(1); // maybe abort instead?
         }
     }
@@ -63,7 +63,7 @@ void detectStackOverflow()
     act.sa_sigaction = sigsegvHandler;
     act.sa_flags = SA_SIGINFO | SA_ONSTACK;
     if (sigaction(SIGSEGV, &act, 0))
-        throw SysError("resetting SIGCHLD");
+        throw SysError("resetting SIGSEGV");
 #endif
 }
 
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 2e9a13e564ca..4527ee6ba660 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -54,17 +54,38 @@ void BinaryCacheStore::init()
     }
 }
 
-std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
+void BinaryCacheStore::getFile(const std::string & path,
+    Callback<std::shared_ptr<std::string>> callback)
+{
+    try {
+        callback(getFile(path));
+    } catch (...) { callback.rethrow(); }
+}
+
+void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
 {
     std::promise<std::shared_ptr<std::string>> promise;
     getFile(path,
-        [&](std::shared_ptr<std::string> result) {
-            promise.set_value(result);
-        },
-        [&](std::exception_ptr exc) {
-            promise.set_exception(exc);
-        });
-    return promise.get_future().get();
+        {[&](std::future<std::shared_ptr<std::string>> result) {
+            try {
+                promise.set_value(result.get());
+            } catch (...) {
+                promise.set_exception(std::current_exception());
+            }
+        }});
+    auto data = promise.get_future().get();
+    sink((unsigned char *) data->data(), data->size());
+}
+
+std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
+{
+    StringSink sink;
+    try {
+        getFile(path, sink);
+    } catch (NoSuchBinaryCacheFile &) {
+        return nullptr;
+    }
+    return sink.s;
 }
 
 Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
@@ -196,30 +217,30 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
 {
     auto info = queryPathInfo(storePath).cast<const NarInfo>();
 
-    auto nar = getFile(info->url);
-
-    if (!nar) throw Error(format("file '%s' missing from binary cache") % info->url);
-
-    stats.narRead++;
-    stats.narReadCompressedBytes += nar->size();
-
     uint64_t narSize = 0;
 
-    StringSource source(*nar);
-
     LambdaSink wrapperSink([&](const unsigned char * data, size_t len) {
         sink(data, len);
         narSize += len;
     });
 
-    decompress(info->compression, source, wrapperSink);
+    auto decompressor = makeDecompressionSink(info->compression, wrapperSink);
 
+    try {
+        getFile(info->url, *decompressor);
+    } catch (NoSuchBinaryCacheFile & e) {
+        throw SubstituteGone(e.what());
+    }
+
+    decompressor->finish();
+
+    stats.narRead++;
+    //stats.narReadCompressedBytes += nar->size(); // FIXME
     stats.narReadBytes += narSize;
 }
 
 void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
-        std::function<void(std::shared_ptr<ValidPathInfo>)> success,
-        std::function<void(std::exception_ptr exc)> failure)
+    Callback<std::shared_ptr<ValidPathInfo>> callback)
 {
     auto uri = getUri();
     auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo,
@@ -229,17 +250,22 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
     auto narInfoFile = narInfoFileFor(storePath);
 
     getFile(narInfoFile,
-        [=](std::shared_ptr<std::string> data) {
-            if (!data) return success(0);
+        {[=](std::future<std::shared_ptr<std::string>> fut) {
+            try {
+                auto data = fut.get();
 
-            stats.narInfoRead++;
+                if (!data) return callback(nullptr);
 
-            callSuccess(success, failure, (std::shared_ptr<ValidPathInfo>)
-                std::make_shared<NarInfo>(*this, *data, narInfoFile));
+                stats.narInfoRead++;
 
-            (void) act; // force Activity into this lambda to ensure it stays alive
-        },
-        failure);
+                callback((std::shared_ptr<ValidPathInfo>)
+                    std::make_shared<NarInfo>(*this, *data, narInfoFile));
+
+                (void) act; // force Activity into this lambda to ensure it stays alive
+            } catch (...) {
+                callback.rethrow();
+            }
+        }});
 }
 
 Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index e20b968442b7..953f3b90af46 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -38,11 +38,16 @@ public:
         const std::string & data,
         const std::string & mimeType) = 0;
 
-    /* Return the contents of the specified file, or null if it
-       doesn't exist. */
+    /* Note: subclasses must implement at least one of the two
+       following getFile() methods. */
+
+    /* Dump the contents of the specified file to a sink. */
+    virtual void getFile(const std::string & path, Sink & sink);
+
+    /* Fetch the specified file and call the specified callback with
+       the result. A subclass may implement this asynchronously. */
     virtual void getFile(const std::string & path,
-        std::function<void(std::shared_ptr<std::string>)> success,
-        std::function<void(std::exception_ptr exc)> failure) = 0;
+        Callback<std::shared_ptr<std::string>> callback);
 
     std::shared_ptr<std::string> getFile(const std::string & path);
 
@@ -67,25 +72,11 @@ public:
 
     bool isValidPathUncached(const Path & path) override;
 
-    PathSet queryAllValidPaths() override
-    { unsupported(); }
-
     void queryPathInfoUncached(const Path & path,
-        std::function<void(std::shared_ptr<ValidPathInfo>)> success,
-        std::function<void(std::exception_ptr exc)> failure) override;
-
-    void queryReferrers(const Path & path,
-        PathSet & referrers) override
-    { unsupported(); }
-
-    PathSet queryDerivationOutputs(const Path & path) override
-    { unsupported(); }
-
-    StringSet queryDerivationOutputNames(const Path & path) override
-    { unsupported(); }
+        Callback<std::shared_ptr<ValidPathInfo>> callback) override;
 
     Path queryPathFromHashPart(const string & hashPart) override
-    { unsupported(); }
+    { unsupported("queryPathFromHashPart"); }
 
     bool wantMassQuery() override { return wantMassQuery_; }
 
@@ -104,22 +95,10 @@ public:
 
     BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override
-    { unsupported(); }
+    { unsupported("buildDerivation"); }
 
     void ensurePath(const Path & path) override
-    { unsupported(); }
-
-    void addTempRoot(const Path & path) override
-    { unsupported(); }
-
-    void addIndirectRoot(const Path & path) override
-    { unsupported(); }
-
-    Roots findRoots() override
-    { unsupported(); }
-
-    void collectGarbage(const GCOptions & options, GCResults & results) override
-    { unsupported(); }
+    { unsupported("ensurePath"); }
 
     ref<FSAccessor> getFSAccessor() override;
 
@@ -131,4 +110,6 @@ public:
 
 };
 
+MakeError(NoSuchBinaryCacheFile, Error);
+
 }
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 122a754e1e1d..0bd7388097c6 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -11,6 +11,8 @@
 #include "compression.hh"
 #include "json.hh"
 #include "nar-info.hh"
+#include "parsed-derivations.hh"
+#include "machines.hh"
 
 #include <algorithm>
 #include <iostream>
@@ -20,6 +22,7 @@
 #include <future>
 #include <chrono>
 #include <regex>
+#include <queue>
 
 #include <limits.h>
 #include <sys/time.h>
@@ -29,7 +32,9 @@
 #include <sys/utsname.h>
 #include <sys/select.h>
 #include <sys/resource.h>
+#include <sys/socket.h>
 #include <fcntl.h>
+#include <netdb.h>
 #include <unistd.h>
 #include <errno.h>
 #include <cstring>
@@ -456,6 +461,28 @@ static void commonChildInit(Pipe & logPipe)
     close(fdDevNull);
 }
 
+void handleDiffHook(uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Path tmpDir)
+{
+    auto diffHook = settings.diffHook;
+    if (diffHook != "" && settings.runDiffHook) {
+        try {
+            RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir});
+            diffHookOptions.searchPath = true;
+            diffHookOptions.uid = uid;
+            diffHookOptions.gid = gid;
+            diffHookOptions.chdir = "/";
+
+            auto diffRes = runProgram(diffHookOptions);
+            if (!statusOk(diffRes.first))
+                throw ExecError(diffRes.first, fmt("diff-hook program '%1%' %2%", diffHook, statusToString(diffRes.first)));
+
+            if (diffRes.second != "")
+                printError(chomp(diffRes.second));
+        } catch (Error & error) {
+            printError("diff hook execution failed: %s", error.what());
+        }
+    }
+}
 
 //////////////////////////////////////////////////////////////////////
 
@@ -672,8 +699,10 @@ HookInstance::HookInstance()
     toHook.readSide = -1;
 
     sink = FdSink(toHook.writeSide.get());
-    for (auto & setting : settings.getSettings())
-        sink << 1 << setting.first << setting.second;
+    std::map<std::string, Config::SettingInfo> settings;
+    globalConfig.getSettings(settings);
+    for (auto & setting : settings)
+        sink << 1 << setting.first << setting.second.value;
     sink << 0;
 }
 
@@ -731,11 +760,13 @@ private:
 
     /* Whether to retry substituting the outputs after building the
        inputs. */
-    bool retrySubstitution = false;
+    bool retrySubstitution;
 
     /* The derivation stored at drvPath. */
     std::unique_ptr<BasicDerivation> drv;
 
+    std::unique_ptr<ParsedDerivation> parsedDrv;
+
     /* The remainder is state held during the build. */
 
     /* Locks on the output paths. */
@@ -842,15 +873,15 @@ private:
     BuildResult result;
 
     /* The current round, if we're building multiple times. */
-    unsigned int curRound = 1;
+    size_t curRound = 1;
 
-    unsigned int nrRounds;
+    size_t nrRounds;
 
     /* Path registration info from the previous round, if we're
        building multiple times. Since this contains the hash, it
        allows us to compare whether two rounds produced the same
        result. */
-    ValidPathInfos prevInfos;
+    std::map<Path, ValidPathInfo> prevInfos;
 
     const uid_t sandboxUid = 1000;
     const gid_t sandboxGid = 100;
@@ -873,6 +904,9 @@ public:
         Worker & worker, BuildMode buildMode = bmNormal);
     ~DerivationGoal();
 
+    /* Whether we need to perform hash rewriting if there are valid output paths. */
+    bool needsHashRewrite();
+
     void timedOut() override;
 
     string key() override
@@ -931,6 +965,11 @@ private:
        as valid. */
     void registerOutputs();
 
+    /* Check that an output meets the requirements specified by the
+       'outputChecks' attribute (or the legacy
+       '{allowed,disallowed}{References,Requisites}' attributes). */
+    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
+
     /* Open a log file and a pipe to it. */
     Path openLogFile();
 
@@ -964,6 +1003,8 @@ private:
     }
 
     void done(BuildResult::Status status, const string & msg = "");
+
+    PathSet exportReferences(PathSet storePaths);
 };
 
 
@@ -1018,6 +1059,17 @@ DerivationGoal::~DerivationGoal()
 }
 
 
+inline bool DerivationGoal::needsHashRewrite()
+{
+#if __linux__
+    return !useChroot;
+#else
+    /* Darwin requires hash rewriting even when sandboxing is enabled. */
+    return true;
+#endif
+}
+
+
 void DerivationGoal::killChild()
 {
     if (pid != -1) {
@@ -1119,6 +1171,8 @@ void DerivationGoal::haveDerivation()
 {
     trace("have derivation");
 
+    retrySubstitution = false;
+
     for (auto & i : drv->outputs)
         worker.store.addTempRoot(i.second.path);
 
@@ -1131,6 +1185,8 @@ void DerivationGoal::haveDerivation()
         return;
     }
 
+    parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
+
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
@@ -1157,7 +1213,7 @@ void DerivationGoal::outputsSubstituted()
     /*  If the substitutes form an incomplete closure, then we should
         build the dependencies of this derivation, but after that, we
         can still use the substitutes for this derivation itself. */
-    if (nrIncompleteClosure > 0 && !retrySubstitution) retrySubstitution = true;
+    if (nrIncompleteClosure > 0) retrySubstitution = true;
 
     nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
 
@@ -1167,7 +1223,7 @@ void DerivationGoal::outputsSubstituted()
         return;
     }
 
-    unsigned int nrInvalid = checkPathValidity(false, buildMode == bmRepair).size();
+    auto nrInvalid = checkPathValidity(false, buildMode == bmRepair).size();
     if (buildMode == bmNormal && nrInvalid == 0) {
         done(BuildResult::Substituted);
         return;
@@ -1387,7 +1443,7 @@ void DerivationGoal::tryToBuild()
     /* Don't do a remote build if the derivation has the attribute
        `preferLocalBuild' set.  Also, check and repair modes are only
        supported for local builds. */
-    bool buildLocally = buildMode != bmNormal || drv->willBuildLocally();
+    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally();
 
     auto started = [&]() {
         auto msg = fmt(
@@ -1633,19 +1689,13 @@ HookReply DerivationGoal::tryBuildHook()
 
     try {
 
-        /* Tell the hook about system features (beyond the system type)
-           required from the build machine.  (The hook could parse the
-           drv file itself, but this is easier.) */
-        Strings features = tokenizeString<Strings>(get(drv->env, "requiredSystemFeatures"));
-        for (auto & i : features) checkStoreName(i); /* !!! abuse */
-
         /* Send the request to the hook. */
         worker.hook->sink
             << "try"
             << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0)
             << drv->platform
             << drvPath
-            << features;
+            << parsedDrv->getRequiredSystemFeatures();
         worker.hook->sink.flush();
 
         /* Read the first line of input, which should be a word indicating
@@ -1730,22 +1780,23 @@ int childEntry(void * arg)
 }
 
 
-PathSet exportReferences(Store & store, PathSet storePaths)
+PathSet DerivationGoal::exportReferences(PathSet storePaths)
 {
     PathSet paths;
 
     for (auto storePath : storePaths) {
 
         /* Check that the store path is valid. */
-        if (!store.isInStore(storePath))
+        if (!worker.store.isInStore(storePath))
             throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'")
                 % storePath);
-        storePath = store.toStorePath(storePath);
-        if (!store.isValidPath(storePath))
-            throw BuildError(format("'exportReferencesGraph' contains an invalid path '%1%'")
-                % storePath);
 
-        store.computeFSClosure(storePath, paths);
+        storePath = worker.store.toStorePath(storePath);
+
+        if (!inputPaths.count(storePath))
+            throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", storePath);
+
+        worker.store.computeFSClosure(storePath, paths);
     }
 
     /* If there are derivations in the graph, then include their
@@ -1756,9 +1807,9 @@ PathSet exportReferences(Store & store, PathSet storePaths)
 
     for (auto & j : paths2) {
         if (isDerivation(j)) {
-            Derivation drv = store.derivationFromPath(j);
+            Derivation drv = worker.store.derivationFromPath(j);
             for (auto & k : drv.outputs)
-                store.computeFSClosure(k.second.path, paths);
+                worker.store.computeFSClosure(k.second.path, paths);
         }
     }
 
@@ -1770,35 +1821,40 @@ static std::once_flag dns_resolve_flag;
 static void preloadNSS() {
     /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
        one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
-       been loaded in the parent. So we force a download of an invalid URL to force the NSS machinery to
+       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
        load its lookup libraries in the parent before any child gets a chance to. */
     std::call_once(dns_resolve_flag, []() {
-        DownloadRequest request("http://this.pre-initializes.the.dns.resolvers.invalid");
-        request.tries = 1; // We only need to do it once, and this also suppresses an annoying warning
-        try { getDownloader()->download(request); } catch (...) {}
+        struct addrinfo *res = NULL;
+
+        if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) {
+            if (res) freeaddrinfo(res);
+        }
     });
 }
 
 void DerivationGoal::startBuilder()
 {
     /* Right platform? */
-    if (!drv->canBuildLocally()) {
-        throw Error(
-            format("a '%1%' is required to build '%3%', but I am a '%2%'")
-            % drv->platform % settings.thisSystem % drvPath);
-    }
+    if (!parsedDrv->canBuildLocally())
+        throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
+            drv->platform,
+            concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
+            drvPath,
+            settings.thisSystem,
+            concatStringsSep(", ", settings.systemFeatures));
 
     if (drv->isBuiltin())
         preloadNSS();
 
 #if __APPLE__
-    additionalSandboxProfile = get(drv->env, "__sandboxProfile");
+    additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
 #endif
 
     /* Are we doing a chroot build? */
     {
+        auto noChroot = parsedDrv->getBoolAttr("__noChroot");
         if (settings.sandboxMode == smEnabled) {
-            if (get(drv->env, "__noChroot") == "1")
+            if (noChroot)
                 throw Error(format("derivation '%1%' has '__noChroot' set, "
                     "but that's not allowed when 'sandbox' is 'true'") % drvPath);
 #if __APPLE__
@@ -1811,7 +1867,7 @@ void DerivationGoal::startBuilder()
         else if (settings.sandboxMode == smDisabled)
             useChroot = false;
         else if (settings.sandboxMode == smRelaxed)
-            useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1";
+            useChroot = !fixedOutput && !noChroot;
     }
 
     if (worker.store.storeDir != worker.store.realStoreDir) {
@@ -1862,7 +1918,7 @@ void DerivationGoal::startBuilder()
     writeStructuredAttrs();
 
     /* Handle exportReferencesGraph(), if set. */
-    if (!drv->env.count("__json")) {
+    if (!parsedDrv->getStructuredAttrs()) {
         /* The `exportReferencesGraph' feature allows the references graph
            to be passed to a builder.  This attribute should be a list of
            pairs [name1 path1 name2 path2 ...].  The references graph of
@@ -1882,7 +1938,7 @@ void DerivationGoal::startBuilder()
             /* Write closure info to <fileName>. */
             writeFile(tmpDir + "/" + fileName,
                 worker.store.makeValidityRegistration(
-                    exportReferences(worker.store, {storePath}), false, false));
+                    exportReferences({storePath}), false, false));
         }
     }
 
@@ -1927,7 +1983,7 @@ void DerivationGoal::startBuilder()
         PathSet allowedPaths = settings.allowedImpureHostPrefixes;
 
         /* This works like the above, except on a per-derivation level */
-        Strings impurePaths = tokenizeString<Strings>(get(drv->env, "__impureHostDeps"));
+        auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings());
 
         for (auto & i : impurePaths) {
             bool found = false;
@@ -1996,7 +2052,7 @@ void DerivationGoal::startBuilder()
 
         /* Create /etc/hosts with localhost entry. */
         if (!fixedOutput)
-            writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n");
+            writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
 
         /* Make the closure of the inputs available in the chroot,
            rather than the whole Nix store.  This prevents any access
@@ -2053,7 +2109,7 @@ void DerivationGoal::startBuilder()
 #endif
     }
 
-    else {
+    if (needsHashRewrite()) {
 
         if (pathExists(homeDir))
             throw Error(format("directory '%1%' exists; please remove it") % homeDir);
@@ -2295,7 +2351,7 @@ void DerivationGoal::initEnv()
        passAsFile is ignored in structure mode because it's not
        needed (attributes are not passed through the environment, so
        there is no size constraint). */
-    if (!drv->env.count("__json")) {
+    if (!parsedDrv->getStructuredAttrs()) {
 
         StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile"));
         int fileNr = 0;
@@ -2342,8 +2398,8 @@ void DerivationGoal::initEnv()
        fixed-output derivations is by definition pure (since we
        already know the cryptographic hash of the output). */
     if (fixedOutput) {
-        Strings varNames = tokenizeString<Strings>(get(drv->env, "impureEnvVars"));
-        for (auto & i : varNames) env[i] = getEnv(i);
+        for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
+            env[i] = getEnv(i);
     }
 
     /* Currently structured log messages piggyback on stderr, but we
@@ -2358,111 +2414,103 @@ static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
 
 void DerivationGoal::writeStructuredAttrs()
 {
-    auto jsonAttr = drv->env.find("__json");
-    if (jsonAttr == drv->env.end()) return;
-
-    try {
-
-        auto jsonStr = rewriteStrings(jsonAttr->second, inputRewrites);
+    auto & structuredAttrs = parsedDrv->getStructuredAttrs();
+    if (!structuredAttrs) return;
 
-        auto json = nlohmann::json::parse(jsonStr);
+    auto json = *structuredAttrs;
 
-        /* Add an "outputs" object containing the output paths. */
-        nlohmann::json outputs;
-        for (auto & i : drv->outputs)
-            outputs[i.first] = rewriteStrings(i.second.path, inputRewrites);
-        json["outputs"] = outputs;
-
-        /* Handle exportReferencesGraph. */
-        auto e = json.find("exportReferencesGraph");
-        if (e != json.end() && e->is_object()) {
-            for (auto i = e->begin(); i != e->end(); ++i) {
-                std::ostringstream str;
-                {
-                    JSONPlaceholder jsonRoot(str, true);
-                    PathSet storePaths;
-                    for (auto & p : *i)
-                        storePaths.insert(p.get<std::string>());
-                    worker.store.pathInfoToJSON(jsonRoot,
-                        exportReferences(worker.store, storePaths), false, true);
-                }
-                json[i.key()] = nlohmann::json::parse(str.str()); // urgh
+    /* Add an "outputs" object containing the output paths. */
+    nlohmann::json outputs;
+    for (auto & i : drv->outputs)
+        outputs[i.first] = rewriteStrings(i.second.path, inputRewrites);
+    json["outputs"] = outputs;
+
+    /* Handle exportReferencesGraph. */
+    auto e = json.find("exportReferencesGraph");
+    if (e != json.end() && e->is_object()) {
+        for (auto i = e->begin(); i != e->end(); ++i) {
+            std::ostringstream str;
+            {
+                JSONPlaceholder jsonRoot(str, true);
+                PathSet storePaths;
+                for (auto & p : *i)
+                    storePaths.insert(p.get<std::string>());
+                worker.store.pathInfoToJSON(jsonRoot,
+                    exportReferences(storePaths), false, true);
             }
+            json[i.key()] = nlohmann::json::parse(str.str()); // urgh
         }
+    }
 
-        writeFile(tmpDir + "/.attrs.json", json.dump());
+    writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
 
-        /* As a convenience to bash scripts, write a shell file that
-           maps all attributes that are representable in bash -
-           namely, strings, integers, nulls, Booleans, and arrays and
-           objects consisting entirely of those values. (So nested
-           arrays or objects are not supported.) */
+    /* As a convenience to bash scripts, write a shell file that
+       maps all attributes that are representable in bash -
+       namely, strings, integers, nulls, Booleans, and arrays and
+       objects consisting entirely of those values. (So nested
+       arrays or objects are not supported.) */
 
-        auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional<std::string> {
-            if (value.is_string())
-                return shellEscape(value);
+    auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
+        if (value.is_string())
+            return shellEscape(value);
 
-            if (value.is_number()) {
-                auto f = value.get<float>();
-                if (std::ceil(f) == f)
-                    return std::to_string(value.get<int>());
-            }
+        if (value.is_number()) {
+            auto f = value.get<float>();
+            if (std::ceil(f) == f)
+                return std::to_string(value.get<int>());
+        }
 
-            if (value.is_null())
-                return std::string("''");
+        if (value.is_null())
+            return std::string("''");
 
-            if (value.is_boolean())
-                return value.get<bool>() ? std::string("1") : std::string("");
+        if (value.is_boolean())
+            return value.get<bool>() ? std::string("1") : std::string("");
 
-            return {};
-        };
+        return {};
+    };
 
-        std::string jsonSh;
+    std::string jsonSh;
 
-        for (auto i = json.begin(); i != json.end(); ++i) {
+    for (auto i = json.begin(); i != json.end(); ++i) {
 
-            if (!std::regex_match(i.key(), shVarName)) continue;
+        if (!std::regex_match(i.key(), shVarName)) continue;
 
-            auto & value = i.value();
+        auto & value = i.value();
 
-            auto s = handleSimpleType(value);
-            if (s)
-                jsonSh += fmt("declare %s=%s\n", i.key(), *s);
+        auto s = handleSimpleType(value);
+        if (s)
+            jsonSh += fmt("declare %s=%s\n", i.key(), *s);
 
-            else if (value.is_array()) {
-                std::string s2;
-                bool good = true;
+        else if (value.is_array()) {
+            std::string s2;
+            bool good = true;
 
-                for (auto i = value.begin(); i != value.end(); ++i) {
-                    auto s3 = handleSimpleType(i.value());
-                    if (!s3) { good = false; break; }
-                    s2 += *s3; s2 += ' ';
-                }
-
-                if (good)
-                    jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
+            for (auto i = value.begin(); i != value.end(); ++i) {
+                auto s3 = handleSimpleType(i.value());
+                if (!s3) { good = false; break; }
+                s2 += *s3; s2 += ' ';
             }
 
-            else if (value.is_object()) {
-                std::string s2;
-                bool good = true;
+            if (good)
+                jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
+        }
 
-                for (auto i = value.begin(); i != value.end(); ++i) {
-                    auto s3 = handleSimpleType(i.value());
-                    if (!s3) { good = false; break; }
-                    s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
-                }
+        else if (value.is_object()) {
+            std::string s2;
+            bool good = true;
 
-                if (good)
-                    jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
+            for (auto i = value.begin(); i != value.end(); ++i) {
+                auto s3 = handleSimpleType(i.value());
+                if (!s3) { good = false; break; }
+                s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
             }
-        }
 
-        writeFile(tmpDir + "/.attrs.sh", jsonSh);
-
-    } catch (std::exception & e) {
-        throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what());
+            if (good)
+                jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
+        }
     }
+
+    writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
 }
 
 
@@ -2488,14 +2536,18 @@ void setupSeccomp()
         seccomp_release(ctx);
     });
 
-    if (settings.thisSystem == "x86_64-linux" &&
+    if (nativeSystem == "x86_64-linux" &&
         seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
         throw SysError("unable to add 32-bit seccomp architecture");
 
-    if (settings.thisSystem == "x86_64-linux" &&
+    if (nativeSystem == "x86_64-linux" &&
         seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0)
         throw SysError("unable to add X32 seccomp architecture");
 
+    if (nativeSystem == "aarch64-linux" &&
+        seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
+        printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
+
     /* Prevent builders from creating setuid/setgid binaries. */
     for (int perm : { S_ISUID, S_ISGID }) {
         if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,
@@ -2613,7 +2665,7 @@ void DerivationGoal::runChild()
                 createDirs(chrootRootDir + "/dev/shm");
                 createDirs(chrootRootDir + "/dev/pts");
                 ss.push_back("/dev/full");
-                if (pathExists("/dev/kvm"))
+                if (settings.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
                     ss.push_back("/dev/kvm");
                 ss.push_back("/dev/null");
                 ss.push_back("/dev/random");
@@ -2696,8 +2748,8 @@ void DerivationGoal::runChild()
                 } else {
                     if (errno != EINVAL)
                         throw SysError("mounting /dev/pts");
-                    doBind("/dev/pts", "/dev/pts");
-                    doBind("/dev/ptmx", "/dev/ptmx");
+                    doBind("/dev/pts", chrootRootDir + "/dev/pts");
+                    doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx");
                 }
             }
 
@@ -2857,6 +2909,10 @@ void DerivationGoal::runChild()
                 for (auto & i : missingPaths) {
                     sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str();
                 }
+                /* Also add redirected outputs to the chroot */
+                for (auto & i : redirectedOutputs) {
+                    sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.second.c_str()).str();
+                }
                 sandboxProfile += ")\n";
 
                 /* Our inputs (transitive dependencies and any impurities computed above)
@@ -2902,7 +2958,7 @@ void DerivationGoal::runChild()
 
             writeFile(sandboxFile, sandboxProfile);
 
-            bool allowLocalNetworking = get(drv->env, "__darwinAllowLocalNetworking") == "1";
+            bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
 
             /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
                to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
@@ -2974,10 +3030,9 @@ void DerivationGoal::runChild()
 /* Parse a list of reference specifiers.  Each element must either be
    a store path, or the symbolic name of the output of the derivation
    (such as `out'). */
-PathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, string attr)
+PathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, const Strings & paths)
 {
     PathSet result;
-    Paths paths = tokenizeString<Paths>(attr);
     for (auto & i : paths) {
         if (store.isStorePath(i))
             result.insert(i);
@@ -3002,7 +3057,7 @@ void DerivationGoal::registerOutputs()
         if (allValid) return;
     }
 
-    ValidPathInfos infos;
+    std::map<std::string, ValidPathInfo> infos;
 
     /* Set of inodes seen during calls to canonicalisePathMetaData()
        for this build's outputs.  This needs to be shared between
@@ -3010,8 +3065,7 @@ void DerivationGoal::registerOutputs()
     InodesSeen inodesSeen;
 
     Path checkSuffix = ".check";
-    bool runDiffHook = settings.runDiffHook;
-    bool keepPreviousRound = settings.keepFailed || runDiffHook;
+    bool keepPreviousRound = settings.keepFailed || settings.runDiffHook;
 
     std::exception_ptr delayedException;
 
@@ -3036,7 +3090,9 @@ void DerivationGoal::registerOutputs()
                         throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path);
             }
             if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path);
-        } else {
+        }
+
+        if (needsHashRewrite()) {
             Path redirected = redirectedOutputs[path];
             if (buildMode == bmRepair
                 && redirectedBadOutputs.find(path) != redirectedBadOutputs.end()
@@ -3106,15 +3162,15 @@ void DerivationGoal::registerOutputs()
                the derivation to its content-addressed location. */
             Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath);
 
-            Path dest = worker.store.makeFixedOutputPath(recursive, h2, drv->env["name"]);
+            Path dest = worker.store.makeFixedOutputPath(recursive, h2, storePathToName(path));
 
             if (h != h2) {
 
                 /* Throw an error after registering the path as
                    valid. */
                 delayedException = std::make_exception_ptr(
-                    BuildError("fixed-output derivation produced path '%s' with %s hash '%s' instead of the expected hash '%s'",
-                        dest, printHashType(h.type), printHash16or32(h2), printHash16or32(h)));
+                    BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
+                        dest, h.to_string(), h2.to_string()));
 
                 Path actualDest = worker.store.toRealPath(dest);
 
@@ -3154,11 +3210,17 @@ void DerivationGoal::registerOutputs()
             if (!worker.store.isValidPath(path)) continue;
             auto info = *worker.store.queryPathInfo(path);
             if (hash.first != info.narHash) {
-                if (settings.keepFailed) {
+                if (settings.runDiffHook || settings.keepFailed) {
                     Path dst = worker.store.toRealPath(path + checkSuffix);
                     deletePath(dst);
                     if (rename(actualPath.c_str(), dst.c_str()))
                         throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst);
+
+                    handleDiffHook(
+                        buildUser ? buildUser->getUID() : getuid(),
+                        buildUser ? buildUser->getGID() : getgid(),
+                        path, dst, drvPath, tmpDir);
+
                     throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'")
                         % drvPath % path % dst);
                 } else
@@ -3187,48 +3249,6 @@ void DerivationGoal::registerOutputs()
                 debug(format("referenced input: '%1%'") % i);
         }
 
-        /* Enforce `allowedReferences' and friends. */
-        auto checkRefs = [&](const string & attrName, bool allowed, bool recursive) {
-            if (drv->env.find(attrName) == drv->env.end()) return;
-
-            PathSet spec = parseReferenceSpecifiers(worker.store, *drv, get(drv->env, attrName));
-
-            PathSet used;
-            if (recursive) {
-                /* Our requisites are the union of the closures of our references. */
-                for (auto & i : references)
-                    /* Don't call computeFSClosure on ourselves. */
-                    if (path != i)
-                        worker.store.computeFSClosure(i, used);
-            } else
-                used = references;
-
-            PathSet badPaths;
-
-            for (auto & i : used)
-                if (allowed) {
-                    if (spec.find(i) == spec.end())
-                        badPaths.insert(i);
-                } else {
-                    if (spec.find(i) != spec.end())
-                        badPaths.insert(i);
-                }
-
-            if (!badPaths.empty()) {
-                string badPathsStr;
-                for (auto & i : badPaths) {
-                    badPathsStr += "\n\t";
-                    badPathsStr += i;
-                }
-                throw BuildError(format("output '%1%' is not allowed to refer to the following paths:%2%") % actualPath % badPathsStr);
-            }
-        };
-
-        checkRefs("allowedReferences", true, false);
-        checkRefs("allowedRequisites", true, true);
-        checkRefs("disallowedReferences", false, false);
-        checkRefs("disallowedRequisites", false, true);
-
         if (curRound == nrRounds) {
             worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences()
             worker.markContentsGood(path);
@@ -3242,11 +3262,16 @@ void DerivationGoal::registerOutputs()
         info.ultimate = true;
         worker.store.signPathInfo(info);
 
-        infos.push_back(info);
+        if (!info.references.empty()) info.ca.clear();
+
+        infos[i.first] = info;
     }
 
     if (buildMode == bmCheck) return;
 
+    /* Apply output checks. */
+    checkOutputs(infos);
+
     /* Compare the result with the previous round, and report which
        path is different, if any.*/
     if (curRound > 1 && prevInfos != infos) {
@@ -3254,22 +3279,16 @@ void DerivationGoal::registerOutputs()
         for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
             if (!(*i == *j)) {
                 result.isNonDeterministic = true;
-                Path prev = i->path + checkSuffix;
+                Path prev = i->second.path + checkSuffix;
                 bool prevExists = keepPreviousRound && pathExists(prev);
                 auto msg = prevExists
-                    ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->path, drvPath, prev)
-                    : fmt("output '%1%' of '%2%' differs from previous round", i->path, drvPath);
-
-                auto diffHook = settings.diffHook;
-                if (prevExists && diffHook != "" && runDiffHook) {
-                    try {
-                        auto diff = runProgram(diffHook, true, {prev, i->path});
-                        if (diff != "")
-                            printError(chomp(diff));
-                    } catch (Error & error) {
-                        printError("diff hook execution failed: %s", error.what());
-                    }
-                }
+                    ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->second.path, drvPath, prev)
+                    : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath);
+
+                handleDiffHook(
+                    buildUser ? buildUser->getUID() : getuid(),
+                    buildUser ? buildUser->getGID() : getgid(),
+                    prev, i->second.path, drvPath, tmpDir);
 
                 if (settings.enforceDeterminism)
                     throw NotDeterministic(msg);
@@ -3308,7 +3327,11 @@ void DerivationGoal::registerOutputs()
     /* Register each output path as valid, and register the sets of
        paths referenced by each of them.  If there are cycles in the
        outputs, this will fail. */
-    worker.store.registerValidPaths(infos);
+    {
+        ValidPathInfos infos2;
+        for (auto & i : infos) infos2.push_back(i.second);
+        worker.store.registerValidPaths(infos2);
+    }
 
     /* In case of a fixed-output derivation hash mismatch, throw an
        exception now that we have registered the output as valid. */
@@ -3317,6 +3340,158 @@ void DerivationGoal::registerOutputs()
 }
 
 
+void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
+{
+    std::map<Path, const ValidPathInfo &> outputsByPath;
+    for (auto & output : outputs)
+        outputsByPath.emplace(output.second.path, output.second);
+
+    for (auto & output : outputs) {
+        auto & outputName = output.first;
+        auto & info = output.second;
+
+        struct Checks
+        {
+            bool ignoreSelfRefs = false;
+            std::optional<uint64_t> maxSize, maxClosureSize;
+            std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
+        };
+
+        /* Compute the closure and closure size of some output. This
+           is slightly tricky because some of its references (namely
+           other outputs) may not be valid yet. */
+        auto getClosure = [&](const Path & path)
+        {
+            uint64_t closureSize = 0;
+            PathSet pathsDone;
+            std::queue<Path> pathsLeft;
+            pathsLeft.push(path);
+
+            while (!pathsLeft.empty()) {
+                auto path = pathsLeft.front();
+                pathsLeft.pop();
+                if (!pathsDone.insert(path).second) continue;
+
+                auto i = outputsByPath.find(path);
+                if (i != outputsByPath.end()) {
+                    closureSize += i->second.narSize;
+                    for (auto & ref : i->second.references)
+                        pathsLeft.push(ref);
+                } else {
+                    auto info = worker.store.queryPathInfo(path);
+                    closureSize += info->narSize;
+                    for (auto & ref : info->references)
+                        pathsLeft.push(ref);
+                }
+            }
+
+            return std::make_pair(pathsDone, closureSize);
+        };
+
+        auto applyChecks = [&](const Checks & checks)
+        {
+            if (checks.maxSize && info.narSize > *checks.maxSize)
+                throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
+                    info.path, info.narSize, *checks.maxSize);
+
+            if (checks.maxClosureSize) {
+                uint64_t closureSize = getClosure(info.path).second;
+                if (closureSize > *checks.maxClosureSize)
+                    throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes",
+                        info.path, closureSize, *checks.maxClosureSize);
+            }
+
+            auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive)
+            {
+                if (!value) return;
+
+                PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value);
+
+                PathSet used = recursive ? getClosure(info.path).first : info.references;
+
+                if (recursive && checks.ignoreSelfRefs)
+                    used.erase(info.path);
+
+                PathSet badPaths;
+
+                for (auto & i : used)
+                    if (allowed) {
+                        if (!spec.count(i))
+                            badPaths.insert(i);
+                    } else {
+                        if (spec.count(i))
+                            badPaths.insert(i);
+                    }
+
+                if (!badPaths.empty()) {
+                    string badPathsStr;
+                    for (auto & i : badPaths) {
+                        badPathsStr += "\n  ";
+                        badPathsStr += i;
+                    }
+                    throw BuildError("output '%s' is not allowed to refer to the following paths:%s", info.path, badPathsStr);
+                }
+            };
+
+            checkRefs(checks.allowedReferences, true, false);
+            checkRefs(checks.allowedRequisites, true, true);
+            checkRefs(checks.disallowedReferences, false, false);
+            checkRefs(checks.disallowedRequisites, false, true);
+        };
+
+        if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
+            auto outputChecks = structuredAttrs->find("outputChecks");
+            if (outputChecks != structuredAttrs->end()) {
+                auto output = outputChecks->find(outputName);
+
+                if (output != outputChecks->end()) {
+                    Checks checks;
+
+                    auto maxSize = output->find("maxSize");
+                    if (maxSize != output->end())
+                        checks.maxSize = maxSize->get<uint64_t>();
+
+                    auto maxClosureSize = output->find("maxClosureSize");
+                    if (maxClosureSize != output->end())
+                        checks.maxClosureSize = maxClosureSize->get<uint64_t>();
+
+                    auto get = [&](const std::string & name) -> std::optional<Strings> {
+                        auto i = output->find(name);
+                        if (i != output->end()) {
+                            Strings res;
+                            for (auto j = i->begin(); j != i->end(); ++j) {
+                                if (!j->is_string())
+                                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
+                                res.push_back(j->get<std::string>());
+                            }
+                            checks.disallowedRequisites = res;
+                            return res;
+                        }
+                        return {};
+                    };
+
+                    checks.allowedReferences = get("allowedReferences");
+                    checks.allowedRequisites = get("allowedRequisites");
+                    checks.disallowedReferences = get("disallowedReferences");
+                    checks.disallowedRequisites = get("disallowedRequisites");
+
+                    applyChecks(checks);
+                }
+            }
+        } else {
+            // legacy non-structured-attributes case
+            Checks checks;
+            checks.ignoreSelfRefs = true;
+            checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences");
+            checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites");
+            checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences");
+            checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites");
+            applyChecks(checks);
+        }
+    }
+}
+
+
 Path DerivationGoal::openLogFile()
 {
     logSize = 0;
@@ -3513,8 +3688,8 @@ private:
     /* The current substituter. */
     std::shared_ptr<Store> sub;
 
-    /* Whether any substituter can realise this path. */
-    bool hasSubstitute;
+    /* Whether a substituter failed. */
+    bool substituterFailed = false;
 
     /* Path info returned by the substituter's query info operation. */
     std::shared_ptr<const ValidPathInfo> info;
@@ -3578,7 +3753,6 @@ public:
 
 SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair)
     : Goal(worker)
-    , hasSubstitute(false)
     , repair(repair)
 {
     this->storePath = storePath;
@@ -3642,9 +3816,9 @@ void SubstitutionGoal::tryNext()
         /* Hack: don't indicate failure if there were no substituters.
            In that case the calling derivation should just do a
            build. */
-        amDone(hasSubstitute ? ecFailed : ecNoSubstituters);
+        amDone(substituterFailed ? ecFailed : ecNoSubstituters);
 
-        if (hasSubstitute) {
+        if (substituterFailed) {
             worker.failedSubstitutions++;
             worker.updateProgress();
         }
@@ -3666,6 +3840,19 @@ void SubstitutionGoal::tryNext()
     } catch (InvalidPath &) {
         tryNext();
         return;
+    } catch (SubstituterDisabled &) {
+        if (settings.tryFallback) {
+            tryNext();
+            return;
+        }
+        throw;
+    } catch (Error & e) {
+        if (settings.tryFallback) {
+            printError(e.what());
+            tryNext();
+            return;
+        }
+        throw;
     }
 
     /* Update the total expected download size. */
@@ -3680,8 +3867,6 @@ void SubstitutionGoal::tryNext()
 
     worker.updateProgress();
 
-    hasSubstitute = true;
-
     /* Bail out early if this substituter lacks a valid
        signature. LocalStore::addToStore() also checks for this, but
        only after we've downloaded the path. */
@@ -3796,8 +3981,19 @@ void SubstitutionGoal::finished()
         state = &SubstitutionGoal::init;
         worker.waitForAWhile(shared_from_this());
         return;
-    } catch (Error & e) {
-        printError(e.msg());
+    } catch (std::exception & e) {
+        printError(e.what());
+
+        /* Cause the parent build to fail unless --fallback is given,
+           or the substitute has disappeared. The latter case behaves
+           the same as the substitute never having existed in the
+           first place. */
+        try {
+            throw;
+        } catch (SubstituteGone &) {
+        } catch (...) {
+            substituterFailed = true;
+        }
 
         /* Try the next substitute. */
         state = &SubstitutionGoal::tryNext;
@@ -4257,6 +4453,11 @@ static void primeCache(Store & store, const PathSet & paths)
     PathSet willBuild, willSubstitute, unknown;
     unsigned long long downloadSize, narSize;
     store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
+
+    if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
+        throw Error(
+            "%d derivations need to be built, but neither local builds ('--max-jobs') "
+            "nor remote builds ('--builders') are enabled", willBuild.size());
 }
 
 
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index 4ca4a838e3c4..92aec63a0379 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -22,52 +22,56 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         return i->second;
     };
 
-    auto fetch = [&](const string & url) {
-        /* No need to do TLS verification, because we check the hash of
-           the result anyway. */
-        DownloadRequest request(url);
-        request.verifyTLS = false;
-        request.decompress = false;
+    Path storePath = getAttr("out");
+    auto mainUrl = getAttr("url");
+    bool unpack = get(drv.env, "unpack", "") == "1";
 
-        /* Note: have to use a fresh downloader here because we're in
-           a forked process. */
-        auto data = makeDownloader()->download(request);
-        assert(data.data);
+    /* Note: have to use a fresh downloader here because we're in
+       a forked process. */
+    auto downloader = makeDownloader();
 
-        return data.data;
-    };
+    auto fetch = [&](const std::string & url) {
+
+        auto source = sinkToSource([&](Sink & sink) {
+
+            /* No need to do TLS verification, because we check the hash of
+               the result anyway. */
+            DownloadRequest request(url);
+            request.verifyTLS = false;
+            request.decompress = false;
+
+            auto decompressor = makeDecompressionSink(
+                unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
+            downloader->download(std::move(request), *decompressor);
+            decompressor->finish();
+        });
+
+        if (unpack)
+            restorePath(storePath, *source);
+        else
+            writeFile(storePath, *source);
 
-    std::shared_ptr<std::string> data;
+        auto executable = drv.env.find("executable");
+        if (executable != drv.env.end() && executable->second == "1") {
+            if (chmod(storePath.c_str(), 0755) == -1)
+                throw SysError(format("making '%1%' executable") % storePath);
+        }
+    };
 
+    /* Try the hashed mirrors first. */
     if (getAttr("outputHashMode") == "flat")
         for (auto hashedMirror : settings.hashedMirrors.get())
             try {
                 if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
                 auto ht = parseHashType(getAttr("outputHashAlgo"));
-                data = fetch(hashedMirror + printHashType(ht) + "/" + Hash(getAttr("outputHash"), ht).to_string(Base16, false));
-                break;
+                fetch(hashedMirror + printHashType(ht) + "/" + Hash(getAttr("outputHash"), ht).to_string(Base16, false));
+                return;
             } catch (Error & e) {
                 debug(e.what());
             }
 
-    if (!data) data = fetch(getAttr("url"));
-
-    Path storePath = getAttr("out");
-
-    auto unpack = drv.env.find("unpack");
-    if (unpack != drv.env.end() && unpack->second == "1") {
-        if (string(*data, 0, 6) == string("\xfd" "7zXZ\0", 6))
-            data = decompress("xz", *data);
-        StringSource source(*data);
-        restorePath(storePath, source);
-    } else
-        writeFile(storePath, *data);
-
-    auto executable = drv.env.find("executable");
-    if (executable != drv.env.end() && executable->second == "1") {
-        if (chmod(storePath.c_str(), 0755) == -1)
-            throw SysError(format("making '%1%' executable") % storePath);
-    }
+    /* Otherwise try the specified URL. */
+    fetch(mainUrl);
 }
 
 }
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index a0a0d78b7d30..3961126fff9c 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -36,12 +36,6 @@ Path BasicDerivation::findOutput(const string & id) const
 }
 
 
-bool BasicDerivation::willBuildLocally() const
-{
-    return get(env, "preferLocalBuild") == "1" && canBuildLocally();
-}
-
-
 bool BasicDerivation::substitutesAllowed() const
 {
     return get(env, "allowSubstitutes", "1") == "1";
@@ -54,22 +48,6 @@ bool BasicDerivation::isBuiltin() const
 }
 
 
-bool BasicDerivation::canBuildLocally() const
-{
-    return platform == settings.thisSystem
-        || isBuiltin()
-#if __linux__
-        || (platform == "i686-linux" && settings.thisSystem == "x86_64-linux")
-        || (platform == "armv6l-linux" && settings.thisSystem == "armv7l-linux")
-        || (platform == "armv5tel-linux" && (settings.thisSystem == "armv7l-linux" || settings.thisSystem == "armv6l-linux"))
-#elif __FreeBSD__
-        || (platform == "i686-linux" && settings.thisSystem == "x86_64-freebsd")
-        || (platform == "i686-linux" && settings.thisSystem == "i686-freebsd")
-#endif
-        ;
-}
-
-
 Path writeDerivation(ref<Store> store,
     const Derivation & drv, const string & name, RepairFlag repair)
 {
@@ -350,7 +328,7 @@ Hash hashDerivationModulo(Store & store, Derivation drv)
         Hash h = drvHashes[i.first];
         if (!h) {
             assert(store.isValidPath(i.first));
-            Derivation drv2 = readDerivation(i.first);
+            Derivation drv2 = readDerivation(store.toRealPath(i.first));
             h = hashDerivationModulo(store, drv2);
             drvHashes[i.first] = h;
         }
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 7b97730d3bf2..9753e796db5f 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -56,14 +56,10 @@ struct BasicDerivation
        the given derivation. */
     Path findOutput(const string & id) const;
 
-    bool willBuildLocally() const;
-
     bool substitutesAllowed() const;
 
     bool isBuiltin() const;
 
-    bool canBuildLocally() const;
-
     /* Return true iff this is a fixed-output derivation. */
     bool isFixedOutput() const;
 
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 4d7f5690192d..22382ab1d6e8 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -7,6 +7,7 @@
 #include "s3.hh"
 #include "compression.hh"
 #include "pathlocks.hh"
+#include "finally.hh"
 
 #ifdef ENABLE_S3
 #include <aws/core/client/ClientConfiguration.h>
@@ -29,12 +30,25 @@ using namespace std::string_literals;
 
 namespace nix {
 
-double getTime()
+struct DownloadSettings : Config
 {
-    struct timeval tv;
-    gettimeofday(&tv, 0);
-    return tv.tv_sec + (tv.tv_usec / 1000000.0);
-}
+    Setting<bool> enableHttp2{this, true, "http2",
+        "Whether to enable HTTP/2 support."};
+
+    Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix",
+        "String appended to the user agent in HTTP requests."};
+
+    Setting<size_t> httpConnections{this, 25, "http-connections",
+        "Number of parallel HTTP connections.",
+        {"binary-caches-parallel-connections"}};
+
+    Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
+        "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."};
+};
+
+static DownloadSettings downloadSettings;
+
+static GlobalConfig::Register r1(&downloadSettings);
 
 std::string resolveUri(const std::string & uri)
 {
@@ -44,16 +58,6 @@ std::string resolveUri(const std::string & uri)
         return uri;
 }
 
-ref<std::string> decodeContent(const std::string & encoding, ref<std::string> data)
-{
-    if (encoding == "")
-        return data;
-    else if (encoding == "br")
-        return decompress(encoding, *data);
-    else
-        throw Error("unsupported Content-Encoding '%s'", encoding);
-}
-
 struct CurlDownloader : public Downloader
 {
     CURLM * curlm = 0;
@@ -61,8 +65,6 @@ struct CurlDownloader : public Downloader
     std::random_device rd;
     std::mt19937 mt19937;
 
-    bool enableHttp2;
-
     struct DownloadItem : public std::enable_shared_from_this<DownloadItem>
     {
         CurlDownloader & downloader;
@@ -70,8 +72,7 @@ struct CurlDownloader : public Downloader
         DownloadResult result;
         Activity act;
         bool done = false; // whether either the success or failure function has been called
-        std::function<void(const DownloadResult &)> success;
-        std::function<void(std::exception_ptr exc)> failure;
+        Callback<DownloadResult> callback;
         CURL * req = 0;
         bool active = false; // whether the handle has been added to the multi object
         std::string status;
@@ -86,10 +87,21 @@ struct CurlDownloader : public Downloader
 
         std::string encoding;
 
-        DownloadItem(CurlDownloader & downloader, const DownloadRequest & request)
+        DownloadItem(CurlDownloader & downloader,
+            const DownloadRequest & request,
+            Callback<DownloadResult> callback)
             : downloader(downloader)
             , request(request)
-            , act(*logger, lvlTalkative, actDownload, fmt("downloading '%s'", request.uri), {request.uri}, request.parentAct)
+            , act(*logger, lvlTalkative, actDownload,
+                fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
+                {request.uri}, request.parentAct)
+            , callback(callback)
+            , finalSink([this](const unsigned char * data, size_t len) {
+                if (this->request.dataCallback)
+                    this->request.dataCallback((char *) data, len);
+                else
+                    this->result.data->append((char *) data, len);
+              })
         {
             if (!request.expectedETag.empty())
                 requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
@@ -113,19 +125,40 @@ struct CurlDownloader : public Downloader
             }
         }
 
-        template<class T>
-        void fail(const T & e)
+        void failEx(std::exception_ptr ex)
         {
             assert(!done);
             done = true;
-            callFailure(failure, std::make_exception_ptr(e));
+            callback.rethrow(ex);
         }
 
+        template<class T>
+        void fail(const T & e)
+        {
+            failEx(std::make_exception_ptr(e));
+        }
+
+        LambdaSink finalSink;
+        std::shared_ptr<CompressionSink> decompressionSink;
+
+        std::exception_ptr writeException;
+
         size_t writeCallback(void * contents, size_t size, size_t nmemb)
         {
-            size_t realSize = size * nmemb;
-            result.data->append((char *) contents, realSize);
-            return realSize;
+            try {
+                size_t realSize = size * nmemb;
+                result.bodySize += realSize;
+
+                if (!decompressionSink)
+                    decompressionSink = makeDecompressionSink(encoding, finalSink);
+
+                (*decompressionSink)((unsigned char *) contents, realSize);
+
+                return realSize;
+            } catch (...) {
+                writeException = std::current_exception();
+                return 0;
+            }
         }
 
         static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
@@ -143,6 +176,7 @@ struct CurlDownloader : public Downloader
                 auto ss = tokenizeString<vector<string>>(line, " ");
                 status = ss.size() >= 2 ? ss[1] : "";
                 result.data = std::make_shared<std::string>();
+                result.bodySize = 0;
                 encoding = "";
             } else {
                 auto i = line.find(':');
@@ -173,7 +207,11 @@ struct CurlDownloader : public Downloader
 
         int progressCallback(double dltotal, double dlnow)
         {
-            act.progress(dlnow, dltotal);
+            try {
+              act.progress(dlnow, dltotal);
+            } catch (nix::Interrupted &) {
+              assert(_isInterrupted);
+            }
             return _isInterrupted;
         }
 
@@ -190,7 +228,7 @@ struct CurlDownloader : public Downloader
         }
 
         size_t readOffset = 0;
-        int readCallback(char *buffer, size_t size, size_t nitems)
+        size_t readCallback(char *buffer, size_t size, size_t nitems)
         {
             if (readOffset == request.data->length())
                 return 0;
@@ -201,7 +239,7 @@ struct CurlDownloader : public Downloader
             return count;
         }
 
-        static int readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp)
+        static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp)
         {
             return ((DownloadItem *) userp)->readCallback(buffer, size, nitems);
         }
@@ -221,15 +259,16 @@ struct CurlDownloader : public Downloader
 
             curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str());
             curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L);
+            curl_easy_setopt(req, CURLOPT_MAXREDIRS, 10);
             curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
             curl_easy_setopt(req, CURLOPT_USERAGENT,
                 ("curl/" LIBCURL_VERSION " Nix/" + nixVersion +
-                    (settings.userAgentSuffix != "" ? " " + settings.userAgentSuffix.get() : "")).c_str());
+                    (downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str());
             #if LIBCURL_VERSION_NUM >= 0x072b00
             curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
             #endif
             #if LIBCURL_VERSION_NUM >= 0x072f00
-            if (downloader.enableHttp2)
+            if (downloadSettings.enableHttp2)
                 curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
             #endif
             curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper);
@@ -261,7 +300,7 @@ struct CurlDownloader : public Downloader
                 curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
             }
 
-            curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, settings.connectTimeout.get());
+            curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get());
 
             curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
             curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, lowSpeedTimeout);
@@ -272,6 +311,7 @@ struct CurlDownloader : public Downloader
             curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
 
             result.data = std::make_shared<std::string>();
+            result.bodySize = 0;
         }
 
         void finish(CURLcode code)
@@ -284,34 +324,40 @@ struct CurlDownloader : public Downloader
             if (effectiveUrlCStr)
                 result.effectiveUrl = effectiveUrlCStr;
 
-            debug(format("finished download of '%s'; curl status = %d, HTTP status = %d, body = %d bytes")
-                % request.uri % code % httpStatus % (result.data ? result.data->size() : 0));
+            debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes",
+                request.verb(), request.uri, code, httpStatus, result.bodySize);
+
+            if (decompressionSink)
+                decompressionSink->finish();
 
             if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) {
                 code = CURLE_OK;
                 httpStatus = 304;
             }
 
-            if (code == CURLE_OK &&
+            if (writeException)
+                failEx(writeException);
+
+            else if (code == CURLE_OK &&
                 (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */))
             {
                 result.cached = httpStatus == 304;
                 done = true;
 
                 try {
-                    if (request.decompress)
-                        result.data = decodeContent(encoding, ref<std::string>(result.data));
-                    callSuccess(success, failure, const_cast<const DownloadResult &>(result));
-                    act.progress(result.data->size(), result.data->size());
+                    act.progress(result.bodySize, result.bodySize);
+                    callback(std::move(result));
                 } catch (...) {
                     done = true;
-                    callFailure(failure, std::current_exception());
+                    callback.rethrow();
                 }
-            } else {
+            }
+
+            else {
                 // We treat most errors as transient, but won't retry when hopeless
                 Error err = Transient;
 
-                if (httpStatus == 404 || code == CURLE_FILE_COULDNT_READ_FILE) {
+                if (httpStatus == 404 || httpStatus == 410 || code == CURLE_FILE_COULDNT_READ_FILE) {
                     // The file is definitely not there
                     err = NotFound;
                 } else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) {
@@ -341,6 +387,8 @@ struct CurlDownloader : public Downloader
                         case CURLE_INTERFACE_FAILED:
                         case CURLE_UNKNOWN_OPTION:
                         case CURLE_SSL_CACERT_BADFILE:
+                        case CURLE_TOO_MANY_REDIRECTS:
+                        case CURLE_WRITE_ERROR:
                             err = Misc;
                             break;
                         default: // Shut up warnings
@@ -352,10 +400,16 @@ struct CurlDownloader : public Downloader
 
                 auto exc =
                     code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
-                    ? DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri)
+                    ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
                     : httpStatus != 0
-                      ? DownloadError(err, format("unable to download '%s': HTTP error %d (curl error: %s)") % request.uri % httpStatus % curl_easy_strerror(code))
-                      : DownloadError(err, format("unable to download '%s': %s (%d)") % request.uri % curl_easy_strerror(code) % code);
+                    ? DownloadError(err,
+                        fmt("unable to %s '%s': HTTP error %d",
+                            request.verb(), request.uri, httpStatus)
+                        + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
+                        )
+                    : DownloadError(err,
+                        fmt("unable to %s '%s': %s (%d)",
+                            request.verb(), request.uri, curl_easy_strerror(code), code));
 
                 /* If this is a transient error, then maybe retry the
                    download after a while. */
@@ -404,11 +458,9 @@ struct CurlDownloader : public Downloader
         #endif
         #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0
         curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS,
-            settings.binaryCachesParallelConnections.get());
+            downloadSettings.httpConnections.get());
         #endif
 
-        enableHttp2 = settings.enableHttp2;
-
         wakeupPipe.create();
         fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK);
 
@@ -476,10 +528,11 @@ struct CurlDownloader : public Downloader
             extraFDs[0].fd = wakeupPipe.readSide.get();
             extraFDs[0].events = CURL_WAIT_POLLIN;
             extraFDs[0].revents = 0;
+            long maxSleepTimeMs = items.empty() ? 10000 : 100;
             auto sleepTimeMs =
                 nextWakeup != std::chrono::steady_clock::time_point()
                 ? std::max(0, (int) std::chrono::duration_cast<std::chrono::milliseconds>(nextWakeup - std::chrono::steady_clock::now()).count())
-                : 10000;
+                : maxSleepTimeMs;
             vomit("download thread waiting for %d ms", sleepTimeMs);
             mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds);
             if (mc != CURLM_OK)
@@ -518,7 +571,7 @@ struct CurlDownloader : public Downloader
             }
 
             for (auto & item : incoming) {
-                debug(format("starting download of %s") % item->request.uri);
+                debug("starting %s of %s", item->request.verb(), item->request.uri);
                 item->init();
                 curl_multi_add_handle(curlm, item->req);
                 item->active = true;
@@ -547,6 +600,11 @@ struct CurlDownloader : public Downloader
 
     void enqueueItem(std::shared_ptr<DownloadItem> item)
     {
+        if (item->request.data
+            && !hasPrefix(item->request.uri, "http://")
+            && !hasPrefix(item->request.uri, "https://"))
+            throw nix::Error("uploading to '%s' is not supported", item->request.uri);
+
         {
             auto state(state_.lock());
             if (state->quit)
@@ -556,48 +614,61 @@ struct CurlDownloader : public Downloader
         writeFull(wakeupPipe.writeSide.get(), " ");
     }
 
+#ifdef ENABLE_S3
+    std::tuple<std::string, std::string, Store::Params> parseS3Uri(std::string uri)
+    {
+        auto [path, params] = splitUriAndParams(uri);
+
+        auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix
+            if (slash == std::string::npos)
+                throw nix::Error("bad S3 URI '%s'", path);
+
+        std::string bucketName(path, 5, slash - 5);
+        std::string key(path, slash + 1);
+
+        return {bucketName, key, params};
+    }
+#endif
+
     void enqueueDownload(const DownloadRequest & request,
-        std::function<void(const DownloadResult &)> success,
-        std::function<void(std::exception_ptr exc)> failure) override
+        Callback<DownloadResult> callback) override
     {
         /* Ugly hack to support s3:// URIs. */
         if (hasPrefix(request.uri, "s3://")) {
             // FIXME: do this on a worker thread
-            sync2async<DownloadResult>(success, failure, [&]() -> DownloadResult {
+            try {
 #ifdef ENABLE_S3
-                S3Helper s3Helper("", Aws::Region::US_EAST_1); // FIXME: make configurable
-                auto slash = request.uri.find('/', 5);
-                if (slash == std::string::npos)
-                    throw nix::Error("bad S3 URI '%s'", request.uri);
-                std::string bucketName(request.uri, 5, slash - 5);
-                std::string key(request.uri, slash + 1);
+                auto [bucketName, key, params] = parseS3Uri(request.uri);
+
+                std::string profile = get(params, "profile", "");
+                std::string region = get(params, "region", Aws::Region::US_EAST_1);
+                std::string scheme = get(params, "scheme", "");
+                std::string endpoint = get(params, "endpoint", "");
+
+                S3Helper s3Helper(profile, region, scheme, endpoint);
+
                 // FIXME: implement ETag
                 auto s3Res = s3Helper.getObject(bucketName, key);
                 DownloadResult res;
                 if (!s3Res.data)
                     throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
                 res.data = s3Res.data;
-                return res;
+                callback(std::move(res));
 #else
                 throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri);
 #endif
-            });
+            } catch (...) { callback.rethrow(); }
             return;
         }
 
-        auto item = std::make_shared<DownloadItem>(*this, request);
-        item->success = success;
-        item->failure = failure;
-        enqueueItem(item);
+        enqueueItem(std::make_shared<DownloadItem>(*this, request, callback));
     }
 };
 
 ref<Downloader> getDownloader()
 {
-    static std::shared_ptr<Downloader> downloader;
-    static std::once_flag downloaderCreated;
-    std::call_once(downloaderCreated, [&]() { downloader = makeDownloader(); });
-    return ref<Downloader>(downloader);
+    static ref<Downloader> downloader = makeDownloader();
+    return downloader;
 }
 
 ref<Downloader> makeDownloader()
@@ -609,8 +680,13 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest &
 {
     auto promise = std::make_shared<std::promise<DownloadResult>>();
     enqueueDownload(request,
-        [promise](const DownloadResult & result) { promise->set_value(result); },
-        [promise](std::exception_ptr exc) { promise->set_exception(exc); });
+        {[promise](std::future<DownloadResult> fut) {
+            try {
+                promise->set_value(fut.get());
+            } catch (...) {
+                promise->set_exception(std::current_exception());
+            }
+        }});
     return promise->get_future();
 }
 
@@ -619,7 +695,102 @@ DownloadResult Downloader::download(const DownloadRequest & request)
     return enqueueDownload(request).get();
 }
 
-Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl)
+void Downloader::download(DownloadRequest && request, Sink & sink)
+{
+    /* Note: we can't call 'sink' via request.dataCallback, because
+       that would cause the sink to execute on the downloader
+       thread. If 'sink' is a coroutine, this will fail. Also, if the
+       sink is expensive (e.g. one that does decompression and writing
+       to the Nix store), it would stall the download thread too much.
+       Therefore we use a buffer to communicate data between the
+       download thread and the calling thread. */
+
+    struct State {
+        bool quit = false;
+        std::exception_ptr exc;
+        std::string data;
+        std::condition_variable avail, request;
+    };
+
+    auto _state = std::make_shared<Sync<State>>();
+
+    /* In case of an exception, wake up the download thread. FIXME:
+       abort the download request. */
+    Finally finally([&]() {
+        auto state(_state->lock());
+        state->quit = true;
+        state->request.notify_one();
+    });
+
+    request.dataCallback = [_state](char * buf, size_t len) {
+
+        auto state(_state->lock());
+
+        if (state->quit) return;
+
+        /* If the buffer is full, then go to sleep until the calling
+           thread wakes us up (i.e. when it has removed data from the
+           buffer). We don't wait forever to prevent stalling the
+           download thread. (Hopefully sleeping will throttle the
+           sender.) */
+        if (state->data.size() > 1024 * 1024) {
+            debug("download buffer is full; going to sleep");
+            state.wait_for(state->request, std::chrono::seconds(10));
+        }
+
+        /* Append data to the buffer and wake up the calling
+           thread. */
+        state->data.append(buf, len);
+        state->avail.notify_one();
+    };
+
+    enqueueDownload(request,
+        {[_state](std::future<DownloadResult> fut) {
+            auto state(_state->lock());
+            state->quit = true;
+            try {
+                fut.get();
+            } catch (...) {
+                state->exc = std::current_exception();
+            }
+            state->avail.notify_one();
+            state->request.notify_one();
+        }});
+
+    while (true) {
+        checkInterrupt();
+
+        std::string chunk;
+
+        /* Grab data if available, otherwise wait for the download
+           thread to wake us up. */
+        {
+            auto state(_state->lock());
+
+            while (state->data.empty()) {
+
+                if (state->quit) {
+                    if (state->exc) std::rethrow_exception(state->exc);
+                    return;
+                }
+
+                state.wait(state->avail);
+            }
+
+            chunk = std::move(state->data);
+
+            state->request.notify_one();
+        }
+
+        /* Flush the data to the sink and wake up the download thread
+           if it's blocked on a full buffer. We don't hold the state
+           lock while doing this to prevent blocking the download
+           thread if sink() takes a long time. */
+        sink((unsigned char *) chunk.data(), chunk.size());
+    }
+}
+
+Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl)
 {
     auto url = resolveUri(url_);
 
@@ -649,7 +820,6 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
 
     string expectedETag;
 
-    int ttl = settings.tarballTtl;
     bool skip = false;
 
     if (pathExists(fileLink) && pathExists(dataFile)) {
@@ -730,7 +900,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
         Hash gotHash = unpack
             ? hashPath(expectedHash.type, store->toRealPath(storePath)).first
             : hashFile(expectedHash.type, store->toRealPath(storePath));
-        throw nix::Error("hash mismatch in file downloaded from '%s': expected hash '%s', got '%s'",
+        throw nix::Error("hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
             url, expectedHash.to_string(), gotHash.to_string());
     }
 
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index 0b8d29b21dfe..f0228f7d053a 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -2,6 +2,7 @@
 
 #include "types.hh"
 #include "hash.hh"
+#include "globals.hh"
 
 #include <string>
 #include <future>
@@ -20,9 +21,15 @@ struct DownloadRequest
     bool decompress = true;
     std::shared_ptr<std::string> data;
     std::string mimeType;
+    std::function<void(char *, size_t)> dataCallback;
 
     DownloadRequest(const std::string & uri)
         : uri(uri), parentAct(getCurActivity()) { }
+
+    std::string verb()
+    {
+        return data ? "upload" : "download";
+    }
 };
 
 struct DownloadResult
@@ -31,6 +38,7 @@ struct DownloadResult
     std::string etag;
     std::string effectiveUrl;
     std::shared_ptr<std::string> data;
+    uint64_t bodySize = 0;
 };
 
 class Store;
@@ -41,20 +49,23 @@ struct Downloader
        the download. The future may throw a DownloadError
        exception. */
     virtual void enqueueDownload(const DownloadRequest & request,
-        std::function<void(const DownloadResult &)> success,
-        std::function<void(std::exception_ptr exc)> failure) = 0;
+        Callback<DownloadResult> callback) = 0;
 
     std::future<DownloadResult> enqueueDownload(const DownloadRequest & request);
 
     /* Synchronously download a file. */
     DownloadResult download(const DownloadRequest & request);
 
+    /* Download a file, writing its data to a sink. The sink will be
+       invoked on the thread of the caller. */
+    void download(DownloadRequest && request, Sink & sink);
+
     /* Check if the specified file is already in ~/.cache/nix/tarballs
        and is more recent than โ€˜tarball-ttlโ€™ seconds. Otherwise,
        use the recorded ETag to verify if the server has a more
        recent version, and if so, download it to the Nix store. */
     Path downloadCached(ref<Store> store, const string & uri, bool unpack, string name = "",
-        const Hash & expectedHash = Hash(), string * effectiveUri = nullptr);
+        const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, int ttl = settings.tarballTtl);
 
     enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
 };
@@ -77,7 +88,4 @@ public:
 
 bool isUri(const string & s);
 
-/* Decode data according to the Content-Encoding header. */
-ref<std::string> decodeContent(const std::string & encoding, ref<std::string> data);
-
 }
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index ba49749d830a..26e2b0dca7ca 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -7,6 +7,7 @@
 #include <queue>
 #include <algorithm>
 #include <regex>
+#include <random>
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -128,8 +129,8 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
        check if the root is in a directory in or linked from the
        gcroots directory. */
     if (settings.checkRootReachability) {
-        Roots roots = findRoots();
-        if (roots.find(gcRoot) == roots.end())
+        Roots roots = findRoots(false);
+        if (roots[storePath].count(gcRoot) == 0)
             printError(
                 format(
                     "warning: '%1%' is not in a directory where the garbage collector looks for roots; "
@@ -196,10 +197,11 @@ void LocalStore::addTempRoot(const Path & path)
 }
 
 
-std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds)
-{
-    std::set<std::pair<pid_t, Path>> tempRoots;
+static std::string censored = "{censored}";
+
 
+void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
+{
     /* Read the `temproots' directory for per-process temporary root
        files. */
     for (auto & i : readDirectory(tempRootsDir)) {
@@ -249,14 +251,12 @@ std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds)
             Path root(contents, pos, end - pos);
             debug("got temporary root '%s'", root);
             assertStorePath(root);
-            tempRoots.emplace(pid, root);
+            tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid));
             pos = end + 1;
         }
 
         fds.push_back(fd); /* keep open */
     }
-
-    return tempRoots;
 }
 
 
@@ -265,7 +265,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
     auto foundRoot = [&](const Path & path, const Path & target) {
         Path storePath = toStorePath(target);
         if (isStorePath(storePath) && isValidPath(storePath))
-            roots[path] = storePath;
+            roots[storePath].emplace(path);
         else
             printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath);
     };
@@ -305,7 +305,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
         else if (type == DT_REG) {
             Path storePath = storeDir + "/" + baseNameOf(path);
             if (isStorePath(storePath) && isValidPath(storePath))
-                roots[path] = storePath;
+                roots[storePath].emplace(path);
         }
 
     }
@@ -320,44 +320,31 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
 }
 
 
-Roots LocalStore::findRootsNoTemp()
+void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
 {
-    Roots roots;
-
     /* Process direct roots in {gcroots,profiles}. */
     findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
     findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
 
-    /* Add additional roots returned by the program specified by the
-       NIX_ROOT_FINDER environment variable.  This is typically used
-       to add running programs to the set of roots (to prevent them
-       from being garbage collected). */
-    size_t n = 0;
-    for (auto & root : findRuntimeRoots())
-        roots[fmt("{memory:%d}", n++)] = root;
-
-    return roots;
+    /* Add additional roots returned by different platforms-specific
+       heuristics.  This is typically used to add running programs to
+       the set of roots (to prevent them from being garbage collected). */
+    findRuntimeRoots(roots, censor);
 }
 
 
-Roots LocalStore::findRoots()
+Roots LocalStore::findRoots(bool censor)
 {
-    Roots roots = findRootsNoTemp();
+    Roots roots;
+    findRootsNoTemp(roots, censor);
 
     FDs fds;
-    pid_t prev = -1;
-    size_t n = 0;
-    for (auto & root : readTempRoots(fds)) {
-        if (prev != root.first) n = 0;
-        prev = root.first;
-        roots[fmt("{temp:%d:%d}", root.first, n++)] = root.second;
-    }
+    findTempRoots(fds, roots, censor);
 
     return roots;
 }
 
-
-static void readProcLink(const string & file, StringSet & paths)
+static void readProcLink(const string & file, Roots & roots)
 {
     /* 64 is the starting buffer size gnu readlink uses... */
     auto bufsiz = ssize_t{64};
@@ -365,7 +352,7 @@ try_again:
     char buf[bufsiz];
     auto res = readlink(file.c_str(), buf, bufsiz);
     if (res == -1) {
-        if (errno == ENOENT || errno == EACCES)
+        if (errno == ENOENT || errno == EACCES || errno == ESRCH)
             return;
         throw SysError("reading symlink");
     }
@@ -376,8 +363,8 @@ try_again:
         goto try_again;
     }
     if (res > 0 && buf[0] == '/')
-        paths.emplace(static_cast<char *>(buf), res);
-    return;
+        roots[std::string(static_cast<char *>(buf), res)]
+            .emplace(file);
 }
 
 static string quoteRegexChars(const string & raw)
@@ -386,20 +373,20 @@ static string quoteRegexChars(const string & raw)
     return std::regex_replace(raw, specialRegex, R"(\$&)");
 }
 
-static void readFileRoots(const char * path, StringSet & paths)
+static void readFileRoots(const char * path, Roots & roots)
 {
     try {
-        paths.emplace(readFile(path));
+        roots[readFile(path)].emplace(path);
     } catch (SysError & e) {
         if (e.errNo != ENOENT && e.errNo != EACCES)
             throw;
     }
 }
 
-PathSet LocalStore::findRuntimeRoots()
+void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
 {
-    PathSet roots;
-    StringSet paths;
+    Roots unchecked;
+
     auto procDir = AutoCloseDir{opendir("/proc")};
     if (procDir) {
         struct dirent * ent;
@@ -409,10 +396,10 @@ PathSet LocalStore::findRuntimeRoots()
         while (errno = 0, ent = readdir(procDir.get())) {
             checkInterrupt();
             if (std::regex_match(ent->d_name, digitsRegex)) {
-                readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), paths);
-                readProcLink((format("/proc/%1%/cwd") % ent->d_name).str(), paths);
+                readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked);
+                readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked);
 
-                auto fdStr = (format("/proc/%1%/fd") % ent->d_name).str();
+                auto fdStr = fmt("/proc/%s/fd", ent->d_name);
                 auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
                 if (!fdDir) {
                     if (errno == ENOENT || errno == EACCES)
@@ -421,29 +408,32 @@ PathSet LocalStore::findRuntimeRoots()
                 }
                 struct dirent * fd_ent;
                 while (errno = 0, fd_ent = readdir(fdDir.get())) {
-                    if (fd_ent->d_name[0] != '.') {
-                        readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), paths);
-                    }
+                    if (fd_ent->d_name[0] != '.')
+                        readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked);
                 }
-                if (errno)
+                if (errno) {
+                    if (errno == ESRCH)
+                        continue;
                     throw SysError(format("iterating /proc/%1%/fd") % ent->d_name);
-                fdDir.reset();
-
-                auto mapLines =
-                    tokenizeString<std::vector<string>>(readFile((format("/proc/%1%/maps") % ent->d_name).str(), true), "\n");
-                for (const auto& line : mapLines) {
-                    auto match = std::smatch{};
-                    if (std::regex_match(line, match, mapRegex))
-                        paths.emplace(match[1]);
                 }
+                fdDir.reset();
 
                 try {
-                    auto envString = readFile((format("/proc/%1%/environ") % ent->d_name).str(), true);
+                    auto mapFile = fmt("/proc/%s/maps", ent->d_name);
+                    auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n");
+                    for (const auto & line : mapLines) {
+                        auto match = std::smatch{};
+                        if (std::regex_match(line, match, mapRegex))
+                            unchecked[match[1]].emplace(mapFile);
+                    }
+
+                    auto envFile = fmt("/proc/%s/environ", ent->d_name);
+                    auto envString = readFile(envFile, true);
                     auto env_end = std::sregex_iterator{};
                     for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
-                        paths.emplace(i->str());
+                        unchecked[i->str()].emplace(envFile);
                 } catch (SysError & e) {
-                    if (errno == ENOENT || errno == EACCES)
+                    if (errno == ENOENT || errno == EACCES || errno == ESRCH)
                         continue;
                     throw;
                 }
@@ -461,7 +451,7 @@ PathSet LocalStore::findRuntimeRoots()
         for (const auto & line : lsofLines) {
             std::smatch match;
             if (std::regex_match(line, match, lsofRegex))
-                paths.emplace(match[1]);
+                unchecked[match[1]].emplace("{lsof}");
         }
     } catch (ExecError & e) {
         /* lsof not installed, lsof failed */
@@ -469,21 +459,23 @@ PathSet LocalStore::findRuntimeRoots()
 #endif
 
 #if defined(__linux__)
-    readFileRoots("/proc/sys/kernel/modprobe", paths);
-    readFileRoots("/proc/sys/kernel/fbsplash", paths);
-    readFileRoots("/proc/sys/kernel/poweroff_cmd", paths);
+    readFileRoots("/proc/sys/kernel/modprobe", unchecked);
+    readFileRoots("/proc/sys/kernel/fbsplash", unchecked);
+    readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked);
 #endif
 
-    for (auto & i : paths)
-        if (isInStore(i)) {
-            Path path = toStorePath(i);
-            if (roots.find(path) == roots.end() && isStorePath(path) && isValidPath(path)) {
+    for (auto & [target, links] : unchecked) {
+        if (isInStore(target)) {
+            Path path = toStorePath(target);
+            if (isStorePath(path) && isValidPath(path)) {
                 debug(format("got additional root '%1%'") % path);
-                roots.insert(path);
+                if (censor)
+                    roots[path].insert(censored);
+                else
+                    roots[path].insert(links.begin(), links.end());
             }
         }
-
-    return roots;
+    }
 }
 
 
@@ -750,16 +742,20 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     /* Find the roots.  Since we've grabbed the GC lock, the set of
        permanent roots cannot increase now. */
     printError(format("finding garbage collector roots..."));
-    Roots rootMap = options.ignoreLiveness ? Roots() : findRootsNoTemp();
+    Roots rootMap;
+    if (!options.ignoreLiveness)
+        findRootsNoTemp(rootMap, true);
 
-    for (auto & i : rootMap) state.roots.insert(i.second);
+    for (auto & i : rootMap) state.roots.insert(i.first);
 
     /* Read the temporary roots.  This acquires read locks on all
        per-process temporary root files.  So after this point no paths
        can be added to the set of temporary roots. */
     FDs fds;
-    for (auto & root : readTempRoots(fds))
-        state.tempRoots.insert(root.second);
+    Roots tempRoots;
+    findTempRoots(fds, tempRoots, true);
+    for (auto & root : tempRoots)
+        state.tempRoots.insert(root.first);
     state.roots.insert(state.tempRoots.begin(), state.tempRoots.end());
 
     /* After this point the set of roots or temporary roots cannot
@@ -829,7 +825,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                alphabetically first (e.g. /nix/store/000...).  This
                matters when using --max-freed etc. */
             vector<Path> entries_(entries.begin(), entries.end());
-            random_shuffle(entries_.begin(), entries_.end());
+            std::mt19937 gen(1);
+            std::shuffle(entries_.begin(), entries_.end(), gen);
 
             for (auto & i : entries_)
                 tryToDelete(state, i);
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 544566e0b573..1c2c08715a14 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -28,9 +28,10 @@ namespace nix {
 
 Settings settings;
 
+static GlobalConfig::Register r1(&settings);
+
 Settings::Settings()
-    : Config({})
-    , nixPrefix(NIX_PREFIX)
+    : nixPrefix(NIX_PREFIX)
     , nixStore(canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR))))
     , nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR)))
     , nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR)))
@@ -69,25 +70,39 @@ Settings::Settings()
     allowedImpureHostPrefixes = tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES);
 }
 
-void Settings::loadConfFile()
+void loadConfFile()
 {
-    applyConfigFile(nixConfDir + "/nix.conf");
+    globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf");
 
     /* We only want to send overrides to the daemon, i.e. stuff from
        ~/.nix/nix.conf or the command line. */
-    resetOverriden();
+    globalConfig.resetOverriden();
 
-    applyConfigFile(getConfigDir() + "/nix/nix.conf");
+    auto dirs = getConfigDirs();
+    // Iterate over them in reverse so that the ones appearing first in the path take priority
+    for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) {
+        globalConfig.applyConfigFile(*dir + "/nix/nix.conf");
+    }
 }
 
-void Settings::set(const string & name, const string & value)
+unsigned int Settings::getDefaultCores()
 {
-    Config::set(name, value);
+    return std::max(1U, std::thread::hardware_concurrency());
 }
 
-unsigned int Settings::getDefaultCores()
+StringSet Settings::getDefaultSystemFeatures()
 {
-    return std::max(1U, std::thread::hardware_concurrency());
+    /* For backwards compatibility, accept some "features" that are
+       used in Nixpkgs to route builds to certain machines but don't
+       actually require anything special on the machines. */
+    StringSet features{"nixos-test", "benchmark", "big-parallel"};
+
+    #if __linux__
+    if (access("/dev/kvm", R_OK | W_OK) == 0)
+        features.insert("kvm");
+    #endif
+
+    return features;
 }
 
 const string nixVersion = PACKAGE_VERSION;
@@ -162,23 +177,11 @@ void initPlugins()
                 throw Error("could not dynamically open plugin file '%s': %s", file, dlerror());
         }
     }
-    /* We handle settings registrations here, since plugins can add settings */
-    if (RegisterSetting::settingRegistrations) {
-        for (auto & registration : *RegisterSetting::settingRegistrations)
-            settings.addSetting(registration);
-        delete RegisterSetting::settingRegistrations;
-    }
-    settings.handleUnknownSettings();
-}
 
-RegisterSetting::SettingRegistrations * RegisterSetting::settingRegistrations;
-
-RegisterSetting::RegisterSetting(AbstractSetting * s)
-{
-    if (!settingRegistrations)
-        settingRegistrations = new SettingRegistrations;
-    settingRegistrations->emplace_back(s);
+    /* Since plugins can add settings, try to re-apply previously
+       unknown settings. */
+    globalConfig.reapplyUnknownSettings();
+    globalConfig.warnUnknownSettings();
 }
 
-
 }
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 7430bbedbe44..53efc6a90fb6 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -13,26 +13,6 @@ namespace nix {
 
 typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode;
 
-extern bool useCaseHack; // FIXME
-
-struct CaseHackSetting : public BaseSetting<bool>
-{
-    CaseHackSetting(Config * options,
-        const std::string & name,
-        const std::string & description,
-        const std::set<std::string> & aliases = {})
-        : BaseSetting<bool>(useCaseHack, name, description, aliases)
-    {
-        options->addSetting(this);
-    }
-
-    void set(const std::string & str) override
-    {
-        BaseSetting<bool>::set(str);
-        nix::useCaseHack = value;
-    }
-};
-
 struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
 {
     MaxBuildJobsSetting(Config * options,
@@ -52,14 +32,12 @@ class Settings : public Config {
 
     unsigned int getDefaultCores();
 
+    StringSet getDefaultSystemFeatures();
+
 public:
 
     Settings();
 
-    void loadConfFile();
-
-    void set(const string & name, const string & value);
-
     Path nixPrefix;
 
     /* The directory where we store sources and derived files. */
@@ -104,9 +82,9 @@ public:
     /* Whether to show build log output in real time. */
     bool verboseBuild = true;
 
-    /* If verboseBuild is false, the number of lines of the tail of
-       the log to show if a build fails. */
-    size_t logLines = 10;
+    Setting<size_t> logLines{this, 10, "log-lines",
+        "If verbose-build is false, the number of lines of the tail of "
+        "the log to show if a build fails."};
 
     MaxBuildJobsSetting maxBuildJobs{this, 1, "max-jobs",
         "Maximum number of parallel build jobs. \"auto\" means use number of cores.",
@@ -217,10 +195,13 @@ public:
     Setting<bool> showTrace{this, false, "show-trace",
         "Whether to show a stack trace on evaluation errors."};
 
-    Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
-        "Whether builtin functions that allow executing native code should be enabled."};
-
-    Setting<SandboxMode> sandboxMode{this, smDisabled, "sandbox",
+    Setting<SandboxMode> sandboxMode{this,
+        #if __linux__
+          smEnabled
+        #else
+          smDisabled
+        #endif
+        , "sandbox",
         "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".",
         {"build-use-chroot", "build-use-sandbox"}};
 
@@ -232,13 +213,6 @@ public:
         "Additional paths to make available inside the build sandbox.",
         {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}};
 
-    Setting<bool> restrictEval{this, false, "restrict-eval",
-        "Whether to restrict file system access to paths in $NIX_PATH, "
-        "and network access to the URI prefixes listed in 'allowed-uris'."};
-
-    Setting<bool> pureEval{this, false, "pure-eval",
-        "Whether to restrict file system and network access to files specified by cryptographic hash."};
-
     Setting<size_t> buildRepeat{this, 0, "repeat",
         "The number of times to repeat a build in order to verify determinism.",
         {"build-repeat"}};
@@ -280,13 +254,6 @@ public:
     Setting<Strings> secretKeyFiles{this, {}, "secret-key-files",
         "Secret keys with which to sign local builds."};
 
-    Setting<size_t> binaryCachesParallelConnections{this, 25, "http-connections",
-        "Number of parallel HTTP connections.",
-        {"binary-caches-parallel-connections"}};
-
-    Setting<bool> enableHttp2{this, true, "http2",
-        "Whether to enable HTTP/2 support."};
-
     Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl",
         "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."};
 
@@ -295,6 +262,17 @@ public:
         "Nix store has a valid signature (that is, one signed using a key "
         "listed in 'trusted-public-keys'."};
 
+    Setting<StringSet> extraPlatforms{this,
+        std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{},
+        "extra-platforms",
+        "Additional platforms that can be built on the local system. "
+        "These may be supported natively (e.g. armv7 on some aarch64 CPUs "
+        "or using hacks like qemu-user."};
+
+    Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
+        "system-features",
+        "Optional features that this system implements (like \"kvm\")."};
+
     Setting<Strings> substituters{this,
         nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(),
         "substituters",
@@ -343,18 +321,6 @@ public:
     /* Path to the SSL CA file used */
     Path caFile;
 
-    Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation",
-        "Whether the evaluator allows importing the result of a derivation."};
-
-    CaseHackSetting useCaseHack{this, "use-case-hack",
-        "Whether to enable a Darwin-specific hack for dealing with file name collisions."};
-
-    Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
-        "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."};
-
-    Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix",
-        "String appended to the user agent in HTTP requests."};
-
 #if __linux__
     Setting<bool> filterSyscalls{this, true, "filter-syscalls",
             "Whether to prevent certain dangerous system calls, such as "
@@ -376,9 +342,6 @@ public:
     Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free",
         "Stop deleting garbage when free disk space is above the specified amount."};
 
-    Setting<Strings> allowedUris{this, {}, "allowed-uris",
-        "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
-
     Setting<Paths> pluginFiles{this, {}, "plugin-files",
         "Plugins to dynamically load at nix initialization time."};
 };
@@ -391,15 +354,8 @@ extern Settings settings;
    anything else */
 void initPlugins();
 
+void loadConfFile();
 
 extern const string nixVersion;
 
-struct RegisterSetting
-{
-    typedef std::vector<AbstractSetting *> SettingRegistrations;
-    static SettingRegistrations * settingRegistrations;
-    RegisterSetting(AbstractSetting * s);
-};
-
-
 }
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index b9e9cd5daba5..8da0e2f9d82a 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -13,6 +13,14 @@ private:
 
     Path cacheUri;
 
+    struct State
+    {
+        bool enabled = true;
+        std::chrono::steady_clock::time_point disabledUntil;
+    };
+
+    Sync<State> _state;
+
 public:
 
     HttpBinaryCacheStore(
@@ -46,8 +54,33 @@ public:
 
 protected:
 
+    void maybeDisable()
+    {
+        auto state(_state.lock());
+        if (state->enabled && settings.tryFallback) {
+            int t = 60;
+            printError("disabling binary cache '%s' for %s seconds", getUri(), t);
+            state->enabled = false;
+            state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t);
+        }
+    }
+
+    void checkEnabled()
+    {
+        auto state(_state.lock());
+        if (state->enabled) return;
+        if (std::chrono::steady_clock::now() > state->disabledUntil) {
+            state->enabled = true;
+            debug("re-enabling binary cache '%s'", getUri());
+            return;
+        }
+        throw SubstituterDisabled("substituter '%s' is disabled", getUri());
+    }
+
     bool fileExists(const std::string & path) override
     {
+        checkEnabled();
+
         try {
             DownloadRequest request(cacheUri + "/" + path);
             request.head = true;
@@ -59,6 +92,7 @@ protected:
                bucket is unlistable, so treat 403 as 404. */
             if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
                 return false;
+            maybeDisable();
             throw;
         }
     }
@@ -73,32 +107,51 @@ protected:
         try {
             getDownloader()->download(req);
         } catch (DownloadError & e) {
-            throw UploadToHTTP(format("uploading to HTTP binary cache at %1% not supported: %2%") % cacheUri % e.msg());
+            throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg());
         }
     }
 
-    void getFile(const std::string & path,
-        std::function<void(std::shared_ptr<std::string>)> success,
-        std::function<void(std::exception_ptr exc)> failure) override
+    DownloadRequest makeRequest(const std::string & path)
     {
         DownloadRequest request(cacheUri + "/" + path);
         request.tries = 8;
+        return request;
+    }
+
+    void getFile(const std::string & path, Sink & sink) override
+    {
+        checkEnabled();
+        auto request(makeRequest(path));
+        try {
+            getDownloader()->download(std::move(request), sink);
+        } catch (DownloadError & e) {
+            if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
+                throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
+            maybeDisable();
+            throw;
+        }
+    }
+
+    void getFile(const std::string & path,
+        Callback<std::shared_ptr<std::string>> callback) override
+    {
+        checkEnabled();
+
+        auto request(makeRequest(path));
 
         getDownloader()->enqueueDownload(request,
-            [success](const DownloadResult & result) {
-                success(result.data);
-            },
-            [success, failure](std::exception_ptr exc) {
+            {[callback, this](std::future<DownloadResult> result) {
                 try {
-                    std::rethrow_exception(exc);
+                    callback(result.get().data);
                 } catch (DownloadError & e) {
                     if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
-                        return success(0);
-                    failure(exc);
+                        return callback(std::shared_ptr<std::string>());
+                    maybeDisable();
+                    callback.rethrow();
                 } catch (...) {
-                    failure(exc);
+                    callback.rethrow();
                 }
-            });
+            }});
     }
 
 };
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 5dee25308f7f..7c9bc2b68ba8 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -17,6 +17,7 @@ struct LegacySSHStore : public Store
     const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"};
     const Setting<bool> compress{this, false, "compress", "whether to compress the connection"};
     const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
+    const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"};
 
     // Hack for getting remote build log output.
     const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
@@ -27,6 +28,7 @@ struct LegacySSHStore : public Store
         FdSink to;
         FdSource from;
         int remoteVersion;
+        bool good = true;
     };
 
     std::string host;
@@ -41,7 +43,7 @@ struct LegacySSHStore : public Store
         , connections(make_ref<Pool<Connection>>(
             std::max(1, (int) maxConnections),
             [this]() { return openConnection(); },
-            [](const ref<Connection> & r) { return true; }
+            [](const ref<Connection> & r) { return r->good; }
             ))
         , master(
             host,
@@ -56,7 +58,9 @@ struct LegacySSHStore : public Store
     ref<Connection> openConnection()
     {
         auto conn = make_ref<Connection>();
-        conn->sshConn = master.startCommand(fmt("%s --serve --write", remoteProgram));
+        conn->sshConn = master.startCommand(
+            fmt("%s --serve --write", remoteProgram)
+            + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
         conn->to = FdSink(conn->sshConn->in.get());
         conn->from = FdSource(conn->sshConn->out.get());
 
@@ -84,10 +88,9 @@ struct LegacySSHStore : public Store
     }
 
     void queryPathInfoUncached(const Path & path,
-        std::function<void(std::shared_ptr<ValidPathInfo>)> success,
-        std::function<void(std::exception_ptr exc)> failure) override
+        Callback<std::shared_ptr<ValidPathInfo>> callback) override
     {
-        sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() -> std::shared_ptr<ValidPathInfo> {
+        try {
             auto conn(connections->get());
 
             debug("querying remote host '%s' for info on '%s'", host, path);
@@ -97,7 +100,7 @@ struct LegacySSHStore : public Store
 
             auto info = std::make_shared<ValidPathInfo>();
             conn->from >> info->path;
-            if (info->path.empty()) return nullptr;
+            if (info->path.empty()) return callback(nullptr);
             assert(path == info->path);
 
             PathSet references;
@@ -116,8 +119,8 @@ struct LegacySSHStore : public Store
             auto s = readString(conn->from);
             assert(s == "");
 
-            return info;
-        });
+            callback(std::move(info));
+        } catch (...) { callback.rethrow(); }
     }
 
     void addToStore(const ValidPathInfo & info, Source & source,
@@ -128,18 +131,48 @@ struct LegacySSHStore : public Store
 
         auto conn(connections->get());
 
-        conn->to
-            << cmdImportPaths
-            << 1;
-        copyNAR(source, conn->to);
-        conn->to
-            << exportMagic
-            << info.path
-            << info.references
-            << info.deriver
-            << 0
-            << 0;
-        conn->to.flush();
+        if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) {
+
+            conn->to
+                << cmdAddToStoreNar
+                << info.path
+                << info.deriver
+                << info.narHash.to_string(Base16, false)
+                << info.references
+                << info.registrationTime
+                << info.narSize
+                << info.ultimate
+                << info.sigs
+                << info.ca;
+            try {
+                copyNAR(source, conn->to);
+            } catch (...) {
+                conn->good = false;
+                throw;
+            }
+            conn->to.flush();
+
+        } else {
+
+            conn->to
+                << cmdImportPaths
+                << 1;
+            try {
+                copyNAR(source, conn->to);
+            } catch (...) {
+                conn->good = false;
+                throw;
+            }
+            conn->to
+                << exportMagic
+                << info.path
+                << info.references
+                << info.deriver
+                << 0
+                << 0;
+            conn->to.flush();
+
+        }
 
         if (readInt(conn->from) != 1)
             throw Error("failed to add path '%s' to remote host '%s', info.path, host");
@@ -154,28 +187,17 @@ struct LegacySSHStore : public Store
         copyNAR(conn->from, sink);
     }
 
-    PathSet queryAllValidPaths() override { unsupported(); }
-
-    void queryReferrers(const Path & path, PathSet & referrers) override
-    { unsupported(); }
-
-    PathSet queryDerivationOutputs(const Path & path) override
-    { unsupported(); }
-
-    StringSet queryDerivationOutputNames(const Path & path) override
-    { unsupported(); }
-
     Path queryPathFromHashPart(const string & hashPart) override
-    { unsupported(); }
+    { unsupported("queryPathFromHashPart"); }
 
     Path addToStore(const string & name, const Path & srcPath,
         bool recursive, HashType hashAlgo,
         PathFilter & filter, RepairFlag repair) override
-    { unsupported(); }
+    { unsupported("addToStore"); }
 
     Path addTextToStore(const string & name, const string & s,
         const PathSet & references, RepairFlag repair) override
-    { unsupported(); }
+    { unsupported("addTextToStore"); }
 
     BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override
@@ -209,25 +231,7 @@ struct LegacySSHStore : public Store
     }
 
     void ensurePath(const Path & path) override
-    { unsupported(); }
-
-    void addTempRoot(const Path & path) override
-    { unsupported(); }
-
-    void addIndirectRoot(const Path & path) override
-    { unsupported(); }
-
-    Roots findRoots() override
-    { unsupported(); }
-
-    void collectGarbage(const GCOptions & options, GCResults & results) override
-    { unsupported(); }
-
-    ref<FSAccessor> getFSAccessor() override
-    { unsupported(); }
-
-    void addSignatures(const Path & storePath, const StringSet & sigs) override
-    { unsupported(); }
+    { unsupported("ensurePath"); }
 
     void computeFSClosure(const PathSet & paths,
         PathSet & out, bool flipDirection = false,
@@ -270,6 +274,12 @@ struct LegacySSHStore : public Store
     {
         auto conn(connections->get());
     }
+
+    unsigned int getProtocol() override
+    {
+        auto conn(connections->get());
+        return conn->remoteVersion;
+    }
 };
 
 static RegisterStoreImplementation regStore([](
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index 2577e90aef23..b7001795be4d 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -34,18 +34,14 @@ protected:
         const std::string & data,
         const std::string & mimeType) override;
 
-    void getFile(const std::string & path,
-        std::function<void(std::shared_ptr<std::string>)> success,
-        std::function<void(std::exception_ptr exc)> failure) override
+    void getFile(const std::string & path, Sink & sink) override
     {
-        sync2async<std::shared_ptr<std::string>>(success, failure, [&]() {
-            try {
-                return std::make_shared<std::string>(readFile(binaryCacheDir + "/" + path));
-            } catch (SysError & e) {
-                if (e.errNo == ENOENT) return std::shared_ptr<std::string>();
-                throw;
-            }
-        });
+        try {
+            readFile(binaryCacheDir + "/" + path, sink);
+        } catch (SysError & e) {
+            if (e.errNo == ENOENT)
+                throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path);
+        }
     }
 
     PathSet queryAllValidPaths() override
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index b63584f28a30..485fdd691932 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -450,7 +450,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
     ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0);
 
     if (eaSize < 0) {
-        if (errno != ENOTSUP)
+        if (errno != ENOTSUP && errno != ENODATA)
             throw SysError("querying extended attributes of '%s'", path);
     } else if (eaSize > 0) {
         std::vector<char> eaBuf(eaSize);
@@ -581,7 +581,8 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation &
 uint64_t LocalStore::addValidPath(State & state,
     const ValidPathInfo & info, bool checkOutputs)
 {
-    assert(info.ca == "" || info.isContentAddressed(*this));
+    if (info.ca != "" && !info.isContentAddressed(*this))
+        throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", info.path);
 
     state.stmtRegisterValidPath.use()
         (info.path)
@@ -628,17 +629,15 @@ uint64_t LocalStore::addValidPath(State & state,
 
 
 void LocalStore::queryPathInfoUncached(const Path & path,
-    std::function<void(std::shared_ptr<ValidPathInfo>)> success,
-    std::function<void(std::exception_ptr exc)> failure)
+    Callback<std::shared_ptr<ValidPathInfo>> callback)
 {
-    sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() {
-
+    try {
         auto info = std::make_shared<ValidPathInfo>();
         info->path = path;
 
         assertStorePath(path);
 
-        return retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() {
+        callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() {
             auto state(_state.lock());
 
             /* Get the path info. */
@@ -678,8 +677,9 @@ void LocalStore::queryPathInfoUncached(const Path & path,
                 info->references.insert(useQueryReferences.getStr(0));
 
             return info;
-        });
-    });
+        }));
+
+    } catch (...) { callback.rethrow(); }
 }
 
 
@@ -880,6 +880,12 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
                     narInfo ? narInfo->fileSize : 0,
                     info->narSize};
             } catch (InvalidPath) {
+            } catch (SubstituterDisabled) {
+            } catch (Error & e) {
+                if (settings.tryFallback)
+                    printError(e.what());
+                else
+                    throw;
             }
         }
     }
@@ -975,7 +981,8 @@ const PublicKeys & LocalStore::getPublicKeys()
 void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
     RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
-    assert(info.narHash);
+    if (!info.narHash)
+        throw Error("cannot add path '%s' because it lacks a hash", info.path);
 
     if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys()))
         throw Error("cannot add path '%s' because it lacks a valid signature", info.path);
@@ -1013,11 +1020,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
             auto hashResult = hashSink.finish();
 
             if (hashResult.first != info.narHash)
-                throw Error("hash mismatch importing path '%s'; expected hash '%s', got '%s'",
+                throw Error("hash mismatch importing path '%s';\n  wanted: %s\n  got:    %s",
                     info.path, info.narHash.to_string(), hashResult.first.to_string());
 
             if (hashResult.second != info.narSize)
-                throw Error("size mismatch importing path '%s'; expected %s, got %s",
+                throw Error("size mismatch importing path '%s';\n  wanted: %s\n  got:   %s",
                     info.path, info.narSize, hashResult.second);
 
             autoGC();
@@ -1331,6 +1338,12 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store,
 }
 
 
+unsigned int LocalStore::getProtocol()
+{
+    return PROTOCOL_VERSION;
+}
+
+
 #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL)
 
 static void makeMutable(const Path & path)
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 1209a06356f7..6b655647b031 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -127,8 +127,7 @@ public:
     PathSet queryAllValidPaths() override;
 
     void queryPathInfoUncached(const Path & path,
-        std::function<void(std::shared_ptr<ValidPathInfo>)> success,
-        std::function<void(std::exception_ptr exc)> failure) override;
+        Callback<std::shared_ptr<ValidPathInfo>> callback) override;
 
     void queryReferrers(const Path & path, PathSet & referrers) override;
 
@@ -181,11 +180,11 @@ private:
     typedef std::shared_ptr<AutoCloseFD> FDPtr;
     typedef list<FDPtr> FDs;
 
-    std::set<std::pair<pid_t, Path>> readTempRoots(FDs & fds);
+    void findTempRoots(FDs & fds, Roots & roots, bool censor);
 
 public:
 
-    Roots findRoots() override;
+    Roots findRoots(bool censor) override;
 
     void collectGarbage(const GCOptions & options, GCResults & results) override;
 
@@ -210,6 +209,8 @@ public:
 
     void registerValidPaths(const ValidPathInfos & infos);
 
+    unsigned int getProtocol() override;
+
     void vacuumDB();
 
     /* Repair the contents of the given path by redownloading it using
@@ -266,9 +267,9 @@ private:
 
     void findRoots(const Path & path, unsigned char type, Roots & roots);
 
-    Roots findRootsNoTemp();
+    void findRootsNoTemp(Roots & roots, bool censor);
 
-    PathSet findRuntimeRoots();
+    void findRuntimeRoots(Roots & roots, bool censor);
 
     void removeUnusedLinks(const GCState & state);
 
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index a7279aa3939f..89fc918c30fd 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -6,7 +6,7 @@ libstore_DIR := $(d)
 
 libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc)
 
-libstore_LIBS = libutil libformat
+libstore_LIBS = libutil
 
 libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
 ifneq ($(OS), FreeBSD)
@@ -18,7 +18,7 @@ libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb
 $(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox)))
 
 ifeq ($(ENABLE_S3), 1)
-	libstore_LDFLAGS += -laws-cpp-sdk-s3 -laws-cpp-sdk-core
+	libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core
 endif
 
 ifeq ($(OS), SunOS)
diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc
index edd03d147832..f848582dafd4 100644
--- a/src/libstore/machines.cc
+++ b/src/libstore/machines.cc
@@ -89,10 +89,11 @@ void parseMachines(const std::string & s, Machines & machines)
 
 Machines getMachines()
 {
-    Machines machines;
-
-    parseMachines(settings.builders, machines);
-
+    static auto machines = [&]() {
+        Machines machines;
+        parseMachines(settings.builders, machines);
+        return machines;
+    }();
     return machines;
 }
 
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index a82aa4e9cfa5..adcce026fa1d 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -33,9 +33,11 @@ void Store::computeFSClosure(const PathSet & startPaths,
             state->pending++;
         }
 
-        queryPathInfo(path,
-            [&, path](ref<ValidPathInfo> info) {
-                // FIXME: calls to isValidPath() should be async
+        queryPathInfo(path, {[&, path](std::future<ref<ValidPathInfo>> fut) {
+            // FIXME: calls to isValidPath() should be async
+
+            try {
+                auto info = fut.get();
 
                 if (flipDirection) {
 
@@ -75,14 +77,13 @@ void Store::computeFSClosure(const PathSet & startPaths,
                     if (!--state->pending) done.notify_one();
                 }
 
-            },
-
-            [&, path](std::exception_ptr exc) {
+            } catch (...) {
                 auto state(state_.lock());
-                if (!state->exc) state->exc = exc;
+                if (!state->exc) state->exc = std::current_exception();
                 assert(state->pending);
                 if (!--state->pending) done.notify_one();
-            });
+            };
+        }});
     };
 
     for (auto & startPath : startPaths)
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 35403e5df56f..32ad7f2b27ff 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -31,6 +31,7 @@ create table if not exists NARs (
     refs             text,
     deriver          text,
     sigs             text,
+    ca               text,
     timestamp        integer not null,
     present          integer not null,
     primary key (cache, hashPart),
@@ -72,7 +73,7 @@ public:
     {
         auto state(_state.lock());
 
-        Path dbPath = getCacheDir() + "/nix/binary-cache-v5.sqlite";
+        Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite";
         createDirs(dirOf(dbPath));
 
         state->db = SQLite(dbPath);
@@ -94,13 +95,13 @@ public:
 
         state->insertNAR.create(state->db,
             "insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, "
-            "narSize, refs, deriver, sigs, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)");
+            "narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)");
 
         state->insertMissingNAR.create(state->db,
             "insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0)");
 
         state->queryNAR.create(state->db,
-            "select * from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))");
+            "select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))");
 
         /* Periodically purge expired entries from the database. */
         retrySQLite<void>([&]() {
@@ -189,27 +190,28 @@ public:
             if (!queryNAR.next())
                 return {oUnknown, 0};
 
-            if (!queryNAR.getInt(13))
+            if (!queryNAR.getInt(0))
                 return {oInvalid, 0};
 
             auto narInfo = make_ref<NarInfo>();
 
-            auto namePart = queryNAR.getStr(2);
+            auto namePart = queryNAR.getStr(1);
             narInfo->path = cache.storeDir + "/" +
                 hashPart + (namePart.empty() ? "" : "-" + namePart);
-            narInfo->url = queryNAR.getStr(3);
-            narInfo->compression = queryNAR.getStr(4);
-            if (!queryNAR.isNull(5))
-                narInfo->fileHash = Hash(queryNAR.getStr(5));
-            narInfo->fileSize = queryNAR.getInt(6);
-            narInfo->narHash = Hash(queryNAR.getStr(7));
-            narInfo->narSize = queryNAR.getInt(8);
-            for (auto & r : tokenizeString<Strings>(queryNAR.getStr(9), " "))
+            narInfo->url = queryNAR.getStr(2);
+            narInfo->compression = queryNAR.getStr(3);
+            if (!queryNAR.isNull(4))
+                narInfo->fileHash = Hash(queryNAR.getStr(4));
+            narInfo->fileSize = queryNAR.getInt(5);
+            narInfo->narHash = Hash(queryNAR.getStr(6));
+            narInfo->narSize = queryNAR.getInt(7);
+            for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
                 narInfo->references.insert(cache.storeDir + "/" + r);
-            if (!queryNAR.isNull(10))
-                narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(10);
-            for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(11), " "))
+            if (!queryNAR.isNull(9))
+                narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(9);
+            for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
                 narInfo->sigs.insert(sig);
+            narInfo->ca = queryNAR.getStr(11);
 
             return {oValid, narInfo};
         });
@@ -243,6 +245,7 @@ public:
                     (concatStringsSep(" ", info->shortRefs()))
                     (info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "")
                     (concatStringsSep(" ", info->sigs))
+                    (info->ca)
                     (time(0)).exec();
 
             } else {
diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in
index 5cf22faadcbe..6d67b1e03808 100644
--- a/src/libstore/nix-store.pc.in
+++ b/src/libstore/nix-store.pc.in
@@ -6,4 +6,4 @@ Name: Nix
 Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Libs: -L${libdir} -lnixstore -lnixutil
-Cflags: -I${includedir}/nix -std=c++14
+Cflags: -I${includedir}/nix -std=c++17
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 7840167d7772..991512f21795 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -104,8 +104,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
        *.app/Contents/Resources/\*.lproj seem to be the only paths affected. See
        https://github.com/NixOS/nix/issues/1443 for more discussion. */
 
-    if (std::regex_search(path, std::regex("\\.app/Contents/PkgInfo$")) ||
-        std::regex_search(path, std::regex("\\.app/Contents/Resources/.+\\.lproj$")))
+    if (std::regex_search(path, std::regex("\\.app/Contents/.+$")))
     {
         debug(format("'%1%' is not allowed to be linked in macOS") % path);
         return;
diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
new file mode 100644
index 000000000000..17fde00a0167
--- /dev/null
+++ b/src/libstore/parsed-derivations.cc
@@ -0,0 +1,111 @@
+#include "parsed-derivations.hh"
+
+namespace nix {
+
+ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv)
+    : drvPath(drvPath), drv(drv)
+{
+    /* Parse the __json attribute, if any. */
+    auto jsonAttr = drv.env.find("__json");
+    if (jsonAttr != drv.env.end()) {
+        try {
+            structuredAttrs = nlohmann::json::parse(jsonAttr->second);
+        } catch (std::exception & e) {
+            throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what());
+        }
+    }
+}
+
+std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const
+{
+    if (structuredAttrs) {
+        auto i = structuredAttrs->find(name);
+        if (i == structuredAttrs->end())
+            return {};
+        else {
+            if (!i->is_string())
+                throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath);
+            return i->get<std::string>();
+        }
+    } else {
+        auto i = drv.env.find(name);
+        if (i == drv.env.end())
+            return {};
+        else
+            return i->second;
+    }
+}
+
+bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const
+{
+    if (structuredAttrs) {
+        auto i = structuredAttrs->find(name);
+        if (i == structuredAttrs->end())
+            return def;
+        else {
+            if (!i->is_boolean())
+                throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath);
+            return i->get<bool>();
+        }
+    } else {
+        auto i = drv.env.find(name);
+        if (i == drv.env.end())
+            return def;
+        else
+            return i->second == "1";
+    }
+}
+
+std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name) const
+{
+    if (structuredAttrs) {
+        auto i = structuredAttrs->find(name);
+        if (i == structuredAttrs->end())
+            return {};
+        else {
+            if (!i->is_array())
+                throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
+            Strings res;
+            for (auto j = i->begin(); j != i->end(); ++j) {
+                if (!j->is_string())
+                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
+                res.push_back(j->get<std::string>());
+            }
+            return res;
+        }
+    } else {
+        auto i = drv.env.find(name);
+        if (i == drv.env.end())
+            return {};
+        else
+            return tokenizeString<Strings>(i->second);
+    }
+}
+
+StringSet ParsedDerivation::getRequiredSystemFeatures() const
+{
+    StringSet res;
+    for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings()))
+        res.insert(i);
+    return res;
+}
+
+bool ParsedDerivation::canBuildLocally() const
+{
+    if (drv.platform != settings.thisSystem.get()
+        && !settings.extraPlatforms.get().count(drv.platform)
+        && !drv.isBuiltin())
+        return false;
+
+    for (auto & feature : getRequiredSystemFeatures())
+        if (!settings.systemFeatures.get().count(feature)) return false;
+
+    return true;
+}
+
+bool ParsedDerivation::willBuildLocally() const
+{
+    return getBoolAttr("preferLocalBuild") && canBuildLocally();
+}
+
+}
diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh
new file mode 100644
index 000000000000..ed07dc652e8d
--- /dev/null
+++ b/src/libstore/parsed-derivations.hh
@@ -0,0 +1,35 @@
+#include "derivations.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+class ParsedDerivation
+{
+    Path drvPath;
+    BasicDerivation & drv;
+    std::optional<nlohmann::json> structuredAttrs;
+
+public:
+
+    ParsedDerivation(const Path & drvPath, BasicDerivation & drv);
+
+    const std::optional<nlohmann::json> & getStructuredAttrs() const
+    {
+        return structuredAttrs;
+    }
+
+    std::optional<std::string> getStringAttr(const std::string & name) const;
+
+    bool getBoolAttr(const std::string & name, bool def = false) const;
+
+    std::optional<Strings> getStringsAttr(const std::string & name) const;
+
+    StringSet getRequiredSystemFeatures() const;
+
+    bool canBuildLocally() const;
+
+    bool willBuildLocally() const;
+};
+
+}
diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index 4a607b584506..4c6af567ae6f 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -157,6 +157,29 @@ void deleteGenerations(const Path & profile, const std::set<unsigned int> & gens
     }
 }
 
+void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun)
+{
+    PathLocks lock;
+    lockProfile(lock, profile);
+
+    int curGen;
+    bool fromCurGen = false;
+    Generations gens = findGenerations(profile, curGen);
+    for (auto i = gens.rbegin(); i != gens.rend(); ++i) {
+        if (i->number == curGen) {
+            fromCurGen = true;
+            max--;
+            continue;
+        }
+        if (fromCurGen) {
+            if (max) {
+                max--;
+                continue;
+            }
+            deleteGeneration2(profile, i->number, dryRun);
+        }
+    }
+}
 
 void deleteOldGenerations(const Path & profile, bool dryRun)
 {
diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh
index 1d4e6d3037db..5fa1533de311 100644
--- a/src/libstore/profiles.hh
+++ b/src/libstore/profiles.hh
@@ -39,6 +39,8 @@ void deleteGeneration(const Path & profile, unsigned int gen);
 
 void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun);
 
+void deleteGenerationsGreaterThan(const Path & profile, const int max, bool dryRun);
+
 void deleteOldGenerations(const Path & profile, bool dryRun);
 
 void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun);
diff --git a/src/libstore/references.cc b/src/libstore/references.cc
index ba9f18b9ca5e..5b7eb1f846af 100644
--- a/src/libstore/references.cc
+++ b/src/libstore/references.cc
@@ -13,7 +13,7 @@ namespace nix {
 static unsigned int refLength = 32; /* characters */
 
 
-static void search(const unsigned char * s, unsigned int len,
+static void search(const unsigned char * s, size_t len,
     StringSet & hashes, StringSet & seen)
 {
     static bool initialised = false;
@@ -25,7 +25,7 @@ static void search(const unsigned char * s, unsigned int len,
         initialised = true;
     }
 
-    for (unsigned int i = 0; i + refLength <= len; ) {
+    for (size_t i = 0; i + refLength <= len; ) {
         int j;
         bool match = true;
         for (j = refLength - 1; j >= 0; --j)
@@ -73,7 +73,7 @@ void RefScanSink::operator () (const unsigned char * data, size_t len)
 
     search(data, len, hashes, seen);
 
-    unsigned int tailLen = len <= refLength ? len : refLength;
+    size_t tailLen = len <= refLength ? len : refLength;
     tail =
         string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) +
         string((const char *) data + len - tailLen, tailLen);
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 080cef93d214..15faf78a526d 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -7,6 +7,7 @@
 #include "globals.hh"
 #include "derivations.hh"
 #include "pool.hh"
+#include "finally.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -160,7 +161,8 @@ void RemoteStore::initConnection(Connection & conn)
         if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11)
             conn.to << false;
 
-        conn.processStderr();
+        auto ex = conn.processStderr();
+        if (ex) std::rethrow_exception(ex);
     }
     catch (Error & e) {
         throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what());
@@ -187,28 +189,75 @@ void RemoteStore::setOptions(Connection & conn)
        << settings.useSubstitutes;
 
     if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) {
-        auto overrides = settings.getSettings(true);
+        std::map<std::string, Config::SettingInfo> overrides;
+        globalConfig.getSettings(overrides, true);
         conn.to << overrides.size();
         for (auto & i : overrides)
-            conn.to << i.first << i.second;
+            conn.to << i.first << i.second.value;
     }
 
-    conn.processStderr();
+    auto ex = conn.processStderr();
+    if (ex) std::rethrow_exception(ex);
+}
+
+
+/* A wrapper around Pool<RemoteStore::Connection>::Handle that marks
+   the connection as bad (causing it to be closed) if a non-daemon
+   exception is thrown before the handle is closed. Such an exception
+   causes a deviation from the expected protocol and therefore a
+   desynchronization between the client and daemon. */
+struct ConnectionHandle
+{
+    Pool<RemoteStore::Connection>::Handle handle;
+    bool daemonException = false;
+
+    ConnectionHandle(Pool<RemoteStore::Connection>::Handle && handle)
+        : handle(std::move(handle))
+    { }
+
+    ConnectionHandle(ConnectionHandle && h)
+        : handle(std::move(h.handle))
+    { }
+
+    ~ConnectionHandle()
+    {
+        if (!daemonException && std::uncaught_exception()) {
+            handle.markBad();
+            debug("closing daemon connection because of an exception");
+        }
+    }
+
+    RemoteStore::Connection * operator -> () { return &*handle; }
+
+    void processStderr(Sink * sink = 0, Source * source = 0)
+    {
+        auto ex = handle->processStderr(sink, source);
+        if (ex) {
+            daemonException = true;
+            std::rethrow_exception(ex);
+        }
+    }
+};
+
+
+ConnectionHandle RemoteStore::getConnection()
+{
+    return ConnectionHandle(connections->get());
 }
 
 
 bool RemoteStore::isValidPathUncached(const Path & path)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopIsValidPath << path;
-    conn->processStderr();
+    conn.processStderr();
     return readInt(conn->from);
 }
 
 
 PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
         PathSet res;
         for (auto & i : paths)
@@ -216,7 +265,7 @@ PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybe
         return res;
     } else {
         conn->to << wopQueryValidPaths << paths;
-        conn->processStderr();
+        conn.processStderr();
         return readStorePaths<PathSet>(*this, conn->from);
     }
 }
@@ -224,27 +273,27 @@ PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybe
 
 PathSet RemoteStore::queryAllValidPaths()
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopQueryAllValidPaths;
-    conn->processStderr();
+    conn.processStderr();
     return readStorePaths<PathSet>(*this, conn->from);
 }
 
 
 PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
         PathSet res;
         for (auto & i : paths) {
             conn->to << wopHasSubstitutes << i;
-            conn->processStderr();
+            conn.processStderr();
             if (readInt(conn->from)) res.insert(i);
         }
         return res;
     } else {
         conn->to << wopQuerySubstitutablePaths << paths;
-        conn->processStderr();
+        conn.processStderr();
         return readStorePaths<PathSet>(*this, conn->from);
     }
 }
@@ -255,14 +304,14 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
 {
     if (paths.empty()) return;
 
-    auto conn(connections->get());
+    auto conn(getConnection());
 
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
 
         for (auto & i : paths) {
             SubstitutablePathInfo info;
             conn->to << wopQuerySubstitutablePathInfo << i;
-            conn->processStderr();
+            conn.processStderr();
             unsigned int reply = readInt(conn->from);
             if (reply == 0) continue;
             info.deriver = readString(conn->from);
@@ -276,7 +325,7 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
     } else {
 
         conn->to << wopQuerySubstitutablePathInfos << paths;
-        conn->processStderr();
+        conn.processStderr();
         size_t count = readNum<size_t>(conn->from);
         for (size_t n = 0; n < count; n++) {
             Path path = readStorePath(*this, conn->from);
@@ -293,47 +342,49 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
 
 
 void RemoteStore::queryPathInfoUncached(const Path & path,
-    std::function<void(std::shared_ptr<ValidPathInfo>)> success,
-    std::function<void(std::exception_ptr exc)> failure)
-{
-    sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() {
-        auto conn(connections->get());
-        conn->to << wopQueryPathInfo << path;
-        try {
-            conn->processStderr();
-        } catch (Error & e) {
-            // Ugly backwards compatibility hack.
-            if (e.msg().find("is not valid") != std::string::npos)
-                throw InvalidPath(e.what());
-            throw;
-        }
-        if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
-            bool valid; conn->from >> valid;
-            if (!valid) throw InvalidPath(format("path '%s' is not valid") % path);
-        }
-        auto info = std::make_shared<ValidPathInfo>();
-        info->path = path;
-        info->deriver = readString(conn->from);
-        if (info->deriver != "") assertStorePath(info->deriver);
-        info->narHash = Hash(readString(conn->from), htSHA256);
-        info->references = readStorePaths<PathSet>(*this, conn->from);
-        conn->from >> info->registrationTime >> info->narSize;
-        if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
-            conn->from >> info->ultimate;
-            info->sigs = readStrings<StringSet>(conn->from);
-            conn->from >> info->ca;
+    Callback<std::shared_ptr<ValidPathInfo>> callback)
+{
+    try {
+        std::shared_ptr<ValidPathInfo> info;
+        {
+            auto conn(getConnection());
+            conn->to << wopQueryPathInfo << path;
+            try {
+                conn.processStderr();
+            } catch (Error & e) {
+                // Ugly backwards compatibility hack.
+                if (e.msg().find("is not valid") != std::string::npos)
+                    throw InvalidPath(e.what());
+                throw;
+            }
+            if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
+                bool valid; conn->from >> valid;
+                if (!valid) throw InvalidPath(format("path '%s' is not valid") % path);
+            }
+            info = std::make_shared<ValidPathInfo>();
+            info->path = path;
+            info->deriver = readString(conn->from);
+            if (info->deriver != "") assertStorePath(info->deriver);
+            info->narHash = Hash(readString(conn->from), htSHA256);
+            info->references = readStorePaths<PathSet>(*this, conn->from);
+            conn->from >> info->registrationTime >> info->narSize;
+            if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
+                conn->from >> info->ultimate;
+                info->sigs = readStrings<StringSet>(conn->from);
+                conn->from >> info->ca;
+            }
         }
-        return info;
-    });
+        callback(std::move(info));
+    } catch (...) { callback.rethrow(); }
 }
 
 
 void RemoteStore::queryReferrers(const Path & path,
     PathSet & referrers)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopQueryReferrers << path;
-    conn->processStderr();
+    conn.processStderr();
     PathSet referrers2 = readStorePaths<PathSet>(*this, conn->from);
     referrers.insert(referrers2.begin(), referrers2.end());
 }
@@ -341,36 +392,36 @@ void RemoteStore::queryReferrers(const Path & path,
 
 PathSet RemoteStore::queryValidDerivers(const Path & path)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopQueryValidDerivers << path;
-    conn->processStderr();
+    conn.processStderr();
     return readStorePaths<PathSet>(*this, conn->from);
 }
 
 
 PathSet RemoteStore::queryDerivationOutputs(const Path & path)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopQueryDerivationOutputs << path;
-    conn->processStderr();
+    conn.processStderr();
     return readStorePaths<PathSet>(*this, conn->from);
 }
 
 
 PathSet RemoteStore::queryDerivationOutputNames(const Path & path)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopQueryDerivationOutputNames << path;
-    conn->processStderr();
+    conn.processStderr();
     return readStrings<PathSet>(conn->from);
 }
 
 
 Path RemoteStore::queryPathFromHashPart(const string & hashPart)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopQueryPathFromHashPart << hashPart;
-    conn->processStderr();
+    conn.processStderr();
     Path path = readString(conn->from);
     if (!path.empty()) assertStorePath(path);
     return path;
@@ -380,7 +431,7 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart)
 void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
     RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
 
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) {
         conn->to << wopImportPaths;
@@ -399,7 +450,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
                 ;
         });
 
-        conn->processStderr(0, source2.get());
+        conn.processStderr(0, source2.get());
 
         auto importedPaths = readStorePaths<PathSet>(*this, conn->from);
         assert(importedPaths.size() <= 1);
@@ -411,8 +462,9 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
                  << info.references << info.registrationTime << info.narSize
                  << info.ultimate << info.sigs << info.ca
                  << repair << !checkSigs;
-        copyNAR(source, conn->to);
-        conn->processStderr();
+        bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21;
+        if (!tunnel) copyNAR(source, conn->to);
+        conn.processStderr(0, tunnel ? &source : nullptr);
     }
 }
 
@@ -422,7 +474,7 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath,
 {
     if (repair) throw Error("repairing is not supported when building through the Nix daemon");
 
-    auto conn(connections->get());
+    auto conn(getConnection());
 
     Path srcPath(absPath(_srcPath));
 
@@ -435,16 +487,18 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath,
         conn->to.written = 0;
         conn->to.warn = true;
         connections->incCapacity();
-        dumpPath(srcPath, conn->to, filter);
-        connections->decCapacity();
+        {
+            Finally cleanup([&]() { connections->decCapacity(); });
+            dumpPath(srcPath, conn->to, filter);
+        }
         conn->to.warn = false;
-        conn->processStderr();
+        conn.processStderr();
     } catch (SysError & e) {
         /* Daemon closed while we were sending the path. Probably OOM
            or I/O error. */
         if (e.errNo == EPIPE)
             try {
-                conn->processStderr();
+                conn.processStderr();
             } catch (EndOfFile & e) { }
         throw;
     }
@@ -458,17 +512,17 @@ Path RemoteStore::addTextToStore(const string & name, const string & s,
 {
     if (repair) throw Error("repairing is not supported when building through the Nix daemon");
 
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopAddTextToStore << name << s << references;
 
-    conn->processStderr();
+    conn.processStderr();
     return readStorePath(*this, conn->from);
 }
 
 
 void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopBuildPaths;
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) {
         conn->to << drvPaths;
@@ -487,7 +541,7 @@ void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
             drvPaths2.insert(string(i, 0, i.find('!')));
         conn->to << drvPaths2;
     }
-    conn->processStderr();
+    conn.processStderr();
     readInt(conn->from);
 }
 
@@ -495,9 +549,9 @@ void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
 BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv,
     BuildMode buildMode)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopBuildDerivation << drvPath << drv << buildMode;
-    conn->processStderr();
+    conn.processStderr();
     BuildResult res;
     unsigned int status;
     conn->from >> status >> res.errorMsg;
@@ -508,51 +562,51 @@ BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDeriva
 
 void RemoteStore::ensurePath(const Path & path)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopEnsurePath << path;
-    conn->processStderr();
+    conn.processStderr();
     readInt(conn->from);
 }
 
 
 void RemoteStore::addTempRoot(const Path & path)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopAddTempRoot << path;
-    conn->processStderr();
+    conn.processStderr();
     readInt(conn->from);
 }
 
 
 void RemoteStore::addIndirectRoot(const Path & path)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopAddIndirectRoot << path;
-    conn->processStderr();
+    conn.processStderr();
     readInt(conn->from);
 }
 
 
 void RemoteStore::syncWithGC()
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopSyncWithGC;
-    conn->processStderr();
+    conn.processStderr();
     readInt(conn->from);
 }
 
 
-Roots RemoteStore::findRoots()
+Roots RemoteStore::findRoots(bool censor)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopFindRoots;
-    conn->processStderr();
+    conn.processStderr();
     size_t count = readNum<size_t>(conn->from);
     Roots result;
     while (count--) {
         Path link = readString(conn->from);
         Path target = readStorePath(*this, conn->from);
-        result[link] = target;
+        result[target].emplace(link);
     }
     return result;
 }
@@ -560,7 +614,7 @@ Roots RemoteStore::findRoots()
 
 void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
 
     conn->to
         << wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness
@@ -568,7 +622,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
         /* removed options */
         << 0 << 0 << 0;
 
-    conn->processStderr();
+    conn.processStderr();
 
     results.paths = readStrings<PathSet>(conn->from);
     results.bytesFreed = readLongLong(conn->from);
@@ -583,27 +637,27 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
 
 void RemoteStore::optimiseStore()
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopOptimiseStore;
-    conn->processStderr();
+    conn.processStderr();
     readInt(conn->from);
 }
 
 
 bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopVerifyStore << checkContents << repair;
-    conn->processStderr();
+    conn.processStderr();
     return readInt(conn->from);
 }
 
 
 void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopAddSignatures << storePath << sigs;
-    conn->processStderr();
+    conn.processStderr();
     readInt(conn->from);
 }
 
@@ -613,13 +667,13 @@ void RemoteStore::queryMissing(const PathSet & targets,
     unsigned long long & downloadSize, unsigned long long & narSize)
 {
     {
-        auto conn(connections->get());
+        auto conn(getConnection());
         if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19)
             // Don't hold the connection handle in the fallback case
             // to prevent a deadlock.
             goto fallback;
         conn->to << wopQueryMissing << targets;
-        conn->processStderr();
+        conn.processStderr();
         willBuild = readStorePaths<PathSet>(*this, conn->from);
         willSubstitute = readStorePaths<PathSet>(*this, conn->from);
         unknown = readStorePaths<PathSet>(*this, conn->from);
@@ -635,7 +689,14 @@ void RemoteStore::queryMissing(const PathSet & targets,
 
 void RemoteStore::connect()
 {
+    auto conn(getConnection());
+}
+
+
+unsigned int RemoteStore::getProtocol()
+{
     auto conn(connections->get());
+    return conn->daemonVersion;
 }
 
 
@@ -672,7 +733,7 @@ static Logger::Fields readFields(Source & from)
 }
 
 
-void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
+std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source)
 {
     to.flush();
 
@@ -697,7 +758,7 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
         else if (msg == STDERR_ERROR) {
             string error = readString(from);
             unsigned int status = readInt(from);
-            throw Error(status, error);
+            return std::make_exception_ptr(Error(status, error));
         }
 
         else if (msg == STDERR_NEXT)
@@ -731,6 +792,8 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
         else
             throw Error("got unknown message type %x from Nix daemon", msg);
     }
+
+    return nullptr;
 }
 
 static std::string uriScheme = "unix://";
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 95fa59a2069d..80f18ab715d9 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -14,6 +14,7 @@ class Pid;
 struct FdSink;
 struct FdSource;
 template<typename T> class Pool;
+struct ConnectionHandle;
 
 
 /* FIXME: RemoteStore is a misnomer - should be something like
@@ -40,8 +41,7 @@ public:
     PathSet queryAllValidPaths() override;
 
     void queryPathInfoUncached(const Path & path,
-        std::function<void(std::shared_ptr<ValidPathInfo>)> success,
-        std::function<void(std::exception_ptr exc)> failure) override;
+        Callback<std::shared_ptr<ValidPathInfo>> callback) override;
 
     void queryReferrers(const Path & path, PathSet & referrers) override;
 
@@ -82,7 +82,7 @@ public:
 
     void syncWithGC() override;
 
-    Roots findRoots() override;
+    Roots findRoots(bool censor) override;
 
     void collectGarbage(const GCOptions & options, GCResults & results) override;
 
@@ -98,12 +98,15 @@ public:
 
     void connect() override;
 
+    unsigned int getProtocol() override;
+
     void flushBadConnections();
 
 protected:
 
     struct Connection
     {
+        AutoCloseFD fd;
         FdSink to;
         FdSource from;
         unsigned int daemonVersion;
@@ -111,7 +114,7 @@ protected:
 
         virtual ~Connection();
 
-        void processStderr(Sink * sink = 0, Source * source = 0);
+        std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0);
     };
 
     ref<Connection> openConnectionWrapper();
@@ -124,6 +127,10 @@ protected:
 
     virtual void setOptions(Connection & conn);
 
+    ConnectionHandle getConnection();
+
+    friend struct ConnectionHandle;
+
 private:
 
     std::atomic_bool failed{false};
@@ -141,13 +148,8 @@ public:
 
 private:
 
-    struct Connection : RemoteStore::Connection
-    {
-        AutoCloseFD fd;
-    };
-
     ref<RemoteStore::Connection> openConnection() override;
-    std::experimental::optional<std::string> path;
+    std::optional<std::string> path;
 };
 
 
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 23af452094cf..cd547a964850 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -17,13 +17,15 @@
 #include <aws/core/client/DefaultRetryStrategy.h>
 #include <aws/core/utils/logging/FormattedLogSystem.h>
 #include <aws/core/utils/logging/LogMacros.h>
+#include <aws/core/utils/threading/Executor.h>
 #include <aws/s3/S3Client.h>
-#include <aws/s3/model/CreateBucketRequest.h>
-#include <aws/s3/model/GetBucketLocationRequest.h>
 #include <aws/s3/model/GetObjectRequest.h>
 #include <aws/s3/model/HeadObjectRequest.h>
 #include <aws/s3/model/ListObjectsRequest.h>
 #include <aws/s3/model/PutObjectRequest.h>
+#include <aws/transfer/TransferManager.h>
+
+using namespace Aws::Transfer;
 
 namespace nix {
 
@@ -80,8 +82,8 @@ static void initAWS()
     });
 }
 
-S3Helper::S3Helper(const std::string & profile, const std::string & region)
-    : config(makeConfig(region))
+S3Helper::S3Helper(const string & profile, const string & region, const string & scheme, const string & endpoint)
+    : config(makeConfig(region, scheme, endpoint))
     , client(make_ref<Aws::S3::S3Client>(
             profile == ""
             ? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
@@ -95,7 +97,7 @@ S3Helper::S3Helper(const std::string & profile, const std::string & region)
 #else
             Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
 #endif
-            false))
+            endpoint.empty()))
 {
 }
 
@@ -112,12 +114,19 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy
     }
 };
 
-ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region)
+ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & scheme, const string & endpoint)
 {
     initAWS();
     auto res = make_ref<Aws::Client::ClientConfiguration>();
     res->region = region;
+    if (!scheme.empty()) {
+        res->scheme = Aws::Http::SchemeMapper::FromString(scheme.c_str());
+    }
+    if (!endpoint.empty()) {
+        res->endpointOverride = endpoint;
+    }
     res->requestTimeoutMs = 600 * 1000;
+    res->connectTimeoutMs = 5 * 1000;
     res->retryStrategy = std::make_shared<RetryStrategy>();
     res->caFile = settings.caFile;
     return res;
@@ -146,10 +155,8 @@ S3Helper::DownloadResult S3Helper::getObject(
         auto result = checkAws(fmt("AWS error fetching '%s'", key),
             client->GetObject(request));
 
-        res.data = decodeContent(
-            result.GetContentEncoding(),
-            make_ref<std::string>(
-                dynamic_cast<std::stringstream &>(result.GetBody()).str()));
+        res.data = decompress(result.GetContentEncoding(),
+            dynamic_cast<std::stringstream &>(result.GetBody()).str());
 
     } catch (S3Error & e) {
         if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw;
@@ -166,9 +173,15 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 {
     const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
     const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
+    const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."};
+    const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
     const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
     const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
     const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
+    const Setting<bool> multipartUpload{
+        this, false, "multipart-upload", "whether to use multi-part uploads"};
+    const Setting<uint64_t> bufferSize{
+        this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
 
     std::string bucketName;
 
@@ -180,7 +193,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         const Params & params, const std::string & bucketName)
         : S3BinaryCacheStore(params)
         , bucketName(bucketName)
-        , s3Helper(profile, region)
+        , s3Helper(profile, region, scheme, endpoint)
     {
         diskCache = getNarInfoDiskCache();
     }
@@ -194,32 +207,6 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     {
         if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) {
 
-            /* Create the bucket if it doesn't already exists. */
-            // FIXME: HeadBucket would be more appropriate, but doesn't return
-            // an easily parsed 404 message.
-            auto res = s3Helper.client->GetBucketLocation(
-                Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName));
-
-            if (!res.IsSuccess()) {
-                if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET)
-                    throw Error(format("AWS error checking bucket '%s': %s") % bucketName % res.GetError().GetMessage());
-
-                printInfo("creating S3 bucket '%s'...", bucketName);
-
-                // Stupid S3 bucket locations.
-                auto bucketConfig = Aws::S3::Model::CreateBucketConfiguration();
-                if (s3Helper.config->region != "us-east-1")
-                    bucketConfig.SetLocationConstraint(
-                        Aws::S3::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName(
-                            s3Helper.config->region));
-
-                checkAws(format("AWS error creating bucket '%s'") % bucketName,
-                    s3Helper.client->CreateBucket(
-                        Aws::S3::Model::CreateBucketRequest()
-                        .WithBucket(bucketName)
-                        .WithCreateBucketConfiguration(bucketConfig)));
-            }
-
             BinaryCacheStore::init();
 
             diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority);
@@ -267,40 +254,100 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         return true;
     }
 
+    std::shared_ptr<TransferManager> transferManager;
+    std::once_flag transferManagerCreated;
+
     void uploadFile(const std::string & path, const std::string & data,
         const std::string & mimeType,
         const std::string & contentEncoding)
     {
-        auto request =
-            Aws::S3::Model::PutObjectRequest()
-            .WithBucket(bucketName)
-            .WithKey(path);
+        auto stream = std::make_shared<istringstream_nocopy>(data);
 
-        request.SetContentType(mimeType);
+        auto maxThreads = std::thread::hardware_concurrency();
+
+        static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
+            executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads);
+
+        std::call_once(transferManagerCreated, [&]()
+        {
+            if (multipartUpload) {
+                TransferManagerConfiguration transferConfig(executor.get());
+
+                transferConfig.s3Client = s3Helper.client;
+                transferConfig.bufferSize = bufferSize;
+
+                transferConfig.uploadProgressCallback =
+                    [](const TransferManager *transferManager,
+                        const std::shared_ptr<const TransferHandle>
+                        &transferHandle)
+                    {
+                        //FIXME: find a way to properly abort the multipart upload.
+                        //checkInterrupt();
+                        debug("upload progress ('%s'): '%d' of '%d' bytes",
+                            transferHandle->GetKey(),
+                            transferHandle->GetBytesTransferred(),
+                            transferHandle->GetBytesTotalSize());
+                    };
+
+                transferManager = TransferManager::Create(transferConfig);
+            }
+        });
 
-        if (contentEncoding != "")
-            request.SetContentEncoding(contentEncoding);
+        auto now1 = std::chrono::steady_clock::now();
 
-        auto stream = std::make_shared<istringstream_nocopy>(data);
+        if (transferManager) {
 
-        request.SetBody(stream);
+            if (contentEncoding != "")
+                throw Error("setting a content encoding is not supported with S3 multi-part uploads");
 
-        stats.put++;
-        stats.putBytes += data.size();
+            std::shared_ptr<TransferHandle> transferHandle =
+                transferManager->UploadFile(
+                    stream, bucketName, path, mimeType,
+                    Aws::Map<Aws::String, Aws::String>(),
+                    nullptr /*, contentEncoding */);
 
-        auto now1 = std::chrono::steady_clock::now();
+            transferHandle->WaitUntilFinished();
+
+            if (transferHandle->GetStatus() == TransferStatus::FAILED)
+                throw Error("AWS error: failed to upload 's3://%s/%s': %s",
+                    bucketName, path, transferHandle->GetLastError().GetMessage());
+
+            if (transferHandle->GetStatus() != TransferStatus::COMPLETED)
+                throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state",
+                    bucketName, path);
+
+        } else {
+
+            auto request =
+                Aws::S3::Model::PutObjectRequest()
+                .WithBucket(bucketName)
+                .WithKey(path);
+
+            request.SetContentType(mimeType);
+
+            if (contentEncoding != "")
+                request.SetContentEncoding(contentEncoding);
 
-        auto result = checkAws(format("AWS error uploading '%s'") % path,
-            s3Helper.client->PutObject(request));
+            auto stream = std::make_shared<istringstream_nocopy>(data);
+
+            request.SetBody(stream);
+
+            auto result = checkAws(fmt("AWS error uploading '%s'", path),
+                s3Helper.client->PutObject(request));
+        }
 
         auto now2 = std::chrono::steady_clock::now();
 
-        auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+        auto duration =
+            std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
+                .count();
 
-        printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms")
-            % bucketName % path % data.size() % duration);
+        printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") %
+                  bucketName % path % data.size() % duration);
 
         stats.putTimeMs += duration;
+        stats.putBytes += data.size();
+        stats.put++;
     }
 
     void upsertFile(const std::string & path, const std::string & data,
@@ -316,24 +363,23 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             uploadFile(path, data, mimeType, "");
     }
 
-    void getFile(const std::string & path,
-        std::function<void(std::shared_ptr<std::string>)> success,
-        std::function<void(std::exception_ptr exc)> failure) override
+    void getFile(const std::string & path, Sink & sink) override
     {
-        sync2async<std::shared_ptr<std::string>>(success, failure, [&]() {
-            stats.get++;
+        stats.get++;
 
-            auto res = s3Helper.getObject(bucketName, path);
+        // FIXME: stream output to sink.
+        auto res = s3Helper.getObject(bucketName, path);
 
-            stats.getBytes += res.data ? res.data->size() : 0;
-            stats.getTimeMs += res.durationMs;
+        stats.getBytes += res.data ? res.data->size() : 0;
+        stats.getTimeMs += res.durationMs;
 
-            if (res.data)
-                printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
-                    bucketName, path, res.data->size(), res.durationMs);
+        if (res.data) {
+            printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
+                bucketName, path, res.data->size(), res.durationMs);
 
-            return res.data;
-        });
+            sink((unsigned char *) res.data->data(), res.data->size());
+        } else
+            throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
     }
 
     PathSet queryAllValidPaths() override
diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh
index 4f996400343c..ef5f23d0f253 100644
--- a/src/libstore/s3.hh
+++ b/src/libstore/s3.hh
@@ -14,9 +14,9 @@ struct S3Helper
     ref<Aws::Client::ClientConfiguration> config;
     ref<Aws::S3::S3Client> client;
 
-    S3Helper(const std::string & profile, const std::string & region);
+    S3Helper(const std::string & profile, const std::string & region, const std::string & scheme, const std::string & endpoint);
 
-    ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region);
+    ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint);
 
     struct DownloadResult
     {
diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh
index f67d1e2580a5..9fae6d5349f1 100644
--- a/src/libstore/serve-protocol.hh
+++ b/src/libstore/serve-protocol.hh
@@ -5,7 +5,7 @@ namespace nix {
 #define SERVE_MAGIC_1 0x390c9deb
 #define SERVE_MAGIC_2 0x5452eecb
 
-#define SERVE_PROTOCOL_VERSION 0x204
+#define SERVE_PROTOCOL_VERSION 0x205
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
@@ -18,6 +18,7 @@ typedef enum {
     cmdBuildPaths = 6,
     cmdQueryClosure = 7,
     cmdBuildDerivation = 8,
+    cmdAddToStoreNar = 9,
 } ServeCommand;
 
 }
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index 42d40e71d8be..a061d64f36d8 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -10,6 +10,7 @@ namespace nix {
 [[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs)
 {
     int err = sqlite3_errcode(db);
+    int exterr = sqlite3_extended_errcode(db);
 
     auto path = sqlite3_db_filename(db, nullptr);
     if (!path) path = "(in-memory)";
@@ -21,7 +22,7 @@ namespace nix {
             : fmt("SQLite database '%s' is busy", path));
     }
     else
-        throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(err), path);
+        throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path);
 }
 
 SQLite::SQLite(const Path & path)
diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index 033c580936ad..5e0e44935cca 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -4,8 +4,9 @@ namespace nix {
 
 SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD)
     : host(host)
+    , fakeSSH(host == "localhost")
     , keyFile(keyFile)
-    , useMaster(useMaster)
+    , useMaster(useMaster && !fakeSSH)
     , compress(compress)
     , logFD(logFD)
 {
@@ -45,12 +46,19 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
         if (logFD != -1 && dup2(logFD, STDERR_FILENO) == -1)
             throw SysError("duping over stderr");
 
-        Strings args = { "ssh", host.c_str(), "-x", "-a" };
-        addCommonSSHOpts(args);
-        if (socketPath != "")
-            args.insert(args.end(), {"-S", socketPath});
-        if (verbosity >= lvlChatty)
-            args.push_back("-v");
+        Strings args;
+
+        if (fakeSSH) {
+            args = { "bash", "-c" };
+        } else {
+            args = { "ssh", host.c_str(), "-x", "-a" };
+            addCommonSSHOpts(args);
+            if (socketPath != "")
+                args.insert(args.end(), {"-S", socketPath});
+            if (verbosity >= lvlChatty)
+                args.push_back("-v");
+        }
+
         args.push_back(command);
         execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
 
diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh
index 1268e6d00054..4f0f0bd29f9f 100644
--- a/src/libstore/ssh.hh
+++ b/src/libstore/ssh.hh
@@ -10,6 +10,7 @@ class SSHMaster
 private:
 
     const std::string host;
+    bool fakeSSH;
     const std::string keyFile;
     const bool useMaster;
     const bool compress;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 1a0d12ca78c2..c13ff11564ec 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -253,6 +253,8 @@ std::string Store::getUri()
 
 bool Store::isValidPath(const Path & storePath)
 {
+    assertStorePath(storePath);
+
     auto hashPart = storePathToHash(storePath);
 
     {
@@ -303,21 +305,23 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
     std::promise<ref<ValidPathInfo>> promise;
 
     queryPathInfo(storePath,
-        [&](ref<ValidPathInfo> info) {
-            promise.set_value(info);
-        },
-        [&](std::exception_ptr exc) {
-            promise.set_exception(exc);
-        });
+        {[&](std::future<ref<ValidPathInfo>> result) {
+            try {
+                promise.set_value(result.get());
+            } catch (...) {
+                promise.set_exception(std::current_exception());
+            }
+        }});
 
     return promise.get_future().get();
 }
 
 
 void Store::queryPathInfo(const Path & storePath,
-    std::function<void(ref<ValidPathInfo>)> success,
-    std::function<void(std::exception_ptr exc)> failure)
+    Callback<ref<ValidPathInfo>> callback)
 {
+    assertStorePath(storePath);
+
     auto hashPart = storePathToHash(storePath);
 
     try {
@@ -328,7 +332,7 @@ void Store::queryPathInfo(const Path & storePath,
                 stats.narInfoReadAverted++;
                 if (!*res)
                     throw InvalidPath(format("path '%s' is not valid") % storePath);
-                return success(ref<ValidPathInfo>(*res));
+                return callback(ref<ValidPathInfo>(*res));
             }
         }
 
@@ -344,35 +348,36 @@ void Store::queryPathInfo(const Path & storePath,
                         (res.second->path != storePath && storePathToName(storePath) != ""))
                         throw InvalidPath(format("path '%s' is not valid") % storePath);
                 }
-                return success(ref<ValidPathInfo>(res.second));
+                return callback(ref<ValidPathInfo>(res.second));
             }
         }
 
-    } catch (std::exception & e) {
-        return callFailure(failure);
-    }
+    } catch (...) { return callback.rethrow(); }
 
     queryPathInfoUncached(storePath,
-        [this, storePath, hashPart, success, failure](std::shared_ptr<ValidPathInfo> info) {
+        {[this, storePath, hashPart, callback](std::future<std::shared_ptr<ValidPathInfo>> fut) {
 
-            if (diskCache)
-                diskCache->upsertNarInfo(getUri(), hashPart, info);
+            try {
+                auto info = fut.get();
 
-            {
-                auto state_(state.lock());
-                state_->pathInfoCache.upsert(hashPart, info);
-            }
+                if (diskCache)
+                    diskCache->upsertNarInfo(getUri(), hashPart, info);
 
-            if (!info
-                || (info->path != storePath && storePathToName(storePath) != ""))
-            {
-                stats.narInfoMissing++;
-                return failure(std::make_exception_ptr(InvalidPath(format("path '%s' is not valid") % storePath)));
-            }
+                {
+                    auto state_(state.lock());
+                    state_->pathInfoCache.upsert(hashPart, info);
+                }
 
-            callSuccess(success, failure, ref<ValidPathInfo>(info));
+                if (!info
+                    || (info->path != storePath && storePathToName(storePath) != ""))
+                {
+                    stats.narInfoMissing++;
+                    throw InvalidPath("path '%s' is not valid", storePath);
+                }
 
-        }, failure);
+                callback(ref<ValidPathInfo>(info));
+            } catch (...) { callback.rethrow(); }
+        }});
 }
 
 
@@ -392,26 +397,19 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
 
     auto doQuery = [&](const Path & path ) {
         checkInterrupt();
-        queryPathInfo(path,
-            [path, &state_, &wakeup](ref<ValidPathInfo> info) {
-                auto state(state_.lock());
+        queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<ValidPathInfo>> fut) {
+            auto state(state_.lock());
+            try {
+                auto info = fut.get();
                 state->valid.insert(path);
-                assert(state->left);
-                if (!--state->left)
-                    wakeup.notify_one();
-            },
-            [path, &state_, &wakeup](std::exception_ptr exc) {
-                auto state(state_.lock());
-                try {
-                    std::rethrow_exception(exc);
-                } catch (InvalidPath &) {
-                } catch (...) {
-                    state->exc = exc;
-                }
-                assert(state->left);
-                if (!--state->left)
-                    wakeup.notify_one();
-            });
+            } catch (InvalidPath &) {
+            } catch (...) {
+                state->exc = std::current_exception();
+            }
+            assert(state->left);
+            if (!--state->left)
+                wakeup.notify_one();
+        }});
     };
 
     for (auto & path : paths)
@@ -564,10 +562,10 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode)
 {
     for (auto & path : paths)
         if (isDerivation(path))
-            unsupported();
+            unsupported("buildPaths");
 
     if (queryValidPaths(paths).size() != paths.size())
-        unsupported();
+        unsupported("buildPaths");
 }
 
 
@@ -590,15 +588,19 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
 
     uint64_t total = 0;
 
-    // FIXME
-#if 0
     if (!info->narHash) {
+        StringSink sink;
+        srcStore->narFromPath({storePath}, sink);
         auto info2 = make_ref<ValidPathInfo>(*info);
         info2->narHash = hashString(htSHA256, *sink.s);
         if (!info->narSize) info2->narSize = sink.s->size();
+        if (info->ultimate) info2->ultimate = false;
         info = info2;
+
+        StringSource source(*sink.s);
+        dstStore->addToStore(*info, source, repair, checkSigs);
+        return;
     }
-#endif
 
     if (info->ultimate) {
         auto info2 = make_ref<ValidPathInfo>(*info);
@@ -613,6 +615,8 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
             act.progress(total, info->narSize);
         });
         srcStore->narFromPath({storePath}, wrapperSink);
+    }, [&]() {
+        throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri());
     });
 
     dstStore->addToStore(*info, *source, repair, checkSigs);
@@ -633,11 +637,12 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
     Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
 
     std::atomic<size_t> nrDone{0};
+    std::atomic<size_t> nrFailed{0};
     std::atomic<uint64_t> bytesExpected{0};
     std::atomic<uint64_t> nrRunning{0};
 
     auto showProgress = [&]() {
-        act.progress(nrDone, missing.size(), nrRunning);
+        act.progress(nrDone, missing.size(), nrRunning, nrFailed);
     };
 
     ThreadPool pool;
@@ -666,7 +671,16 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
             if (!dstStore->isValidPath(storePath)) {
                 MaintainCount<decltype(nrRunning)> mc(nrRunning);
                 showProgress();
-                copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
+                try {
+                    copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
+                } catch (Error &e) {
+                    nrFailed++;
+                    if (!settings.keepGoing)
+                        throw e;
+                    logger->log(lvlError, format("could not copy %s: %s") % storePath % e.what());
+                    showProgress();
+                    return;
+                }
             }
 
             nrDone++;
@@ -828,26 +842,50 @@ namespace nix {
 
 RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
 
-
-ref<Store> openStore(const std::string & uri_,
-    const Store::Params & extraParams)
+/* Split URI into protocol+hierarchy part and its parameter set. */
+std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_)
 {
     auto uri(uri_);
-    Store::Params params(extraParams);
+    Store::Params params;
     auto q = uri.find('?');
     if (q != std::string::npos) {
         for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
             auto e = s.find('=');
-            if (e != std::string::npos)
-                params[s.substr(0, e)] = s.substr(e + 1);
+            if (e != std::string::npos) {
+                auto value = s.substr(e + 1);
+                std::string decoded;
+                for (size_t i = 0; i < value.size(); ) {
+                    if (value[i] == '%') {
+                        if (i + 2 >= value.size())
+                            throw Error("invalid URI parameter '%s'", value);
+                        try {
+                            decoded += std::stoul(std::string(value, i + 1, 2), 0, 16);
+                            i += 3;
+                        } catch (...) {
+                            throw Error("invalid URI parameter '%s'", value);
+                        }
+                    } else
+                        decoded += value[i++];
+                }
+                params[s.substr(0, e)] = decoded;
+            }
         }
         uri = uri_.substr(0, q);
     }
+    return {uri, params};
+}
+
+ref<Store> openStore(const std::string & uri_,
+    const Store::Params & extraParams)
+{
+    auto [uri, uriParams] = splitUriAndParams(uri_);
+    auto params = extraParams;
+    params.insert(uriParams.begin(), uriParams.end());
 
     for (auto fun : *RegisterStoreImplementation::implementations) {
         auto store = fun(uri, params);
         if (store) {
-            store->handleUnknownSettings();
+            store->warnUnknownSettings();
             return ref<Store>(store);
         }
     }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index ea259f07e8ab..7a1b31d0ff59 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -11,6 +11,8 @@
 #include <atomic>
 #include <limits>
 #include <map>
+#include <unordered_map>
+#include <unordered_set>
 #include <memory>
 #include <string>
 
@@ -22,6 +24,8 @@ MakeError(SubstError, Error)
 MakeError(BuildError, Error) /* denotes a permanent build failure */
 MakeError(InvalidPath, Error)
 MakeError(Unsupported, Error)
+MakeError(SubstituteGone, Error)
+MakeError(SubstituterDisabled, Error)
 
 
 struct BasicDerivation;
@@ -45,7 +49,7 @@ const size_t storePathHashLen = 32; // i.e. 160 bits
 const uint32_t exportMagic = 0x4558494e;
 
 
-typedef std::map<Path, Path> Roots;
+typedef std::unordered_map<Path, std::unordered_set<std::string>> Roots;
 
 
 struct GCOptions
@@ -347,7 +351,8 @@ public:
        (i.e. you'll get /nix/store/<hash> rather than
        /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the
        full store path. */
-    virtual PathSet queryAllValidPaths() = 0;
+    virtual PathSet queryAllValidPaths()
+    { unsupported("queryAllValidPaths"); }
 
     /* Query information about a valid path. It is permitted to omit
        the name part of the store path. */
@@ -355,21 +360,19 @@ public:
 
     /* Asynchronous version of queryPathInfo(). */
     void queryPathInfo(const Path & path,
-        std::function<void(ref<ValidPathInfo>)> success,
-        std::function<void(std::exception_ptr exc)> failure);
+        Callback<ref<ValidPathInfo>> callback);
 
 protected:
 
     virtual void queryPathInfoUncached(const Path & path,
-        std::function<void(std::shared_ptr<ValidPathInfo>)> success,
-        std::function<void(std::exception_ptr exc)> failure) = 0;
+        Callback<std::shared_ptr<ValidPathInfo>> callback) = 0;
 
 public:
 
     /* Queries the set of incoming FS references for a store path.
        The result is not cleared. */
-    virtual void queryReferrers(const Path & path,
-        PathSet & referrers) = 0;
+    virtual void queryReferrers(const Path & path, PathSet & referrers)
+    { unsupported("queryReferrers"); }
 
     /* Return all currently valid derivations that have `path' as an
        output.  (Note that the result of `queryDeriver()' is the
@@ -378,10 +381,12 @@ public:
     virtual PathSet queryValidDerivers(const Path & path) { return {}; };
 
     /* Query the outputs of the derivation denoted by `path'. */
-    virtual PathSet queryDerivationOutputs(const Path & path) = 0;
+    virtual PathSet queryDerivationOutputs(const Path & path)
+    { unsupported("queryDerivationOutputs"); }
 
     /* Query the output names of the derivation denoted by `path'. */
-    virtual StringSet queryDerivationOutputNames(const Path & path) = 0;
+    virtual StringSet queryDerivationOutputNames(const Path & path)
+    { unsupported("queryDerivationOutputNames"); }
 
     /* Query the full store path given the hash part of a valid store
        path, or "" if the path doesn't exist. */
@@ -447,14 +452,16 @@ public:
 
     /* Add a store path as a temporary root of the garbage collector.
        The root disappears as soon as we exit. */
-    virtual void addTempRoot(const Path & path) = 0;
+    virtual void addTempRoot(const Path & path)
+    { unsupported("addTempRoot"); }
 
     /* Add an indirect root, which is merely a symlink to `path' from
        /nix/var/nix/gcroots/auto/<hash of `path'>.  `path' is supposed
        to be a symlink to a store path.  The garbage collector will
        automatically remove the indirect root when it finds that
        `path' has disappeared. */
-    virtual void addIndirectRoot(const Path & path) = 0;
+    virtual void addIndirectRoot(const Path & path)
+    { unsupported("addIndirectRoot"); }
 
     /* Acquire the global GC lock, then immediately release it.  This
        function must be called after registering a new permanent root,
@@ -478,11 +485,15 @@ public:
 
     /* Find the roots of the garbage collector.  Each root is a pair
        (link, storepath) where `link' is the path of the symlink
-       outside of the Nix store that point to `storePath'.  */
-    virtual Roots findRoots() = 0;
+       outside of the Nix store that point to `storePath'. If
+       'censor' is true, privacy-sensitive information about roots
+       found in /proc is censored. */
+    virtual Roots findRoots(bool censor)
+    { unsupported("findRoots"); }
 
     /* Perform a garbage collection. */
-    virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0;
+    virtual void collectGarbage(const GCOptions & options, GCResults & results)
+    { unsupported("collectGarbage"); }
 
     /* Return a string representing information about the path that
        can be loaded into the database using `nix-store --load-db' or
@@ -513,11 +524,13 @@ public:
     virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) { return false; };
 
     /* Return an object to access files in the Nix store. */
-    virtual ref<FSAccessor> getFSAccessor() = 0;
+    virtual ref<FSAccessor> getFSAccessor()
+    { unsupported("getFSAccessor"); }
 
     /* Add signatures to the specified store path. The signatures are
        not verified. */
-    virtual void addSignatures(const Path & storePath, const StringSet & sigs) = 0;
+    virtual void addSignatures(const Path & storePath, const StringSet & sigs)
+    { unsupported("addSignatures"); }
 
     /* Utility functions. */
 
@@ -599,6 +612,12 @@ public:
        a notion of connection. Otherwise this is a no-op. */
     virtual void connect() { };
 
+    /* Get the protocol version of this store or it's connection. */
+    virtual unsigned int getProtocol()
+    {
+        return 0;
+    };
+
     /* Get the priority of the store, used to order substituters. In
        particular, binary caches can specify a priority field in their
        "nix-cache-info" file. Lower value means higher priority. */
@@ -614,9 +633,9 @@ protected:
     Stats stats;
 
     /* Unsupported methods. */
-    [[noreturn]] void unsupported()
+    [[noreturn]] void unsupported(const std::string & op)
     {
-        throw Unsupported("requested operation is not supported by store '%s'", getUri());
+        throw Unsupported("operation '%s' is not supported by store '%s'", op, getUri());
     }
 
 };
@@ -783,4 +802,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str,
    for paths created by makeFixedOutputPath() / addToStore(). */
 std::string makeFixedOutputCA(bool recursive, const Hash & hash);
 
+
+/* Split URI into protocol+hierarchy part and its parameter set. */
+std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
+
 }
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 996e1d25355f..5ebdfaf134d6 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -6,7 +6,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x114
+#define PROTOCOL_VERSION 0x115
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 154e2d20430c..3aa120270970 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -13,17 +13,25 @@
 
 #include "archive.hh"
 #include "util.hh"
-
+#include "config.hh"
 
 namespace nix {
 
+struct ArchiveSettings : Config
+{
+    Setting<bool> useCaseHack{this,
+        #if __APPLE__
+            true,
+        #else
+            false,
+        #endif
+        "use-case-hack",
+        "Whether to enable a Darwin-specific hack for dealing with file name collisions."};
+};
 
-bool useCaseHack =
-#if __APPLE__
-    true;
-#else
-    false;
-#endif
+static ArchiveSettings archiveSettings;
+
+static GlobalConfig::Register r1(&archiveSettings);
 
 const std::string narVersionMagic1 = "nix-archive-1";
 
@@ -78,7 +86,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
            the case hack applied by restorePath(). */
         std::map<string, string> unhacked;
         for (auto & i : readDirectory(path))
-            if (useCaseHack) {
+            if (archiveSettings.useCaseHack) {
                 string name(i.name);
                 size_t pos = i.name.find(caseHackSuffix);
                 if (pos != string::npos) {
@@ -243,7 +251,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
                     if (name <= prevName)
                         throw Error("NAR directory is not sorted");
                     prevName = name;
-                    if (useCaseHack) {
+                    if (archiveSettings.useCaseHack) {
                         auto i = names.find(name);
                         if (i != names.end()) {
                             debug(format("case collision between '%1%' and '%2%'") % i->first % name);
@@ -275,7 +283,7 @@ void parseDump(ParseSink & sink, Source & source)
 {
     string version;
     try {
-        version = readString(source);
+        version = readString(source, narVersionMagic1.size());
     } catch (SerialisationError & e) {
         /* This generally means the integer at the start couldn't be
            decoded.  Ignore and throw the exception below. */
@@ -323,7 +331,7 @@ struct RestoreSink : ParseSink
                filesystem doesn't support preallocation (e.g. on
                OpenSolaris).  Since preallocation is just an
                optimisation, ignore it. */
-            if (errno && errno != EINVAL)
+            if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS)
                 throw SysError(format("preallocating file of %1% bytes") % len);
         }
 #endif
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index 7a0e688e4201..25be426c1a4d 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -78,10 +78,6 @@ void restorePath(const Path & path, Source & source);
 void copyNAR(Source & source, Sink & sink);
 
 
-// FIXME: global variables are bad m'kay.
-extern bool useCaseHack;
-
-
 extern const std::string narVersionMagic1;
 
 
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 81cb5e98c763..0dd84e32034a 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -8,246 +8,265 @@
 #include <cstdio>
 #include <cstring>
 
-#if HAVE_BROTLI
 #include <brotli/decode.h>
 #include <brotli/encode.h>
-#endif // HAVE_BROTLI
 
 #include <iostream>
 
 namespace nix {
 
-static const size_t bufSize = 32 * 1024;
-
-static void decompressNone(Source & source, Sink & sink)
+// Don't feed brotli too much at once.
+struct ChunkedCompressionSink : CompressionSink
 {
-    std::vector<unsigned char> buf(bufSize);
-    while (true) {
-        size_t n;
-        try {
-            n = source.read(buf.data(), buf.size());
-        } catch (EndOfFile &) {
-            break;
+    uint8_t outbuf[32 * 1024];
+
+    void write(const unsigned char * data, size_t len) override
+    {
+        const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
+        while (len) {
+            size_t n = std::min(CHUNK_SIZE, len);
+            writeInternal(data, n);
+            data += n;
+            len -= n;
         }
-        sink(buf.data(), n);
     }
-}
 
-static void decompressXZ(Source & source, Sink & sink)
+    virtual void writeInternal(const unsigned char * data, size_t len) = 0;
+};
+
+struct NoneSink : CompressionSink
 {
-    lzma_stream strm(LZMA_STREAM_INIT);
-
-    lzma_ret ret = lzma_stream_decoder(
-        &strm, UINT64_MAX, LZMA_CONCATENATED);
-    if (ret != LZMA_OK)
-        throw CompressionError("unable to initialise lzma decoder");
-
-    Finally free([&]() { lzma_end(&strm); });
-
-    lzma_action action = LZMA_RUN;
-    std::vector<uint8_t> inbuf(bufSize), outbuf(bufSize);
-    strm.next_in = nullptr;
-    strm.avail_in = 0;
-    strm.next_out = outbuf.data();
-    strm.avail_out = outbuf.size();
-    bool eof = false;
-
-    while (true) {
-        checkInterrupt();
-
-        if (strm.avail_in == 0 && !eof) {
-            strm.next_in = inbuf.data();
-            try {
-                strm.avail_in = source.read((unsigned char *) strm.next_in, inbuf.size());
-            } catch (EndOfFile &) {
-                eof = true;
-            }
-        }
+    Sink & nextSink;
+    NoneSink(Sink & nextSink) : nextSink(nextSink) { }
+    void finish() override { flush(); }
+    void write(const unsigned char * data, size_t len) override { nextSink(data, len); }
+};
 
-        if (strm.avail_in == 0)
-            action = LZMA_FINISH;
+struct XzDecompressionSink : CompressionSink
+{
+    Sink & nextSink;
+    uint8_t outbuf[BUFSIZ];
+    lzma_stream strm = LZMA_STREAM_INIT;
+    bool finished = false;
 
-        lzma_ret ret = lzma_code(&strm, action);
+    XzDecompressionSink(Sink & nextSink) : nextSink(nextSink)
+    {
+        lzma_ret ret = lzma_stream_decoder(
+            &strm, UINT64_MAX, LZMA_CONCATENATED);
+        if (ret != LZMA_OK)
+            throw CompressionError("unable to initialise lzma decoder");
 
-        if (strm.avail_out < outbuf.size()) {
-            sink((unsigned char *) outbuf.data(), outbuf.size() - strm.avail_out);
-            strm.next_out = outbuf.data();
-            strm.avail_out = outbuf.size();
-        }
+        strm.next_out = outbuf;
+        strm.avail_out = sizeof(outbuf);
+    }
 
-        if (ret == LZMA_STREAM_END) return;
+    ~XzDecompressionSink()
+    {
+        lzma_end(&strm);
+    }
 
-        if (ret != LZMA_OK)
-            throw CompressionError("error %d while decompressing xz file", ret);
+    void finish() override
+    {
+        CompressionSink::flush();
+        write(nullptr, 0);
     }
-}
 
-static void decompressBzip2(Source & source, Sink & sink)
-{
-    bz_stream strm;
-    memset(&strm, 0, sizeof(strm));
-
-    int ret = BZ2_bzDecompressInit(&strm, 0, 0);
-    if (ret != BZ_OK)
-        throw CompressionError("unable to initialise bzip2 decoder");
-
-    Finally free([&]() { BZ2_bzDecompressEnd(&strm); });
-
-    std::vector<char> inbuf(bufSize), outbuf(bufSize);
-    strm.next_in = nullptr;
-    strm.avail_in = 0;
-    strm.next_out = outbuf.data();
-    strm.avail_out = outbuf.size();
-    bool eof = false;
-
-    while (true) {
-        checkInterrupt();
-
-        if (strm.avail_in == 0 && !eof) {
-            strm.next_in = inbuf.data();
-            try {
-                strm.avail_in = source.read((unsigned char *) strm.next_in, inbuf.size());
-            } catch (EndOfFile &) {
-                eof = true;
-            }
-        }
+    void write(const unsigned char * data, size_t len) override
+    {
+        strm.next_in = data;
+        strm.avail_in = len;
+
+        while (!finished && (!data || strm.avail_in)) {
+            checkInterrupt();
 
-        int ret = BZ2_bzDecompress(&strm);
+            lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH);
+            if (ret != LZMA_OK && ret != LZMA_STREAM_END)
+                throw CompressionError("error %d while decompressing xz file", ret);
 
-        if (strm.avail_in == 0 && strm.avail_out == outbuf.size() && eof)
-            throw CompressionError("bzip2 data ends prematurely");
+            finished = ret == LZMA_STREAM_END;
 
-        if (strm.avail_out < outbuf.size()) {
-            sink((unsigned char *) outbuf.data(), outbuf.size() - strm.avail_out);
-            strm.next_out = outbuf.data();
-            strm.avail_out = outbuf.size();
+            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
+                nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
+                strm.next_out = outbuf;
+                strm.avail_out = sizeof(outbuf);
+            }
         }
+    }
+};
 
-        if (ret == BZ_STREAM_END) return;
+struct BzipDecompressionSink : ChunkedCompressionSink
+{
+    Sink & nextSink;
+    bz_stream strm;
+    bool finished = false;
 
+    BzipDecompressionSink(Sink & nextSink) : nextSink(nextSink)
+    {
+        memset(&strm, 0, sizeof(strm));
+        int ret = BZ2_bzDecompressInit(&strm, 0, 0);
         if (ret != BZ_OK)
-            throw CompressionError("error while decompressing bzip2 file");
+            throw CompressionError("unable to initialise bzip2 decoder");
+
+        strm.next_out = (char *) outbuf;
+        strm.avail_out = sizeof(outbuf);
     }
-}
 
-static void decompressBrotli(Source & source, Sink & sink)
-{
-#if !HAVE_BROTLI
-    RunOptions options(BROTLI, {"-d"});
-    options.standardIn = &source;
-    options.standardOut = &sink;
-    runProgram2(options);
-#else
-    auto *s = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
-    if (!s)
-        throw CompressionError("unable to initialize brotli decoder");
-
-    Finally free([s]() { BrotliDecoderDestroyInstance(s); });
-
-    std::vector<uint8_t> inbuf(bufSize), outbuf(bufSize);
-    const uint8_t * next_in = nullptr;
-    size_t avail_in = 0;
-    bool eof = false;
-
-    while (true) {
-        checkInterrupt();
-
-        if (avail_in == 0 && !eof) {
-            next_in = inbuf.data();
-            try {
-                avail_in = source.read((unsigned char *) next_in, inbuf.size());
-            } catch (EndOfFile &) {
-                eof = true;
+    ~BzipDecompressionSink()
+    {
+        BZ2_bzDecompressEnd(&strm);
+    }
+
+    void finish() override
+    {
+        flush();
+        write(nullptr, 0);
+    }
+
+    void writeInternal(const unsigned char * data, size_t len) override
+    {
+        assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
+
+        strm.next_in = (char *) data;
+        strm.avail_in = len;
+
+        while (strm.avail_in) {
+            checkInterrupt();
+
+            int ret = BZ2_bzDecompress(&strm);
+            if (ret != BZ_OK && ret != BZ_STREAM_END)
+                throw CompressionError("error while decompressing bzip2 file");
+
+            finished = ret == BZ_STREAM_END;
+
+            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
+                nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
+                strm.next_out = (char *) outbuf;
+                strm.avail_out = sizeof(outbuf);
             }
         }
+    }
+};
 
-        uint8_t * next_out = outbuf.data();
-        size_t avail_out = outbuf.size();
-
-        auto ret = BrotliDecoderDecompressStream(s,
-                &avail_in, &next_in,
-                &avail_out, &next_out,
-                nullptr);
-
-        switch (ret) {
-        case BROTLI_DECODER_RESULT_ERROR:
-            throw CompressionError("error while decompressing brotli file");
-        case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
-            if (eof)
-                throw CompressionError("incomplete or corrupt brotli file");
-            break;
-        case BROTLI_DECODER_RESULT_SUCCESS:
-            if (avail_in != 0)
-                throw CompressionError("unexpected input after brotli decompression");
-            break;
-        case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
-            // I'm not sure if this can happen, but abort if this happens with empty buffer
-            if (avail_out == outbuf.size())
-                throw CompressionError("brotli decompression requires larger buffer");
-            break;
-        }
+struct BrotliDecompressionSink : ChunkedCompressionSink
+{
+    Sink & nextSink;
+    BrotliDecoderState * state;
+    bool finished = false;
 
-        // Always ensure we have full buffer for next invocation
-        if (avail_out < outbuf.size())
-            sink((unsigned char *) outbuf.data(), outbuf.size() - avail_out);
+    BrotliDecompressionSink(Sink & nextSink) : nextSink(nextSink)
+    {
+        state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
+        if (!state)
+            throw CompressionError("unable to initialize brotli decoder");
+    }
 
-        if (ret == BROTLI_DECODER_RESULT_SUCCESS) return;
+    ~BrotliDecompressionSink()
+    {
+        BrotliDecoderDestroyInstance(state);
     }
-#endif // HAVE_BROTLI
-}
+
+    void finish() override
+    {
+        flush();
+        writeInternal(nullptr, 0);
+    }
+
+    void writeInternal(const unsigned char * data, size_t len) override
+    {
+        const uint8_t * next_in = data;
+        size_t avail_in = len;
+        uint8_t * next_out = outbuf;
+        size_t avail_out = sizeof(outbuf);
+
+        while (!finished && (!data || avail_in)) {
+            checkInterrupt();
+
+            if (!BrotliDecoderDecompressStream(state,
+                    &avail_in, &next_in,
+                    &avail_out, &next_out,
+                    nullptr))
+                throw CompressionError("error while decompressing brotli file");
+
+            if (avail_out < sizeof(outbuf) || avail_in == 0) {
+                nextSink(outbuf, sizeof(outbuf) - avail_out);
+                next_out = outbuf;
+                avail_out = sizeof(outbuf);
+            }
+
+            finished = BrotliDecoderIsFinished(state);
+        }
+    }
+};
 
 ref<std::string> decompress(const std::string & method, const std::string & in)
 {
-    StringSource source(in);
-    StringSink sink;
-    decompress(method, source, sink);
-    return sink.s;
+    StringSink ssink;
+    auto sink = makeDecompressionSink(method, ssink);
+    (*sink)(in);
+    sink->finish();
+    return ssink.s;
 }
 
-void decompress(const std::string & method, Source & source, Sink & sink)
+ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
 {
-    if (method == "none")
-        return decompressNone(source, sink);
+    if (method == "none" || method == "")
+        return make_ref<NoneSink>(nextSink);
     else if (method == "xz")
-        return decompressXZ(source, sink);
+        return make_ref<XzDecompressionSink>(nextSink);
     else if (method == "bzip2")
-        return decompressBzip2(source, sink);
+        return make_ref<BzipDecompressionSink>(nextSink);
     else if (method == "br")
-        return decompressBrotli(source, sink);
+        return make_ref<BrotliDecompressionSink>(nextSink);
     else
         throw UnknownCompressionMethod("unknown compression method '%s'", method);
 }
 
-struct NoneSink : CompressionSink
-{
-    Sink & nextSink;
-    NoneSink(Sink & nextSink) : nextSink(nextSink) { }
-    void finish() override { flush(); }
-    void write(const unsigned char * data, size_t len) override { nextSink(data, len); }
-};
-
-struct XzSink : CompressionSink
+struct XzCompressionSink : CompressionSink
 {
     Sink & nextSink;
     uint8_t outbuf[BUFSIZ];
     lzma_stream strm = LZMA_STREAM_INIT;
     bool finished = false;
 
-    template <typename F>
-    XzSink(Sink & nextSink, F&& initEncoder) : nextSink(nextSink) {
-        lzma_ret ret = initEncoder();
+    XzCompressionSink(Sink & nextSink, bool parallel) : nextSink(nextSink)
+    {
+        lzma_ret ret;
+        bool done = false;
+
+        if (parallel) {
+#ifdef HAVE_LZMA_MT
+            lzma_mt mt_options = {};
+            mt_options.flags = 0;
+            mt_options.timeout = 300; // Using the same setting as the xz cmd line
+            mt_options.preset = LZMA_PRESET_DEFAULT;
+            mt_options.filters = NULL;
+            mt_options.check = LZMA_CHECK_CRC64;
+            mt_options.threads = lzma_cputhreads();
+            mt_options.block_size = 0;
+            if (mt_options.threads == 0)
+                mt_options.threads = 1;
+            // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
+            // number of threads.
+            ret = lzma_stream_encoder_mt(&strm, &mt_options);
+            done = true;
+#else
+            printMsg(lvlError, "warning: parallel XZ compression requested but not supported, falling back to single-threaded compression");
+#endif
+        }
+
+        if (!done)
+            ret = lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64);
+
         if (ret != LZMA_OK)
             throw CompressionError("unable to initialise lzma encoder");
+
         // FIXME: apply the x86 BCJ filter?
 
         strm.next_out = outbuf;
         strm.avail_out = sizeof(outbuf);
     }
-    XzSink(Sink & nextSink) : XzSink(nextSink, [this]() {
-        return lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64);
-    }) {}
 
-    ~XzSink()
+    ~XzCompressionSink()
     {
         lzma_end(&strm);
     }
@@ -255,43 +274,25 @@ struct XzSink : CompressionSink
     void finish() override
     {
         CompressionSink::flush();
-
-        assert(!finished);
-        finished = true;
-
-        while (true) {
-            checkInterrupt();
-
-            lzma_ret ret = lzma_code(&strm, LZMA_FINISH);
-            if (ret != LZMA_OK && ret != LZMA_STREAM_END)
-                throw CompressionError("error while flushing xz file");
-
-            if (strm.avail_out == 0 || ret == LZMA_STREAM_END) {
-                nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
-                strm.next_out = outbuf;
-                strm.avail_out = sizeof(outbuf);
-            }
-
-            if (ret == LZMA_STREAM_END) break;
-        }
+        write(nullptr, 0);
     }
 
     void write(const unsigned char * data, size_t len) override
     {
-        assert(!finished);
-
         strm.next_in = data;
         strm.avail_in = len;
 
-        while (strm.avail_in) {
+        while (!finished && (!data || strm.avail_in)) {
             checkInterrupt();
 
-            lzma_ret ret = lzma_code(&strm, LZMA_RUN);
-            if (ret != LZMA_OK)
-                throw CompressionError("error while compressing xz file");
+            lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH);
+            if (ret != LZMA_OK && ret != LZMA_STREAM_END)
+                throw CompressionError("error %d while compressing xz file", ret);
+
+            finished = ret == LZMA_STREAM_END;
 
-            if (strm.avail_out == 0) {
-                nextSink(outbuf, sizeof(outbuf));
+            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
+                nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
                 strm.next_out = outbuf;
                 strm.avail_out = sizeof(outbuf);
             }
@@ -299,46 +300,24 @@ struct XzSink : CompressionSink
     }
 };
 
-#ifdef HAVE_LZMA_MT
-struct ParallelXzSink : public XzSink
-{
-  ParallelXzSink(Sink &nextSink) : XzSink(nextSink, [this]() {
-        lzma_mt mt_options = {};
-        mt_options.flags = 0;
-        mt_options.timeout = 300; // Using the same setting as the xz cmd line
-        mt_options.preset = LZMA_PRESET_DEFAULT;
-        mt_options.filters = NULL;
-        mt_options.check = LZMA_CHECK_CRC64;
-        mt_options.threads = lzma_cputhreads();
-        mt_options.block_size = 0;
-        if (mt_options.threads == 0)
-            mt_options.threads = 1;
-        // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
-        // number of threads.
-        return lzma_stream_encoder_mt(&strm, &mt_options);
-  }) {}
-};
-#endif
-
-struct BzipSink : CompressionSink
+struct BzipCompressionSink : ChunkedCompressionSink
 {
     Sink & nextSink;
-    char outbuf[BUFSIZ];
     bz_stream strm;
     bool finished = false;
 
-    BzipSink(Sink & nextSink) : nextSink(nextSink)
+    BzipCompressionSink(Sink & nextSink) : nextSink(nextSink)
     {
         memset(&strm, 0, sizeof(strm));
         int ret = BZ2_bzCompressInit(&strm, 9, 0, 30);
         if (ret != BZ_OK)
             throw CompressionError("unable to initialise bzip2 encoder");
 
-        strm.next_out = outbuf;
+        strm.next_out = (char *) outbuf;
         strm.avail_out = sizeof(outbuf);
     }
 
-    ~BzipSink()
+    ~BzipCompressionSink()
     {
         BZ2_bzCompressEnd(&strm);
     }
@@ -346,101 +325,49 @@ struct BzipSink : CompressionSink
     void finish() override
     {
         flush();
-
-        assert(!finished);
-        finished = true;
-
-        while (true) {
-            checkInterrupt();
-
-            int ret = BZ2_bzCompress(&strm, BZ_FINISH);
-            if (ret != BZ_FINISH_OK && ret != BZ_STREAM_END)
-                throw CompressionError("error while flushing bzip2 file");
-
-            if (strm.avail_out == 0 || ret == BZ_STREAM_END) {
-                nextSink((unsigned char *) outbuf, sizeof(outbuf) - strm.avail_out);
-                strm.next_out = outbuf;
-                strm.avail_out = sizeof(outbuf);
-            }
-
-            if (ret == BZ_STREAM_END) break;
-        }
+        writeInternal(nullptr, 0);
     }
 
-    void write(const unsigned char * data, size_t len) override
+    void writeInternal(const unsigned char * data, size_t len) override
     {
-        assert(!finished);
+        assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
 
         strm.next_in = (char *) data;
         strm.avail_in = len;
 
-        while (strm.avail_in) {
+        while (!finished && (!data || strm.avail_in)) {
             checkInterrupt();
 
-            int ret = BZ2_bzCompress(&strm, BZ_RUN);
-            if (ret != BZ_OK)
-                CompressionError("error while compressing bzip2 file");
+            int ret = BZ2_bzCompress(&strm, data ? BZ_RUN : BZ_FINISH);
+            if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END)
+                throw CompressionError("error %d while compressing bzip2 file", ret);
 
-            if (strm.avail_out == 0) {
-                nextSink((unsigned char *) outbuf, sizeof(outbuf));
-                strm.next_out = outbuf;
+            finished = ret == BZ_STREAM_END;
+
+            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
+                nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
+                strm.next_out = (char *) outbuf;
                 strm.avail_out = sizeof(outbuf);
             }
         }
     }
 };
 
-struct LambdaCompressionSink : CompressionSink
-{
-    Sink & nextSink;
-    std::string data;
-    using CompressFnTy = std::function<std::string(const std::string&)>;
-    CompressFnTy compressFn;
-    LambdaCompressionSink(Sink& nextSink, CompressFnTy compressFn)
-        : nextSink(nextSink)
-        , compressFn(std::move(compressFn))
-    {
-    };
-
-    void finish() override
-    {
-        flush();
-        nextSink(compressFn(data));
-    }
-
-    void write(const unsigned char * data, size_t len) override
-    {
-        checkInterrupt();
-        this->data.append((const char *) data, len);
-    }
-};
-
-struct BrotliCmdSink : LambdaCompressionSink
-{
-    BrotliCmdSink(Sink& nextSink)
-        : LambdaCompressionSink(nextSink, [](const std::string& data) {
-            return runProgram(BROTLI, true, {}, data);
-        })
-    {
-    }
-};
-
-#if HAVE_BROTLI
-struct BrotliSink : CompressionSink
+struct BrotliCompressionSink : ChunkedCompressionSink
 {
     Sink & nextSink;
     uint8_t outbuf[BUFSIZ];
     BrotliEncoderState *state;
     bool finished = false;
 
-    BrotliSink(Sink & nextSink) : nextSink(nextSink)
+    BrotliCompressionSink(Sink & nextSink) : nextSink(nextSink)
     {
         state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr);
         if (!state)
             throw CompressionError("unable to initialise brotli encoder");
     }
 
-    ~BrotliSink()
+    ~BrotliCompressionSink()
     {
         BrotliEncoderDestroyInstance(state);
     }
@@ -448,96 +375,47 @@ struct BrotliSink : CompressionSink
     void finish() override
     {
         flush();
-        assert(!finished);
-
-        const uint8_t *next_in = nullptr;
-        size_t avail_in = 0;
-        uint8_t *next_out = outbuf;
-        size_t avail_out = sizeof(outbuf);
-        while (!finished) {
-            checkInterrupt();
-
-            if (!BrotliEncoderCompressStream(state,
-                        BROTLI_OPERATION_FINISH,
-                        &avail_in, &next_in,
-                        &avail_out, &next_out,
-                        nullptr))
-                throw CompressionError("error while finishing brotli file");
-
-            finished = BrotliEncoderIsFinished(state);
-            if (avail_out == 0 || finished) {
-                nextSink(outbuf, sizeof(outbuf) - avail_out);
-                next_out = outbuf;
-                avail_out = sizeof(outbuf);
-            }
-        }
+        writeInternal(nullptr, 0);
     }
 
-    void write(const unsigned char * data, size_t len) override
-    {
-        assert(!finished);
-
-        // Don't feed brotli too much at once
-        const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
-        while (len) {
-          size_t n = std::min(CHUNK_SIZE, len);
-          writeInternal(data, n);
-          data += n;
-          len -= n;
-        }
-    }
-  private:
-    void writeInternal(const unsigned char * data, size_t len)
+    void writeInternal(const unsigned char * data, size_t len) override
     {
-        assert(!finished);
-
-        const uint8_t *next_in = data;
+        const uint8_t * next_in = data;
         size_t avail_in = len;
-        uint8_t *next_out = outbuf;
+        uint8_t * next_out = outbuf;
         size_t avail_out = sizeof(outbuf);
 
-        while (avail_in > 0) {
+        while (!finished && (!data || avail_in)) {
             checkInterrupt();
 
             if (!BrotliEncoderCompressStream(state,
-                      BROTLI_OPERATION_PROCESS,
-                      &avail_in, &next_in,
-                      &avail_out, &next_out,
-                      nullptr))
-                throw CompressionError("error while compressing brotli file");
+                    data ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH,
+                    &avail_in, &next_in,
+                    &avail_out, &next_out,
+                    nullptr))
+                throw CompressionError("error while compressing brotli compression");
 
             if (avail_out < sizeof(outbuf) || avail_in == 0) {
                 nextSink(outbuf, sizeof(outbuf) - avail_out);
                 next_out = outbuf;
                 avail_out = sizeof(outbuf);
             }
+
+            finished = BrotliEncoderIsFinished(state);
         }
     }
 };
-#endif // HAVE_BROTLI
 
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
 {
-    if (parallel) {
-#ifdef HAVE_LZMA_MT
-        if (method == "xz")
-            return make_ref<ParallelXzSink>(nextSink);
-#endif
-        printMsg(lvlError, format("Warning: parallel compression requested but not supported for method '%1%', falling back to single-threaded compression") % method);
-    }
-
     if (method == "none")
         return make_ref<NoneSink>(nextSink);
     else if (method == "xz")
-        return make_ref<XzSink>(nextSink);
+        return make_ref<XzCompressionSink>(nextSink, parallel);
     else if (method == "bzip2")
-        return make_ref<BzipSink>(nextSink);
+        return make_ref<BzipCompressionSink>(nextSink);
     else if (method == "br")
-#if HAVE_BROTLI
-        return make_ref<BrotliSink>(nextSink);
-#else
-        return make_ref<BrotliCmdSink>(nextSink);
-#endif
+        return make_ref<BrotliCompressionSink>(nextSink);
     else
         throw UnknownCompressionMethod(format("unknown compression method '%s'") % method);
 }
diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh
index f7a3e3fbd32e..dd666a4e19fd 100644
--- a/src/libutil/compression.hh
+++ b/src/libutil/compression.hh
@@ -8,17 +8,17 @@
 
 namespace nix {
 
-ref<std::string> decompress(const std::string & method, const std::string & in);
-
-void decompress(const std::string & method, Source & source, Sink & sink);
-
-ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);
-
 struct CompressionSink : BufferedSink
 {
     virtual void finish() = 0;
 };
 
+ref<std::string> decompress(const std::string & method, const std::string & in);
+
+ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
+
+ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);
+
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false);
 
 MakeError(UnknownCompressionMethod, Error);
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index ce6858f0d65a..9023cb1bb6de 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -4,15 +4,13 @@
 
 namespace nix {
 
-void Config::set(const std::string & name, const std::string & value)
+bool Config::set(const std::string & name, const std::string & value)
 {
     auto i = _settings.find(name);
-    if (i == _settings.end()) {
-        extras.emplace(name, value);
-    } else {
-        i->second.setting->set(value);
-        i->second.setting->overriden = true;
-    }
+    if (i == _settings.end()) return false;
+    i->second.setting->set(value);
+    i->second.setting->overriden = true;
+    return true;
 }
 
 void Config::addSetting(AbstractSetting * setting)
@@ -23,46 +21,51 @@ void Config::addSetting(AbstractSetting * setting)
 
     bool set = false;
 
-    auto i = extras.find(setting->name);
-    if (i != extras.end()) {
+    auto i = unknownSettings.find(setting->name);
+    if (i != unknownSettings.end()) {
         setting->set(i->second);
         setting->overriden = true;
-        extras.erase(i);
+        unknownSettings.erase(i);
         set = true;
     }
 
     for (auto & alias : setting->aliases) {
-        auto i = extras.find(alias);
-        if (i != extras.end()) {
+        auto i = unknownSettings.find(alias);
+        if (i != unknownSettings.end()) {
             if (set)
                 warn("setting '%s' is set, but it's an alias of '%s' which is also set",
                     alias, setting->name);
             else {
                 setting->set(i->second);
                 setting->overriden = true;
-                extras.erase(i);
+                unknownSettings.erase(i);
                 set = true;
             }
         }
     }
 }
 
-void Config::handleUnknownSettings()
+void AbstractConfig::warnUnknownSettings()
 {
-    for (auto & s : extras)
+    for (auto & s : unknownSettings)
         warn("unknown setting '%s'", s.first);
 }
 
-StringMap Config::getSettings(bool overridenOnly)
+void AbstractConfig::reapplyUnknownSettings()
+{
+    auto unknownSettings2 = std::move(unknownSettings);
+    for (auto & s : unknownSettings2)
+        set(s.first, s.second);
+}
+
+void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly)
 {
-    StringMap res;
     for (auto & opt : _settings)
         if (!opt.second.isAlias && (!overridenOnly || opt.second.setting->overriden))
-            res.emplace(opt.first, opt.second.setting->to_string());
-    return res;
+            res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description});
 }
 
-void Config::applyConfigFile(const Path & path)
+void AbstractConfig::applyConfigFile(const Path & path)
 {
     try {
         string contents = readFile(path);
@@ -287,4 +290,49 @@ void PathSetting::set(const std::string & str)
         value = canonPath(str);
 }
 
+bool GlobalConfig::set(const std::string & name, const std::string & value)
+{
+    for (auto & config : *configRegistrations)
+        if (config->set(name, value)) return true;
+
+    unknownSettings.emplace(name, value);
+
+    return false;
+}
+
+void GlobalConfig::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly)
+{
+    for (auto & config : *configRegistrations)
+        config->getSettings(res, overridenOnly);
+}
+
+void GlobalConfig::resetOverriden()
+{
+    for (auto & config : *configRegistrations)
+        config->resetOverriden();
+}
+
+void GlobalConfig::toJSON(JSONObject & out)
+{
+    for (auto & config : *configRegistrations)
+        config->toJSON(out);
+}
+
+void GlobalConfig::convertToArgs(Args & args, const std::string & category)
+{
+    for (auto & config : *configRegistrations)
+        config->convertToArgs(args, category);
+}
+
+GlobalConfig globalConfig;
+
+GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations;
+
+GlobalConfig::Register::Register(Config * config)
+{
+    if (!configRegistrations)
+        configRegistrations = new ConfigRegistrations;
+    configRegistrations->emplace_back(config);
+}
+
 }
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index d2e7faf17434..d86c65ff033a 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -12,6 +12,40 @@ class AbstractSetting;
 class JSONPlaceholder;
 class JSONObject;
 
+class AbstractConfig
+{
+protected:
+    StringMap unknownSettings;
+
+    AbstractConfig(const StringMap & initials = {})
+        : unknownSettings(initials)
+    { }
+
+public:
+
+    virtual bool set(const std::string & name, const std::string & value) = 0;
+
+    struct SettingInfo
+    {
+        std::string value;
+        std::string description;
+    };
+
+    virtual void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) = 0;
+
+    void applyConfigFile(const Path & path);
+
+    virtual void resetOverriden() = 0;
+
+    virtual void toJSON(JSONObject & out) = 0;
+
+    virtual void convertToArgs(Args & args, const std::string & category) = 0;
+
+    void warnUnknownSettings();
+
+    void reapplyUnknownSettings();
+};
+
 /* A class to simplify providing configuration settings. The typical
    use is to inherit Config and add Setting<T> members:
 
@@ -27,7 +61,7 @@ class JSONObject;
    };
 */
 
-class Config
+class Config : public AbstractConfig
 {
     friend class AbstractSetting;
 
@@ -48,31 +82,23 @@ private:
 
     Settings _settings;
 
-    StringMap extras;
-
 public:
 
-    Config(const StringMap & initials)
-        : extras(initials)
+    Config(const StringMap & initials = {})
+        : AbstractConfig(initials)
     { }
 
-    void set(const std::string & name, const std::string & value);
+    bool set(const std::string & name, const std::string & value) override;
 
     void addSetting(AbstractSetting * setting);
 
-    void handleUnknownSettings();
-
-    StringMap getSettings(bool overridenOnly = false);
+    void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override;
 
-    const Settings & _getSettings() { return _settings; }
-
-    void applyConfigFile(const Path & path);
+    void resetOverriden() override;
 
-    void resetOverriden();
+    void toJSON(JSONObject & out) override;
 
-    void toJSON(JSONObject & out);
-
-    void convertToArgs(Args & args, const std::string & category);
+    void convertToArgs(Args & args, const std::string & category) override;
 };
 
 class AbstractSetting
@@ -209,4 +235,27 @@ public:
     void operator =(const Path & v) { this->assign(v); }
 };
 
+struct GlobalConfig : public AbstractConfig
+{
+    typedef std::vector<Config*> ConfigRegistrations;
+    static ConfigRegistrations * configRegistrations;
+
+    bool set(const std::string & name, const std::string & value) override;
+
+    void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override;
+
+    void resetOverriden() override;
+
+    void toJSON(JSONObject & out) override;
+
+    void convertToArgs(Args & args, const std::string & category) override;
+
+    struct Register
+    {
+        Register(Config * config);
+    };
+};
+
+extern GlobalConfig globalConfig;
+
 }
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index a01d651e1ef5..1c14ebb187cc 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -82,7 +82,7 @@ static string printHash32(const Hash & hash)
     string s;
     s.reserve(len);
 
-    for (int n = len - 1; n >= 0; n--) {
+    for (int n = (int) len - 1; n >= 0; n--) {
         unsigned int b = n * 5;
         unsigned int i = b / 8;
         unsigned int j = b % 8;
@@ -105,9 +105,9 @@ string printHash16or32(const Hash & hash)
 std::string Hash::to_string(Base base, bool includeType) const
 {
     std::string s;
-    if (includeType) {
+    if (base == SRI || includeType) {
         s += printHashType(type);
-        s += ':';
+        s += base == SRI ? '-' : ':';
     }
     switch (base) {
     case Base16:
@@ -117,6 +117,7 @@ std::string Hash::to_string(Base base, bool includeType) const
         s += printHash32(*this);
         break;
     case Base64:
+    case SRI:
         s += base64Encode(std::string((const char *) hash, hashSize));
         break;
     }
@@ -127,28 +128,33 @@ std::string Hash::to_string(Base base, bool includeType) const
 Hash::Hash(const std::string & s, HashType type)
     : type(type)
 {
-    auto colon = s.find(':');
-
     size_t pos = 0;
-
-    if (colon == string::npos) {
-        if (type == htUnknown)
+    bool isSRI = false;
+
+    auto sep = s.find(':');
+    if (sep == string::npos) {
+        sep = s.find('-');
+        if (sep != string::npos) {
+            isSRI = true;
+        } else if (type == htUnknown)
             throw BadHash("hash '%s' does not include a type", s);
-    } else {
-        string hts = string(s, 0, colon);
+    }
+
+    if (sep != string::npos) {
+        string hts = string(s, 0, sep);
         this->type = parseHashType(hts);
         if (this->type == htUnknown)
             throw BadHash("unknown hash type '%s'", hts);
         if (type != htUnknown && type != this->type)
             throw BadHash("hash '%s' should have type '%s'", s, printHashType(type));
-        pos = colon + 1;
+        pos = sep + 1;
     }
 
     init();
 
     size_t size = s.size() - pos;
 
-    if (size == base16Len()) {
+    if (!isSRI && size == base16Len()) {
 
         auto parseHexDigit = [&](char c) {
             if (c >= '0' && c <= '9') return c - '0';
@@ -164,7 +170,7 @@ Hash::Hash(const std::string & s, HashType type)
         }
     }
 
-    else if (size == base32Len()) {
+    else if (!isSRI && size == base32Len()) {
 
         for (unsigned int n = 0; n < size; ++n) {
             char c = s[pos + size - n - 1];
@@ -187,10 +193,10 @@ Hash::Hash(const std::string & s, HashType type)
         }
     }
 
-    else if (size == base64Len()) {
+    else if (isSRI || size == base64Len()) {
         auto d = base64Decode(std::string(s, pos));
         if (d.size() != hashSize)
-            throw BadHash("invalid base-64 hash '%s'", s);
+            throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s);
         assert(hashSize);
         memcpy(hash, d.data(), hashSize);
     }
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index fd7a61df8e46..2dbc3b630814 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -20,7 +20,7 @@ const int sha512HashSize = 64;
 
 extern const string base32Chars;
 
-enum Base : int { Base64, Base32, Base16 };
+enum Base : int { Base64, Base32, Base16, SRI };
 
 
 struct Hash
@@ -38,8 +38,9 @@ struct Hash
     Hash(HashType type) : type(type) { init(); };
 
     /* Initialize the hash from a string representation, in the format
-       "[<type>:]<base16|base32|base64>". If the 'type' argument is
-       htUnknown, then the hash type must be specified in the
+       "[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a
+       Subresource Integrity hash expression). If the 'type' argument
+       is htUnknown, then the hash type must be specified in the
        string. */
     Hash(const std::string & s, HashType type = htUnknown);
 
diff --git a/src/libutil/json.cc b/src/libutil/json.cc
index 813b257016e4..0a6fb65f0605 100644
--- a/src/libutil/json.cc
+++ b/src/libutil/json.cc
@@ -31,6 +31,7 @@ template<> void toJSON<unsigned long>(std::ostream & str, const unsigned long &
 template<> void toJSON<long long>(std::ostream & str, const long long & n) { str << n; }
 template<> void toJSON<unsigned long long>(std::ostream & str, const unsigned long long & n) { str << n; }
 template<> void toJSON<float>(std::ostream & str, const float & n) { str << n; }
+template<> void toJSON<double>(std::ostream & str, const double & n) { str << n; }
 
 template<> void toJSON<std::string>(std::ostream & str, const std::string & s)
 {
diff --git a/src/libutil/local.mk b/src/libutil/local.mk
index 824f48fbfc9f..3ccc23fd5c1b 100644
--- a/src/libutil/local.mk
+++ b/src/libutil/local.mk
@@ -7,7 +7,3 @@ libutil_DIR := $(d)
 libutil_SOURCES := $(wildcard $(d)/*.cc)
 
 libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) -lboost_context
-
-libutil_LIBS = libformat
-
-libutil_CXXFLAGS = -DBROTLI=\"$(brotli)\"
diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh
index 9b8290e634c9..8b83f842c324 100644
--- a/src/libutil/lru-cache.hh
+++ b/src/libutil/lru-cache.hh
@@ -2,7 +2,7 @@
 
 #include <map>
 #include <list>
-#include <experimental/optional>
+#include <optional>
 
 namespace nix {
 
@@ -64,7 +64,7 @@ public:
 
     /* Look up an item in the cache. If it exists, it becomes the most
        recently used item. */
-    std::experimental::optional<Value> get(const Key & key)
+    std::optional<Value> get(const Key & key)
     {
         auto i = data.find(key);
         if (i == data.end()) return {};
diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh
index 0b142b0597c7..d49067bb95dc 100644
--- a/src/libutil/pool.hh
+++ b/src/libutil/pool.hh
@@ -97,6 +97,7 @@ public:
     private:
         Pool & pool;
         std::shared_ptr<R> r;
+        bool bad = false;
 
         friend Pool;
 
@@ -112,7 +113,8 @@ public:
             if (!r) return;
             {
                 auto state_(pool.state.lock());
-                state_->idle.push_back(ref<R>(r));
+                if (!bad)
+                    state_->idle.push_back(ref<R>(r));
                 assert(state_->inUse);
                 state_->inUse--;
             }
@@ -121,6 +123,8 @@ public:
 
         R * operator -> () { return &*r; }
         R & operator * () { return *r; }
+
+        void markBad() { bad = true; }
     };
 
     Handle get()
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 33ae1ea389d7..8201549fd7d0 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -133,7 +133,7 @@ size_t FdSource::readUnbuffered(unsigned char * data, size_t len)
     ssize_t n;
     do {
         checkInterrupt();
-        n = ::read(fd, (char *) data, bufSize);
+        n = ::read(fd, (char *) data, len);
     } while (n == -1 && errno == EINTR);
     if (n == -1) { _good = false; throw SysError("reading from file"); }
     if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); }
@@ -157,21 +157,25 @@ size_t StringSource::read(unsigned char * data, size_t len)
 }
 
 
-std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun)
+#if BOOST_VERSION >= 106300 && BOOST_VERSION < 106600
+#error Coroutines are broken in this version of Boost!
+#endif
+
+std::unique_ptr<Source> sinkToSource(
+    std::function<void(Sink &)> fun,
+    std::function<void()> eof)
 {
     struct SinkToSource : Source
     {
         typedef boost::coroutines2::coroutine<std::string> coro_t;
 
-        coro_t::pull_type coro;
+        std::function<void(Sink &)> fun;
+        std::function<void()> eof;
+        std::optional<coro_t::pull_type> coro;
+        bool started = false;
 
-        SinkToSource(std::function<void(Sink &)> fun)
-            : coro([&](coro_t::push_type & yield) {
-                LambdaSink sink([&](const unsigned char * data, size_t len) {
-                    if (len) yield(std::string((const char *) data, len));
-                });
-                fun(sink);
-            })
+        SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof)
+            : fun(fun), eof(eof)
         {
         }
 
@@ -181,11 +185,18 @@ std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun)
         size_t read(unsigned char * data, size_t len) override
         {
             if (!coro)
-                throw EndOfFile("coroutine has finished");
+                coro = coro_t::pull_type([&](coro_t::push_type & yield) {
+                    LambdaSink sink([&](const unsigned char * data, size_t len) {
+                            if (len) yield(std::string((const char *) data, len));
+                        });
+                    fun(sink);
+                });
+
+            if (!*coro) { eof(); abort(); }
 
             if (pos == cur.size()) {
-                if (!cur.empty()) coro();
-                cur = coro.get();
+                if (!cur.empty()) (*coro)();
+                cur = coro->get();
                 pos = 0;
             }
 
@@ -197,7 +208,7 @@ std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun)
         }
     };
 
-    return std::make_unique<SinkToSource>(fun);
+    return std::make_unique<SinkToSource>(fun, eof);
 }
 
 
@@ -261,16 +272,17 @@ void readPadding(size_t len, Source & source)
 size_t readString(unsigned char * buf, size_t max, Source & source)
 {
     auto len = readNum<size_t>(source);
-    if (len > max) throw Error("string is too long");
+    if (len > max) throw SerialisationError("string is too long");
     source(buf, len);
     readPadding(len, source);
     return len;
 }
 
 
-string readString(Source & source)
+string readString(Source & source, size_t max)
 {
     auto len = readNum<size_t>(source);
+    if (len > max) throw SerialisationError("string is too long");
     std::string res(len, 0);
     source((unsigned char*) res.data(), len);
     readPadding(len, source);
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index d0b4552e3399..969e4dff383d 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -77,10 +77,12 @@ struct BufferedSource : Source
 
     size_t read(unsigned char * data, size_t len) override;
 
-    /* Underlying read call, to be overridden. */
-    virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0;
 
     bool hasData();
+
+protected:
+    /* Underlying read call, to be overridden. */
+    virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0;
 };
 
 
@@ -134,8 +136,9 @@ struct FdSource : BufferedSource
         return *this;
     }
 
-    size_t readUnbuffered(unsigned char * data, size_t len) override;
     bool good() override;
+protected:
+    size_t readUnbuffered(unsigned char * data, size_t len) override;
 private:
     bool _good = true;
 };
@@ -211,7 +214,11 @@ struct LambdaSource : Source
 
 /* Convert a function that feeds data into a Sink into a Source. The
    Source executes the function as a coroutine. */
-std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun);
+std::unique_ptr<Source> sinkToSource(
+    std::function<void(Sink &)> fun,
+    std::function<void()> eof = []() {
+        throw EndOfFile("coroutine has finished");
+    });
 
 
 void writePadding(size_t len, Sink & sink);
@@ -227,7 +234,7 @@ inline Sink & operator << (Sink & sink, uint64_t n)
     buf[4] = (n >> 32) & 0xff;
     buf[5] = (n >> 40) & 0xff;
     buf[6] = (n >> 48) & 0xff;
-    buf[7] = (n >> 56) & 0xff;
+    buf[7] = (unsigned char) (n >> 56) & 0xff;
     sink(buf, sizeof(buf));
     return sink;
 }
@@ -259,7 +266,7 @@ T readNum(Source & source)
     if (n > std::numeric_limits<T>::max())
         throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name());
 
-    return n;
+    return (T) n;
 }
 
 
@@ -277,7 +284,7 @@ inline uint64_t readLongLong(Source & source)
 
 void readPadding(size_t len, Source & source);
 size_t readString(unsigned char * buf, size_t max, Source & source);
-string readString(Source & source);
+string readString(Source & source, size_t max = std::numeric_limits<size_t>::max());
 template<class T> T readStrings(Source & source);
 
 Source & operator >> (Source & in, string & s);
diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh
index 3b2710f6fd30..e1d591d77a84 100644
--- a/src/libutil/sync.hh
+++ b/src/libutil/sync.hh
@@ -57,11 +57,11 @@ public:
         }
 
         template<class Rep, class Period>
-        void wait_for(std::condition_variable & cv,
+        std::cv_status wait_for(std::condition_variable & cv,
             const std::chrono::duration<Rep, Period> & duration)
         {
             assert(s);
-            cv.wait_for(lk, duration);
+            return cv.wait_for(lk, duration);
         }
 
         template<class Rep, class Period, class Predicate>
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 15962236ec65..17aee2d5c3d0 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -16,6 +16,7 @@
 #include <future>
 
 #include <fcntl.h>
+#include <grp.h>
 #include <limits.h>
 #include <pwd.h>
 #include <sys/ioctl.h>
@@ -38,6 +39,9 @@ extern char * * environ;
 namespace nix {
 
 
+const std::string nativeSystem = SYSTEM;
+
+
 BaseError & BaseError::addPrefix(const FormatOrString & fs)
 {
     prefix_ = fs.s + prefix_;
@@ -167,7 +171,7 @@ Path dirOf(const Path & path)
 {
     Path::size_type pos = path.rfind('/');
     if (pos == string::npos)
-        throw Error(format("invalid file name '%1%'") % path);
+        return ".";
     return pos == 0 ? "/" : Path(path, 0, pos);
 }
 
@@ -202,7 +206,7 @@ bool isInDir(const Path & path, const Path & dir)
 
 bool isDirOrInDir(const Path & path, const Path & dir)
 {
-    return path == dir or isInDir(path, dir);
+    return path == dir || isInDir(path, dir);
 }
 
 
@@ -311,6 +315,14 @@ string readFile(const Path & path, bool drain)
 }
 
 
+void readFile(const Path & path, Sink & sink)
+{
+    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
+    if (!fd) throw SysError("opening file '%s'", path);
+    drainFD(fd.get(), sink);
+}
+
+
 void writeFile(const Path & path, const string & s, mode_t mode)
 {
     AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
@@ -320,6 +332,23 @@ void writeFile(const Path & path, const string & s, mode_t mode)
 }
 
 
+void writeFile(const Path & path, Source & source, mode_t mode)
+{
+    AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
+    if (!fd)
+        throw SysError(format("opening file '%1%'") % path);
+
+    std::vector<unsigned char> buf(64 * 1024);
+
+    while (true) {
+        try {
+            auto n = source.read(buf.data(), buf.size());
+            writeFull(fd.get(), (unsigned char *) buf.data(), n);
+        } catch (EndOfFile &) { break; }
+    }
+}
+
+
 string readLine(int fd)
 {
     string s;
@@ -443,7 +472,7 @@ static Lazy<Path> getHome2([]() {
         std::vector<char> buf(16384);
         struct passwd pwbuf;
         struct passwd * pw;
-        if (getpwuid_r(getuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0
+        if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0
             || !pw || !pw->pw_dir || !pw->pw_dir[0])
             throw Error("cannot determine user's home directory");
         homeDir = pw->pw_dir;
@@ -471,6 +500,15 @@ Path getConfigDir()
     return configDir;
 }
 
+std::vector<Path> getConfigDirs()
+{
+    Path configHome = getConfigDir();
+    string configDirs = getEnv("XDG_CONFIG_DIRS");
+    std::vector<Path> result = tokenizeString<std::vector<string>>(configDirs, ":");
+    result.insert(result.begin(), configHome);
+    return result;
+}
+
 
 Path getDataDir()
 {
@@ -593,7 +631,7 @@ void drainFD(int fd, Sink & sink, bool block)
             throw SysError("making file descriptor non-blocking");
     }
 
-    std::vector<unsigned char> buf(4096);
+    std::vector<unsigned char> buf(64 * 1024);
     while (1) {
         checkInterrupt();
         ssize_t rd = read(fd, buf.data(), buf.size());
@@ -931,7 +969,7 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss)
 
 
 string runProgram(Path program, bool searchPath, const Strings & args,
-    const std::experimental::optional<std::string> & input)
+    const std::optional<std::string> & input)
 {
     RunOptions opts(program, args);
     opts.searchPath = searchPath;
@@ -988,6 +1026,16 @@ void runProgram2(const RunOptions & options)
         if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1)
             throw SysError("dupping stdin");
 
+        if (options.chdir && chdir((*options.chdir).c_str()) == -1)
+            throw SysError("chdir failed");
+        if (options.gid && setgid(*options.gid) == -1)
+            throw SysError("setgid failed");
+        /* Drop all other groups if we're setgid. */
+        if (options.gid && setgroups(0, 0) == -1)
+            throw SysError("setgroups failed");
+        if (options.uid && setuid(*options.uid) == -1)
+            throw SysError("setuid failed");
+
         Strings args_(options.args);
         args_.push_front(options.program);
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 743d238611fc..fce3cab8def5 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -14,7 +14,8 @@
 #include <cstdio>
 #include <map>
 #include <sstream>
-#include <experimental/optional>
+#include <optional>
+#include <future>
 
 #ifndef HAVE_STRUCT_DIRENT_D_TYPE
 #define DT_UNKNOWN 0
@@ -29,6 +30,10 @@ struct Sink;
 struct Source;
 
 
+/* The system for which Nix is compiled. */
+extern const std::string nativeSystem;
+
+
 /* Return an environment variable. */
 string getEnv(const string & key, const string & def = "");
 
@@ -97,10 +102,13 @@ unsigned char getFileType(const Path & path);
 /* Read the contents of a file into a string. */
 string readFile(int fd);
 string readFile(const Path & path, bool drain = false);
+void readFile(const Path & path, Sink & sink);
 
 /* Write a string to a file. */
 void writeFile(const Path & path, const string & s, mode_t mode = 0666);
 
+void writeFile(const Path & path, Source & source, mode_t mode = 0666);
+
 /* Read a line from a file descriptor. */
 string readLine(int fd);
 
@@ -127,6 +135,9 @@ Path getCacheDir();
 /* Return $XDG_CONFIG_HOME or $HOME/.config. */
 Path getConfigDir();
 
+/* Return the directories to search for user configuration files */
+std::vector<Path> getConfigDirs();
+
 /* Return $XDG_DATA_HOME or $HOME/.local/share. */
 Path getDataDir();
 
@@ -252,14 +263,17 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = P
    shell backtick operator). */
 string runProgram(Path program, bool searchPath = false,
     const Strings & args = Strings(),
-    const std::experimental::optional<std::string> & input = {});
+    const std::optional<std::string> & input = {});
 
 struct RunOptions
 {
+    std::optional<uid_t> uid;
+    std::optional<uid_t> gid;
+    std::optional<Path> chdir;
     Path program;
     bool searchPath = true;
     Strings args;
-    std::experimental::optional<std::string> input;
+    std::optional<std::string> input;
     Source * standardIn = nullptr;
     Sink * standardOut = nullptr;
     bool _killStderr = false;
@@ -394,6 +408,7 @@ void ignoreException();
 /* Some ANSI escape sequences. */
 #define ANSI_NORMAL "\e[0m"
 #define ANSI_BOLD "\e[1m"
+#define ANSI_FAINT "\e[2m"
 #define ANSI_RED "\e[31;1m"
 #define ANSI_GREEN "\e[32;1m"
 #define ANSI_BLUE "\e[34;1m"
@@ -424,44 +439,30 @@ string get(const T & map, const string & key, const string & def = "")
 }
 
 
-/* Call โ€˜failureโ€™ with the current exception as argument. If โ€˜failureโ€™
-   throws an exception, abort the program. */
-void callFailure(const std::function<void(std::exception_ptr exc)> & failure,
-    std::exception_ptr exc = std::current_exception());
+/* A callback is a wrapper around a lambda that accepts a valid of
+   type T or an exception. (We abuse std::future<T> to pass the value or
+   exception.) */
+template<typename T>
+struct Callback
+{
+    std::function<void(std::future<T>)> fun;
 
+    Callback(std::function<void(std::future<T>)> fun) : fun(fun) { }
 
-/* Evaluate the function โ€˜fโ€™. If it returns a value, call โ€˜successโ€™
-   with that value as its argument. If it or โ€˜successโ€™ throws an
-   exception, call โ€˜failureโ€™. If โ€˜failureโ€™ throws an exception, abort
-   the program. */
-template<class T>
-void sync2async(
-    const std::function<void(T)> & success,
-    const std::function<void(std::exception_ptr exc)> & failure,
-    const std::function<T()> & f)
-{
-    try {
-        success(f());
-    } catch (...) {
-        callFailure(failure);
+    void operator()(T && t) const
+    {
+        std::promise<T> promise;
+        promise.set_value(std::move(t));
+        fun(promise.get_future());
     }
-}
-
 
-/* Call the function โ€˜successโ€™. If it throws an exception, call
-   โ€˜failureโ€™. If that throws an exception, abort the program. */
-template<class T>
-void callSuccess(
-    const std::function<void(T)> & success,
-    const std::function<void(std::exception_ptr exc)> & failure,
-    T && arg)
-{
-    try {
-        success(arg);
-    } catch (...) {
-        callFailure(failure);
+    void rethrow(const std::exception_ptr & exc = std::current_exception()) const
+    {
+        std::promise<T> promise;
+        promise.set_exception(exc);
+        fun(promise.get_future());
     }
-}
+};
 
 
 /* Start a thread that handles various signals. Also block those signals
diff --git a/src/libutil/xml-writer.cc b/src/libutil/xml-writer.cc
index 98bd058d18be..e5cc2e9fc719 100644
--- a/src/libutil/xml-writer.cc
+++ b/src/libutil/xml-writer.cc
@@ -28,7 +28,7 @@ void XMLWriter::close()
 }
 
 
-void XMLWriter::indent_(unsigned int depth)
+void XMLWriter::indent_(size_t depth)
 {
     if (!indent) return;
     output << string(depth * 2, ' ');
@@ -75,7 +75,7 @@ void XMLWriter::writeAttrs(const XMLAttrs & attrs)
 {
     for (auto & i : attrs) {
         output << " " << i.first << "=\"";
-        for (unsigned int j = 0; j < i.second.size(); ++j) {
+        for (size_t j = 0; j < i.second.size(); ++j) {
             char c = i.second[j];
             if (c == '"') output << "&quot;";
             else if (c == '<') output << "&lt;";
diff --git a/src/libutil/xml-writer.hh b/src/libutil/xml-writer.hh
index 3cefe3712c08..b98b445265a2 100644
--- a/src/libutil/xml-writer.hh
+++ b/src/libutil/xml-writer.hh
@@ -44,7 +44,7 @@ public:
 private:
     void writeAttrs(const XMLAttrs & attrs);
 
-    void indent_(unsigned int depth);
+    void indent_(size_t depth);
 };
 
 
diff --git a/src/linenoise/ConvertUTF.cpp b/src/linenoise/ConvertUTF.cpp
deleted file mode 100644
index f7e5915d5e8f..000000000000
--- a/src/linenoise/ConvertUTF.cpp
+++ /dev/null
@@ -1,542 +0,0 @@
-/*
- * Copyright 2001-2004 Unicode, Inc.
- * 
- * Disclaimer
- * 
- * This source code is provided as is by Unicode, Inc. No claims are
- * made as to fitness for any particular purpose. No warranties of any
- * kind are expressed or implied. The recipient agrees to determine
- * applicability of information provided. If this file has been
- * purchased on magnetic or optical media from Unicode, Inc., the
- * sole remedy for any claim will be exchange of defective media
- * within 90 days of receipt.
- * 
- * Limitations on Rights to Redistribute This Code
- * 
- * Unicode, Inc. hereby grants the right to freely use the information
- * supplied in this file in the creation of products supporting the
- * Unicode Standard, and to make copies of this file in any form
- * for internal or external distribution as long as this notice
- * remains attached.
- */
-
-/* ---------------------------------------------------------------------
-
-    Conversions between UTF32, UTF-16, and UTF-8. Source code file.
-    Author: Mark E. Davis, 1994.
-    Rev History: Rick McGowan, fixes & updates May 2001.
-    Sept 2001: fixed const & error conditions per
-        mods suggested by S. Parent & A. Lillich.
-    June 2002: Tim Dodd added detection and handling of incomplete
-        source sequences, enhanced error detection, added casts
-        to eliminate compiler warnings.
-    July 2003: slight mods to back out aggressive FFFE detection.
-    Jan 2004: updated switches in from-UTF8 conversions.
-    Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions.
-
-    See the header file "ConvertUTF.h" for complete documentation.
-
------------------------------------------------------------------------- */
-
-#include "ConvertUTF.h"
-#ifdef CVTUTF_DEBUG
-#include <stdio.h>
-#endif
-
-namespace linenoise_ng {
-
-static const int halfShift  = 10; /* used for shifting by 10 bits */
-
-static const UTF32 halfBase = 0x0010000UL;
-static const UTF32 halfMask = 0x3FFUL;
-
-#define UNI_SUR_HIGH_START  (UTF32)0xD800
-#define UNI_SUR_HIGH_END    (UTF32)0xDBFF
-#define UNI_SUR_LOW_START   (UTF32)0xDC00
-#define UNI_SUR_LOW_END     (UTF32)0xDFFF
-#define false      0
-#define true        1
-
-/* --------------------------------------------------------------------- */
-
-ConversionResult ConvertUTF32toUTF16 (
-        const UTF32** sourceStart, const UTF32* sourceEnd, 
-        char16_t** targetStart, char16_t* targetEnd, ConversionFlags flags) {
-    ConversionResult result = conversionOK;
-    const UTF32* source = *sourceStart;
-    char16_t* target = *targetStart;
-    while (source < sourceEnd) {
-        UTF32 ch;
-        if (target >= targetEnd) {
-            result = targetExhausted; break;
-        }
-        ch = *source++;
-        if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
-            /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */
-            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
-                if (flags == strictConversion) {
-                    --source; /* return to the illegal value itself */
-                    result = sourceIllegal;
-                    break;
-                } else {
-                    *target++ = UNI_REPLACEMENT_CHAR;
-                }
-            } else {
-                *target++ = (UTF16)ch; /* normal case */
-            }
-        } else if (ch > UNI_MAX_LEGAL_UTF32) {
-            if (flags == strictConversion) {
-                result = sourceIllegal;
-            } else {
-                *target++ = UNI_REPLACEMENT_CHAR;
-            }
-        } else {
-            /* target is a character in range 0xFFFF - 0x10FFFF. */
-            if (target + 1 >= targetEnd) {
-                --source; /* Back up source pointer! */
-                result = targetExhausted; break;
-            }
-            ch -= halfBase;
-            *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
-            *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
-        }
-    }
-    *sourceStart = source;
-    *targetStart = target;
-    return result;
-}
-
-/* --------------------------------------------------------------------- */
-
-ConversionResult ConvertUTF16toUTF32 (
-        const UTF16** sourceStart, const UTF16* sourceEnd, 
-        UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
-    ConversionResult result = conversionOK;
-    const UTF16* source = *sourceStart;
-    UTF32* target = *targetStart;
-    UTF32 ch, ch2;
-    while (source < sourceEnd) {
-        const UTF16* oldSource = source; /*  In case we have to back up because of target overflow. */
-        ch = *source++;
-        /* If we have a surrogate pair, convert to UTF32 first. */
-        if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
-            /* If the 16 bits following the high surrogate are in the source buffer... */
-            if (source < sourceEnd) {
-                ch2 = *source;
-                /* If it's a low surrogate, convert to UTF32. */
-                if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
-                    ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
-                        + (ch2 - UNI_SUR_LOW_START) + halfBase;
-                    ++source;
-                } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
-                    --source; /* return to the illegal value itself */
-                    result = sourceIllegal;
-                    break;
-                }
-            } else { /* We don't have the 16 bits following the high surrogate. */
-                --source; /* return to the high surrogate */
-                result = sourceExhausted;
-                break;
-            }
-        } else if (flags == strictConversion) {
-            /* UTF-16 surrogate values are illegal in UTF-32 */
-            if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
-                --source; /* return to the illegal value itself */
-                result = sourceIllegal;
-                break;
-            }
-        }
-        if (target >= targetEnd) {
-            source = oldSource; /* Back up source pointer! */
-            result = targetExhausted; break;
-        }
-        *target++ = ch;
-    }
-    *sourceStart = source;
-    *targetStart = target;
-#ifdef CVTUTF_DEBUG
-if (result == sourceIllegal) {
-    fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2);
-    fflush(stderr);
-}
-#endif
-    return result;
-}
-
-/* --------------------------------------------------------------------- */
-
-/*
- * Index into the table below with the first byte of a UTF-8 sequence to
- * get the number of trailing bytes that are supposed to follow it.
- * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
- * left as-is for anyone who may want to do such conversion, which was
- * allowed in earlier algorithms.
- */
-static const char trailingBytesForUTF8[256] = {
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
-};
-
-/*
- * Magic values subtracted from a buffer value during UTF8 conversion.
- * This table contains as many values as there might be trailing bytes
- * in a UTF-8 sequence.
- */
-static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, 
-                     0x03C82080UL, 0xFA082080UL, 0x82082080UL };
-
-/*
- * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
- * into the first byte, depending on how many bytes follow.  There are
- * as many entries in this table as there are UTF-8 sequence types.
- * (I.e., one byte sequence, two byte... etc.). Remember that sequencs
- * for *legal* UTF-8 will be 4 or fewer bytes total.
- */
-static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
-
-/* --------------------------------------------------------------------- */
-
-/* The interface converts a whole buffer to avoid function-call overhead.
- * Constants have been gathered. Loops & conditionals have been removed as
- * much as possible for efficiency, in favor of drop-through switches.
- * (See "Note A" at the bottom of the file for equivalent code.)
- * If your compiler supports it, the "isLegalUTF8" call can be turned
- * into an inline function.
- */
-
-/* --------------------------------------------------------------------- */
-
-ConversionResult ConvertUTF16toUTF8 (
-        const UTF16** sourceStart, const UTF16* sourceEnd, 
-        UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
-    ConversionResult result = conversionOK;
-    const UTF16* source = *sourceStart;
-    UTF8* target = *targetStart;
-    while (source < sourceEnd) {
-        UTF32 ch;
-        unsigned short bytesToWrite = 0;
-        const UTF32 byteMask = 0xBF;
-        const UTF32 byteMark = 0x80; 
-        const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */
-        ch = *source++;
-        /* If we have a surrogate pair, convert to UTF32 first. */
-        if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
-            /* If the 16 bits following the high surrogate are in the source buffer... */
-            if (source < sourceEnd) {
-                UTF32 ch2 = *source;
-                /* If it's a low surrogate, convert to UTF32. */
-                if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
-                    ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
-                        + (ch2 - UNI_SUR_LOW_START) + halfBase;
-                    ++source;
-                } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
-                    --source; /* return to the illegal value itself */
-                    result = sourceIllegal;
-                    break;
-                }
-            } else { /* We don't have the 16 bits following the high surrogate. */
-                --source; /* return to the high surrogate */
-                result = sourceExhausted;
-                break;
-            }
-        } else if (flags == strictConversion) {
-            /* UTF-16 surrogate values are illegal in UTF-32 */
-            if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
-                --source; /* return to the illegal value itself */
-                result = sourceIllegal;
-                break;
-            }
-        }
-        /* Figure out how many bytes the result will require */
-        if (ch < (UTF32)0x80) {      bytesToWrite = 1;
-        } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
-        } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
-        } else if (ch < (UTF32)0x110000) {  bytesToWrite = 4;
-        } else {                            bytesToWrite = 3;
-                                            ch = UNI_REPLACEMENT_CHAR;
-        }
-
-        target += bytesToWrite;
-        if (target > targetEnd) {
-            source = oldSource; /* Back up source pointer! */
-            target -= bytesToWrite; result = targetExhausted; break;
-        }
-        switch (bytesToWrite) { /* note: everything falls through. */
-            case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
-            case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
-            case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
-            case 1: *--target =  (UTF8)(ch | firstByteMark[bytesToWrite]);
-        }
-        target += bytesToWrite;
-    }
-    *sourceStart = source;
-    *targetStart = target;
-    return result;
-}
-
-/* --------------------------------------------------------------------- */
-
-/*
- * Utility routine to tell whether a sequence of bytes is legal UTF-8.
- * This must be called with the length pre-determined by the first byte.
- * If not calling this from ConvertUTF8to*, then the length can be set by:
- *  length = trailingBytesForUTF8[*source]+1;
- * and the sequence is illegal right away if there aren't that many bytes
- * available.
- * If presented with a length > 4, this returns false.  The Unicode
- * definition of UTF-8 goes up to 4-byte sequences.
- */
-
-static Boolean isLegalUTF8(const UTF8 *source, int length) {
-    UTF8 a;
-    const UTF8 *srcptr = source+length;
-    switch (length) {
-    default: return false;
-        /* Everything else falls through when "true"... */
-    case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
-    case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
-    case 2: if ((a = (*--srcptr)) > 0xBF) return false;
-
-        switch (*source) {
-            /* no fall-through in this inner switch */
-            case 0xE0: if (a < 0xA0) return false; break;
-            case 0xED: if (a > 0x9F) return false; break;
-            case 0xF0: if (a < 0x90) return false; break;
-            case 0xF4: if (a > 0x8F) return false; break;
-            default:   if (a < 0x80) return false;
-        }
-
-    case 1: if (*source >= 0x80 && *source < 0xC2) return false;
-    }
-    if (*source > 0xF4) return false;
-    return true;
-}
-
-/* --------------------------------------------------------------------- */
-
-/*
- * Exported function to return whether a UTF-8 sequence is legal or not.
- * This is not used here; it's just exported.
- */
-Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) {
-    int length = trailingBytesForUTF8[*source]+1;
-    if (source+length > sourceEnd) {
-        return false;
-    }
-    return isLegalUTF8(source, length);
-}
-
-/* --------------------------------------------------------------------- */
-
-ConversionResult ConvertUTF8toUTF16 (
-        const UTF8** sourceStart, const UTF8* sourceEnd, 
-        UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
-    ConversionResult result = conversionOK;
-    const UTF8* source = *sourceStart;
-    UTF16* target = *targetStart;
-    while (source < sourceEnd) {
-        UTF32 ch = 0;
-        unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
-        if (source + extraBytesToRead >= sourceEnd) {
-            result = sourceExhausted; break;
-        }
-        /* Do this check whether lenient or strict */
-        if (! isLegalUTF8(source, extraBytesToRead+1)) {
-            result = sourceIllegal;
-            break;
-        }
-        /*
-         * The cases all fall through. See "Note A" below.
-         */
-        switch (extraBytesToRead) {
-            case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
-            case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
-            case 3: ch += *source++; ch <<= 6;
-            case 2: ch += *source++; ch <<= 6;
-            case 1: ch += *source++; ch <<= 6;
-            case 0: ch += *source++;
-        }
-        ch -= offsetsFromUTF8[extraBytesToRead];
-
-        if (target >= targetEnd) {
-            source -= (extraBytesToRead+1); /* Back up source pointer! */
-            result = targetExhausted; break;
-        }
-        if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
-            /* UTF-16 surrogate values are illegal in UTF-32 */
-            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
-                if (flags == strictConversion) {
-                    source -= (extraBytesToRead+1); /* return to the illegal value itself */
-                    result = sourceIllegal;
-                    break;
-                } else {
-                    *target++ = UNI_REPLACEMENT_CHAR;
-                }
-            } else {
-                *target++ = (UTF16)ch; /* normal case */
-            }
-        } else if (ch > UNI_MAX_UTF16) {
-            if (flags == strictConversion) {
-                result = sourceIllegal;
-                source -= (extraBytesToRead+1); /* return to the start */
-                break; /* Bail out; shouldn't continue */
-            } else {
-                *target++ = UNI_REPLACEMENT_CHAR;
-            }
-        } else {
-            /* target is a character in range 0xFFFF - 0x10FFFF. */
-            if (target + 1 >= targetEnd) {
-                source -= (extraBytesToRead+1); /* Back up source pointer! */
-                result = targetExhausted; break;
-            }
-            ch -= halfBase;
-            *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
-            *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
-        }
-    }
-    *sourceStart = source;
-    *targetStart = target;
-    return result;
-}
-
-/* --------------------------------------------------------------------- */
-
-ConversionResult ConvertUTF32toUTF8 (
-        const UTF32** sourceStart, const UTF32* sourceEnd, 
-        UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
-    ConversionResult result = conversionOK;
-    const UTF32* source = *sourceStart;
-    UTF8* target = *targetStart;
-    while (source < sourceEnd) {
-        UTF32 ch;
-        unsigned short bytesToWrite = 0;
-        const UTF32 byteMask = 0xBF;
-        const UTF32 byteMark = 0x80; 
-        ch = *source++;
-        if (flags == strictConversion ) {
-            /* UTF-16 surrogate values are illegal in UTF-32 */
-            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
-                --source; /* return to the illegal value itself */
-                result = sourceIllegal;
-                break;
-            }
-        }
-        /*
-         * Figure out how many bytes the result will require. Turn any
-         * illegally large UTF32 things (> Plane 17) into replacement chars.
-         */
-        if (ch < (UTF32)0x80) {      bytesToWrite = 1;
-        } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
-        } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
-        } else if (ch <= UNI_MAX_LEGAL_UTF32) {  bytesToWrite = 4;
-        } else {                            bytesToWrite = 3;
-                                            ch = UNI_REPLACEMENT_CHAR;
-                                            result = sourceIllegal;
-        }
-        
-        target += bytesToWrite;
-        if (target > targetEnd) {
-            --source; /* Back up source pointer! */
-            target -= bytesToWrite; result = targetExhausted; break;
-        }
-        switch (bytesToWrite) { /* note: everything falls through. */
-            case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
-            case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
-            case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
-            case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]);
-        }
-        target += bytesToWrite;
-    }
-    *sourceStart = source;
-    *targetStart = target;
-    return result;
-}
-
-/* --------------------------------------------------------------------- */
-
-ConversionResult ConvertUTF8toUTF32 (
-        const UTF8** sourceStart, const UTF8* sourceEnd, 
-        UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
-    ConversionResult result = conversionOK;
-    const UTF8* source = *sourceStart;
-    UTF32* target = *targetStart;
-    while (source < sourceEnd) {
-        UTF32 ch = 0;
-        unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
-        if (source + extraBytesToRead >= sourceEnd) {
-            result = sourceExhausted; break;
-        }
-        /* Do this check whether lenient or strict */
-        if (! isLegalUTF8(source, extraBytesToRead+1)) {
-            result = sourceIllegal;
-            break;
-        }
-        /*
-         * The cases all fall through. See "Note A" below.
-         */
-        switch (extraBytesToRead) {
-            case 5: ch += *source++; ch <<= 6;
-            case 4: ch += *source++; ch <<= 6;
-            case 3: ch += *source++; ch <<= 6;
-            case 2: ch += *source++; ch <<= 6;
-            case 1: ch += *source++; ch <<= 6;
-            case 0: ch += *source++;
-        }
-        ch -= offsetsFromUTF8[extraBytesToRead];
-
-        if (target >= targetEnd) {
-            source -= (extraBytesToRead+1); /* Back up the source pointer! */
-            result = targetExhausted; break;
-        }
-        if (ch <= UNI_MAX_LEGAL_UTF32) {
-            /*
-             * UTF-16 surrogate values are illegal in UTF-32, and anything
-             * over Plane 17 (> 0x10FFFF) is illegal.
-             */
-            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
-                if (flags == strictConversion) {
-                    source -= (extraBytesToRead+1); /* return to the illegal value itself */
-                    result = sourceIllegal;
-                    break;
-                } else {
-                    *target++ = UNI_REPLACEMENT_CHAR;
-                }
-            } else {
-                *target++ = ch;
-            }
-        } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */
-            result = sourceIllegal;
-            *target++ = UNI_REPLACEMENT_CHAR;
-        }
-    }
-    *sourceStart = source;
-    *targetStart = target;
-    return result;
-}
-
-}
-
-/* ---------------------------------------------------------------------
-
-    Note A.
-    The fall-through switches in UTF-8 reading code save a
-    temp variable, some decrements & conditionals.  The switches
-    are equivalent to the following loop:
-        {
-            int tmpBytesToRead = extraBytesToRead+1;
-            do {
-                ch += *source++;
-                --tmpBytesToRead;
-                if (tmpBytesToRead) ch <<= 6;
-            } while (tmpBytesToRead > 0);
-        }
-    In UTF-8 writing code, the switches on "bytesToWrite" are
-    similarly unrolled loops.
-
-   --------------------------------------------------------------------- */
diff --git a/src/linenoise/ConvertUTF.h b/src/linenoise/ConvertUTF.h
deleted file mode 100755
index 8a296235dcd9..000000000000
--- a/src/linenoise/ConvertUTF.h
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Copyright 2001-2004 Unicode, Inc.
- * 
- * Disclaimer
- * 
- * This source code is provided as is by Unicode, Inc. No claims are
- * made as to fitness for any particular purpose. No warranties of any
- * kind are expressed or implied. The recipient agrees to determine
- * applicability of information provided. If this file has been
- * purchased on magnetic or optical media from Unicode, Inc., the
- * sole remedy for any claim will be exchange of defective media
- * within 90 days of receipt.
- * 
- * Limitations on Rights to Redistribute This Code
- * 
- * Unicode, Inc. hereby grants the right to freely use the information
- * supplied in this file in the creation of products supporting the
- * Unicode Standard, and to make copies of this file in any form
- * for internal or external distribution as long as this notice
- * remains attached.
- */
-
-/* ---------------------------------------------------------------------
-
-    Conversions between UTF32, UTF-16, and UTF-8.  Header file.
-
-    Several funtions are included here, forming a complete set of
-    conversions between the three formats.  UTF-7 is not included
-    here, but is handled in a separate source file.
-
-    Each of these routines takes pointers to input buffers and output
-    buffers.  The input buffers are const.
-
-    Each routine converts the text between *sourceStart and sourceEnd,
-    putting the result into the buffer between *targetStart and
-    targetEnd. Note: the end pointers are *after* the last item: e.g. 
-    *(sourceEnd - 1) is the last item.
-
-    The return result indicates whether the conversion was successful,
-    and if not, whether the problem was in the source or target buffers.
-    (Only the first encountered problem is indicated.)
-
-    After the conversion, *sourceStart and *targetStart are both
-    updated to point to the end of last text successfully converted in
-    the respective buffers.
-
-    Input parameters:
-        sourceStart - pointer to a pointer to the source buffer.
-                The contents of this are modified on return so that
-                it points at the next thing to be converted.
-        targetStart - similarly, pointer to pointer to the target buffer.
-        sourceEnd, targetEnd - respectively pointers to the ends of the
-                two buffers, for overflow checking only.
-
-    These conversion functions take a ConversionFlags argument. When this
-    flag is set to strict, both irregular sequences and isolated surrogates
-    will cause an error.  When the flag is set to lenient, both irregular
-    sequences and isolated surrogates are converted.
-
-    Whether the flag is strict or lenient, all illegal sequences will cause
-    an error return. This includes sequences such as: <F4 90 80 80>, <C0 80>,
-    or <A0> in UTF-8, and values above 0x10FFFF in UTF-32. Conformant code
-    must check for illegal sequences.
-
-    When the flag is set to lenient, characters over 0x10FFFF are converted
-    to the replacement character; otherwise (when the flag is set to strict)
-    they constitute an error.
-
-    Output parameters:
-        The value "sourceIllegal" is returned from some routines if the input
-        sequence is malformed.  When "sourceIllegal" is returned, the source
-        value will point to the illegal value that caused the problem. E.g.,
-        in UTF-8 when a sequence is malformed, it points to the start of the
-        malformed sequence.  
-
-    Author: Mark E. Davis, 1994.
-    Rev History: Rick McGowan, fixes & updates May 2001.
-                 Fixes & updates, Sept 2001.
-
------------------------------------------------------------------------- */
-
-/* ---------------------------------------------------------------------
-    The following 4 definitions are compiler-specific.
-    The C standard does not guarantee that wchar_t has at least
-    16 bits, so wchar_t is no less portable than unsigned short!
-    All should be unsigned values to avoid sign extension during
-    bit mask & shift operations.
------------------------------------------------------------------------- */
-
-#if 0
-typedef unsigned long  UTF32; /* at least 32 bits */
-typedef unsigned short UTF16; /* at least 16 bits */
-typedef unsigned char  UTF8;  /* typically 8 bits */
-#endif
-
-#include <stdint.h>
-#include <string>
-
-namespace linenoise_ng {
-
-typedef uint32_t         UTF32;
-typedef uint16_t         UTF16;
-typedef uint8_t          UTF8;
-typedef unsigned char    Boolean; /* 0 or 1 */
-
-/* Some fundamental constants */
-#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD
-#define UNI_MAX_BMP (UTF32)0x0000FFFF
-#define UNI_MAX_UTF16 (UTF32)0x0010FFFF
-#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF
-#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF
-
-typedef enum {
-        conversionOK,           /* conversion successful */
-        sourceExhausted,        /* partial character in source, but hit end */
-        targetExhausted,        /* insuff. room in target for conversion */
-        sourceIllegal           /* source sequence is illegal/malformed */
-} ConversionResult;
-
-typedef enum {
-        strictConversion = 0,
-        lenientConversion
-} ConversionFlags;
-
-// /* This is for C++ and does no harm in C */
-// #ifdef __cplusplus
-// extern "C" {
-// #endif
-
-ConversionResult ConvertUTF8toUTF16 (
-                const UTF8** sourceStart, const UTF8* sourceEnd, 
-                UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags);
-
-ConversionResult ConvertUTF16toUTF8 (
-                const UTF16** sourceStart, const UTF16* sourceEnd, 
-                UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags);
-                
-ConversionResult ConvertUTF8toUTF32 (
-                const UTF8** sourceStart, const UTF8* sourceEnd, 
-                UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags);
-
-ConversionResult ConvertUTF32toUTF8 (
-                const UTF32** sourceStart, const UTF32* sourceEnd, 
-                UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags);
-                
-ConversionResult ConvertUTF16toUTF32 (
-                const UTF16** sourceStart, const UTF16* sourceEnd, 
-                UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags);
-
-ConversionResult ConvertUTF32toUTF16 (
-                const UTF32** sourceStart, const UTF32* sourceEnd, 
-                char16_t** targetStart, char16_t* targetEnd, ConversionFlags flags);
-
-Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd);
-
-// #ifdef __cplusplus
-// }
-// #endif
-
-}
-
-/* --------------------------------------------------------------------- */
diff --git a/src/linenoise/LICENSE b/src/linenoise/LICENSE
deleted file mode 100644
index b7c58c445860..000000000000
--- a/src/linenoise/LICENSE
+++ /dev/null
@@ -1,66 +0,0 @@
-linenoise.cpp
-=============
-
-Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com>
-Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com>
-
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-  * Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-  * Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-  * Neither the name of Redis nor the names of its contributors may be used
-    to endorse or promote products derived from this software without
-    specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-
-wcwidth.cpp
-===========
-
-Markus Kuhn -- 2007-05-26 (Unicode 5.0)
-
-Permission to use, copy, modify, and distribute this software
-for any purpose and without fee is hereby granted. The author
-disclaims all warranties with regard to this software.
-
-
-
-ConvertUTF.cpp
-==============
-
-Copyright 2001-2004 Unicode, Inc.
- 
-Disclaimer
- 
-This source code is provided as is by Unicode, Inc. No claims are
-made as to fitness for any particular purpose. No warranties of any
-kind are expressed or implied. The recipient agrees to determine
-applicability of information provided. If this file has been
-purchased on magnetic or optical media from Unicode, Inc., the
-sole remedy for any claim will be exchange of defective media
-within 90 days of receipt.
- 
-Limitations on Rights to Redistribute This Code
- 
-Unicode, Inc. hereby grants the right to freely use the information
-supplied in this file in the creation of products supporting the
-Unicode Standard, and to make copies of this file in any form
-for internal or external distribution as long as this notice
-remains attached.
diff --git a/src/linenoise/linenoise.cpp b/src/linenoise/linenoise.cpp
deleted file mode 100644
index c57505d2fa97..000000000000
--- a/src/linenoise/linenoise.cpp
+++ /dev/null
@@ -1,3450 +0,0 @@
-/* linenoise.c -- guerrilla line editing library against the idea that a
- * line editing lib needs to be 20,000 lines of C code.
- *
- * Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- *
- * line editing lib needs to be 20,000 lines of C code.
- *
- * You can find the latest source code at:
- *
- *   http://github.com/antirez/linenoise
- *
- * Does a number of crazy assumptions that happen to be true in 99.9999% of
- * the 2010 UNIX computers around.
- *
- * References:
- * - http://invisible-island.net/xterm/ctlseqs/ctlseqs.html
- * - http://www.3waylabs.com/nw/WWW/products/wizcon/vt220.html
- *
- * Todo list:
- * - Switch to gets() if $TERM is something we can't support.
- * - Filter bogus Ctrl+<char> combinations.
- * - Win32 support
- *
- * Bloat:
- * - Completion?
- * - History search like Ctrl+r in readline?
- *
- * List of escape sequences used by this program, we do everything just
- * with three sequences. In order to be so cheap we may have some
- * flickering effect with some slow terminal, but the lesser sequences
- * the more compatible.
- *
- * CHA (Cursor Horizontal Absolute)
- *    Sequence: ESC [ n G
- *    Effect: moves cursor to column n (1 based)
- *
- * EL (Erase Line)
- *    Sequence: ESC [ n K
- *    Effect: if n is 0 or missing, clear from cursor to end of line
- *    Effect: if n is 1, clear from beginning of line to cursor
- *    Effect: if n is 2, clear entire line
- *
- * CUF (Cursor Forward)
- *    Sequence: ESC [ n C
- *    Effect: moves cursor forward of n chars
- *
- * The following are used to clear the screen: ESC [ H ESC [ 2 J
- * This is actually composed of two sequences:
- *
- * cursorhome
- *    Sequence: ESC [ H
- *    Effect: moves the cursor to upper left corner
- *
- * ED2 (Clear entire screen)
- *    Sequence: ESC [ 2 J
- *    Effect: clear the whole screen
- *
- */
-
-#ifdef _WIN32
-
-#include <conio.h>
-#include <windows.h>
-#include <io.h>
-
-#if defined(_MSC_VER) && _MSC_VER < 1900
-#define snprintf _snprintf  // Microsoft headers use underscores in some names
-#endif
-
-#if !defined GNUC
-#define strcasecmp _stricmp
-#endif
-
-#define strdup _strdup
-#define isatty _isatty
-#define write _write
-#define STDIN_FILENO 0
-
-#else /* _WIN32 */
-
-#include <signal.h>
-#include <termios.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/ioctl.h>
-#include <cctype>
-#include <wctype.h>
-
-#endif /* _WIN32 */
-
-#include <stdio.h>
-#include <errno.h>
-#include <fcntl.h>
-
-#include "linenoise.h"
-#include "ConvertUTF.h"
-
-#include <string>
-#include <vector>
-#include <memory>
-
-using std::string;
-using std::vector;
-using std::unique_ptr;
-using namespace linenoise_ng;
-
-typedef unsigned char char8_t;
-
-static ConversionResult copyString8to32(char32_t* dst, size_t dstSize,
-                                        size_t& dstCount, const char* src) {
-  const UTF8* sourceStart = reinterpret_cast<const UTF8*>(src);
-  const UTF8* sourceEnd = sourceStart + strlen(src);
-  UTF32* targetStart = reinterpret_cast<UTF32*>(dst);
-  UTF32* targetEnd = targetStart + dstSize;
-
-  ConversionResult res = ConvertUTF8toUTF32(
-      &sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion);
-
-  if (res == conversionOK) {
-    dstCount = targetStart - reinterpret_cast<UTF32*>(dst);
-
-    if (dstCount < dstSize) {
-      *targetStart = 0;
-    }
-  }
-
-  return res;
-}
-
-static ConversionResult copyString8to32(char32_t* dst, size_t dstSize,
-                                        size_t& dstCount, const char8_t* src) {
-  return copyString8to32(dst, dstSize, dstCount,
-                         reinterpret_cast<const char*>(src));
-}
-
-static size_t strlen32(const char32_t* str) {
-  const char32_t* ptr = str;
-
-  while (*ptr) {
-    ++ptr;
-  }
-
-  return ptr - str;
-}
-
-static size_t strlen8(const char8_t* str) {
-  return strlen(reinterpret_cast<const char*>(str));
-}
-
-static char8_t* strdup8(const char* src) {
-  return reinterpret_cast<char8_t*>(strdup(src));
-}
-
-#ifdef _WIN32
-static const int FOREGROUND_WHITE =
-    FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE;
-static const int BACKGROUND_WHITE =
-    BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_BLUE;
-static const int INTENSITY = FOREGROUND_INTENSITY | BACKGROUND_INTENSITY;
-
-class WinAttributes {
- public:
-  WinAttributes() {
-    CONSOLE_SCREEN_BUFFER_INFO info;
-    GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &info);
-    _defaultAttribute = info.wAttributes & INTENSITY;
-    _defaultColor = info.wAttributes & FOREGROUND_WHITE;
-    _defaultBackground = info.wAttributes & BACKGROUND_WHITE;
-
-    _consoleAttribute = _defaultAttribute;
-    _consoleColor = _defaultColor | _defaultBackground;
-  }
-
- public:
-  int _defaultAttribute;
-  int _defaultColor;
-  int _defaultBackground;
-
-  int _consoleAttribute;
-  int _consoleColor;
-};
-
-static WinAttributes WIN_ATTR;
-
-static void copyString32to16(char16_t* dst, size_t dstSize, size_t* dstCount,
-                             const char32_t* src, size_t srcSize) {
-  const UTF32* sourceStart = reinterpret_cast<const UTF32*>(src);
-  const UTF32* sourceEnd = sourceStart + srcSize;
-  char16_t* targetStart = reinterpret_cast<char16_t*>(dst);
-  char16_t* targetEnd = targetStart + dstSize;
-
-  ConversionResult res = ConvertUTF32toUTF16(
-      &sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion);
-
-  if (res == conversionOK) {
-    *dstCount = targetStart - reinterpret_cast<char16_t*>(dst);
-
-    if (*dstCount < dstSize) {
-      *targetStart = 0;
-    }
-  }
-}
-#endif
-
-static void copyString32to8(char* dst, size_t dstSize, size_t* dstCount,
-                            const char32_t* src, size_t srcSize) {
-  const UTF32* sourceStart = reinterpret_cast<const UTF32*>(src);
-  const UTF32* sourceEnd = sourceStart + srcSize;
-  UTF8* targetStart = reinterpret_cast<UTF8*>(dst);
-  UTF8* targetEnd = targetStart + dstSize;
-
-  ConversionResult res = ConvertUTF32toUTF8(
-      &sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion);
-
-  if (res == conversionOK) {
-    *dstCount = targetStart - reinterpret_cast<UTF8*>(dst);
-
-    if (*dstCount < dstSize) {
-      *targetStart = 0;
-    }
-  }
-}
-
-static void copyString32to8(char* dst, size_t dstLen, const char32_t* src) {
-  size_t dstCount = 0;
-  copyString32to8(dst, dstLen, &dstCount, src, strlen32(src));
-}
-
-static void copyString32(char32_t* dst, const char32_t* src, size_t len) {
-  while (0 < len && *src) {
-    *dst++ = *src++;
-    --len;
-  }
-
-  *dst = 0;
-}
-
-static int strncmp32(const char32_t* left, const char32_t* right, size_t len) {
-  while (0 < len && *left) {
-    if (*left != *right) {
-      return *left - *right;
-    }
-
-    ++left;
-    ++right;
-    --len;
-  }
-
-  return 0;
-}
-
-#ifdef _WIN32
-#include <iostream>
-
-static size_t OutputWin(char16_t* text16, char32_t* text32, size_t len32) {
-  size_t count16 = 0;
-
-  copyString32to16(text16, len32, &count16, text32, len32);
-  WriteConsoleW(GetStdHandle(STD_OUTPUT_HANDLE), text16,
-                static_cast<DWORD>(count16), nullptr, nullptr);
-
-  return count16;
-}
-
-static char32_t* HandleEsc(char32_t* p, char32_t* end) {
-  if (*p == '[') {
-    int code = 0;
-
-    for (++p; p < end; ++p) {
-      char32_t c = *p;
-
-      if ('0' <= c && c <= '9') {
-        code = code * 10 + (c - '0');
-      } else if (c == 'm' || c == ';') {
-        switch (code) {
-          case 0:
-            WIN_ATTR._consoleAttribute = WIN_ATTR._defaultAttribute;
-            WIN_ATTR._consoleColor =
-                WIN_ATTR._defaultColor | WIN_ATTR._defaultBackground;
-            break;
-
-          case 1:  // BOLD
-          case 5:  // BLINK
-            WIN_ATTR._consoleAttribute =
-                (WIN_ATTR._defaultAttribute ^ FOREGROUND_INTENSITY) & INTENSITY;
-            break;
-
-          case 30:
-            WIN_ATTR._consoleColor = BACKGROUND_WHITE;
-            break;
-
-          case 31:
-            WIN_ATTR._consoleColor =
-                FOREGROUND_RED | WIN_ATTR._defaultBackground;
-            break;
-
-          case 32:
-            WIN_ATTR._consoleColor =
-                FOREGROUND_GREEN | WIN_ATTR._defaultBackground;
-            break;
-
-          case 33:
-            WIN_ATTR._consoleColor =
-                FOREGROUND_RED | FOREGROUND_GREEN | WIN_ATTR._defaultBackground;
-            break;
-
-          case 34:
-            WIN_ATTR._consoleColor =
-                FOREGROUND_BLUE | WIN_ATTR._defaultBackground;
-            break;
-
-          case 35:
-            WIN_ATTR._consoleColor =
-                FOREGROUND_BLUE | FOREGROUND_RED | WIN_ATTR._defaultBackground;
-            break;
-
-          case 36:
-            WIN_ATTR._consoleColor = FOREGROUND_BLUE | FOREGROUND_GREEN |
-                                     WIN_ATTR._defaultBackground;
-            break;
-
-          case 37:
-            WIN_ATTR._consoleColor = FOREGROUND_GREEN | FOREGROUND_RED |
-                                     FOREGROUND_BLUE |
-                                     WIN_ATTR._defaultBackground;
-            break;
-        }
-
-        code = 0;
-      }
-
-      if (*p == 'm') {
-        ++p;
-        break;
-      }
-    }
-  } else {
-    ++p;
-  }
-
-  auto handle = GetStdHandle(STD_OUTPUT_HANDLE);
-  SetConsoleTextAttribute(handle,
-                          WIN_ATTR._consoleAttribute | WIN_ATTR._consoleColor);
-
-  return p;
-}
-
-static size_t WinWrite32(char16_t* text16, char32_t* text32, size_t len32) {
-  char32_t* p = text32;
-  char32_t* q = p;
-  char32_t* e = text32 + len32;
-  size_t count16 = 0;
-
-  while (p < e) {
-    if (*p == 27) {
-      if (q < p) {
-        count16 += OutputWin(text16, q, p - q);
-      }
-
-      q = p = HandleEsc(p + 1, e);
-    } else {
-      ++p;
-    }
-  }
-
-  if (q < p) {
-    count16 += OutputWin(text16, q, p - q);
-  }
-
-  return count16;
-}
-#endif
-
-static int write32(int fd, char32_t* text32, int len32) {
-#ifdef _WIN32
-  if (isatty(fd)) {
-    size_t len16 = 2 * len32 + 1;
-    unique_ptr<char16_t[]> text16(new char16_t[len16]);
-    size_t count16 = WinWrite32(text16.get(), text32, len32);
-
-    return static_cast<int>(count16);
-  } else {
-    size_t len8 = 4 * len32 + 1;
-    unique_ptr<char[]> text8(new char[len8]);
-    size_t count8 = 0;
-
-    copyString32to8(text8.get(), len8, &count8, text32, len32);
-
-    return write(fd, text8.get(), static_cast<unsigned int>(count8));
-  }
-#else
-  size_t len8 = 4 * len32 + 1;
-  unique_ptr<char[]> text8(new char[len8]);
-  size_t count8 = 0;
-
-  copyString32to8(text8.get(), len8, &count8, text32, len32);
-
-  return write(fd, text8.get(), count8);
-#endif
-}
-
-class Utf32String {
- public:
-  Utf32String() : _length(0), _data(nullptr) { 
-    // note: parens intentional, _data must be properly initialized
-    _data = new char32_t[1](); 
-  }
-
-  explicit Utf32String(const char* src) : _length(0), _data(nullptr) {
-    size_t len = strlen(src);
-    // note: parens intentional, _data must be properly initialized
-    _data = new char32_t[len + 1]();
-    copyString8to32(_data, len + 1, _length, src);
-  }
-
-  explicit Utf32String(const char8_t* src) : _length(0), _data(nullptr) {
-    size_t len = strlen(reinterpret_cast<const char*>(src));
-    // note: parens intentional, _data must be properly initialized
-    _data = new char32_t[len + 1]();
-    copyString8to32(_data, len + 1, _length, src);
-  }
-
-  explicit Utf32String(const char32_t* src) : _length(0), _data(nullptr) {
-    for (_length = 0; src[_length] != 0; ++_length) {
-    }
-
-    // note: parens intentional, _data must be properly initialized
-    _data = new char32_t[_length + 1]();
-    memcpy(_data, src, _length * sizeof(char32_t));
-  }
-
-  explicit Utf32String(const char32_t* src, int len) : _length(len), _data(nullptr) {
-    // note: parens intentional, _data must be properly initialized
-    _data = new char32_t[len + 1]();
-    memcpy(_data, src, len * sizeof(char32_t));
-  }
-
-  explicit Utf32String(int len) : _length(0), _data(nullptr) { 
-    // note: parens intentional, _data must be properly initialized
-    _data = new char32_t[len](); 
-  }
-
-  explicit Utf32String(const Utf32String& that) : _length(that._length), _data(nullptr) {
-    // note: parens intentional, _data must be properly initialized
-    _data = new char32_t[_length + 1]();
-    memcpy(_data, that._data, sizeof(char32_t) * _length);
-  }
-
-  Utf32String& operator=(const Utf32String& that) {
-    if (this != &that) {
-      delete[] _data;
-      _data = new char32_t[that._length]();
-      _length = that._length;
-      memcpy(_data, that._data, sizeof(char32_t) * _length);
-    }
-
-    return *this;
-  }
-
-  ~Utf32String() { delete[] _data; }
-
- public:
-  char32_t* get() const { return _data; }
-
-  size_t length() const { return _length; }
-
-  size_t chars() const { return _length; }
-
-  void initFromBuffer() {
-    for (_length = 0; _data[_length] != 0; ++_length) {
-    }
-  }
-
-  const char32_t& operator[](size_t pos) const { return _data[pos]; }
-
-  char32_t& operator[](size_t pos) { return _data[pos]; }
-
- private:
-  size_t _length;
-  char32_t* _data;
-};
-
-class Utf8String {
-  Utf8String(const Utf8String&) = delete;
-  Utf8String& operator=(const Utf8String&) = delete;
-
- public:
-  explicit Utf8String(const Utf32String& src) {
-    size_t len = src.length() * 4 + 1;
-    _data = new char[len];
-    copyString32to8(_data, len, src.get());
-  }
-
-  ~Utf8String() { delete[] _data; }
-
- public:
-  char* get() const { return _data; }
-
- private:
-  char* _data;
-};
-
-struct linenoiseCompletions {
-  vector<Utf32String> completionStrings;
-};
-
-#define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100
-#define LINENOISE_MAX_LINE 4096
-
-// make control-characters more readable
-#define ctrlChar(upperCaseASCII) (upperCaseASCII - 0x40)
-
-/**
- * Recompute widths of all characters in a char32_t buffer
- * @param text          input buffer of Unicode characters
- * @param widths        output buffer of character widths
- * @param charCount     number of characters in buffer
- */
-namespace linenoise_ng {
-int mk_wcwidth(char32_t ucs);
-}
-
-static void recomputeCharacterWidths(const char32_t* text, char* widths,
-                                     int charCount) {
-  for (int i = 0; i < charCount; ++i) {
-    widths[i] = mk_wcwidth(text[i]);
-  }
-}
-
-/**
- * Calculate a new screen position given a starting position, screen width and
- * character count
- * @param x             initial x position (zero-based)
- * @param y             initial y position (zero-based)
- * @param screenColumns screen column count
- * @param charCount     character positions to advance
- * @param xOut          returned x position (zero-based)
- * @param yOut          returned y position (zero-based)
- */
-static void calculateScreenPosition(int x, int y, int screenColumns,
-                                    int charCount, int& xOut, int& yOut) {
-  xOut = x;
-  yOut = y;
-  int charsRemaining = charCount;
-  while (charsRemaining > 0) {
-    int charsThisRow = (x + charsRemaining < screenColumns) ? charsRemaining
-                                                            : screenColumns - x;
-    xOut = x + charsThisRow;
-    yOut = y;
-    charsRemaining -= charsThisRow;
-    x = 0;
-    ++y;
-  }
-  if (xOut == screenColumns) {  // we have to special-case line wrap
-    xOut = 0;
-    ++yOut;
-  }
-}
-
-/**
- * Calculate a column width using mk_wcswidth()
- * @param buf32  text to calculate
- * @param len    length of text to calculate
- */
-namespace linenoise_ng {
-int mk_wcswidth(const char32_t* pwcs, size_t n);
-}
-
-static int calculateColumnPosition(char32_t* buf32, int len) {
-  int width = mk_wcswidth(reinterpret_cast<const char32_t*>(buf32), len);
-  if (width == -1)
-    return len;
-  else
-    return width;
-}
-
-static bool isControlChar(char32_t testChar) {
-  return (testChar < ' ') ||                      // C0 controls
-         (testChar >= 0x7F && testChar <= 0x9F);  // DEL and C1 controls
-}
-
-struct PromptBase {            // a convenience struct for grouping prompt info
-  Utf32String promptText;      // our copy of the prompt text, edited
-  char* promptCharWidths;      // character widths from mk_wcwidth()
-  int promptChars;             // chars in promptText
-  int promptBytes;             // bytes in promptText
-  int promptExtraLines;        // extra lines (beyond 1) occupied by prompt
-  int promptIndentation;       // column offset to end of prompt
-  int promptLastLinePosition;  // index into promptText where last line begins
-  int promptPreviousInputLen;  // promptChars of previous input line, for
-                               // clearing
-  int promptCursorRowOffset;   // where the cursor is relative to the start of
-                               // the prompt
-  int promptScreenColumns;     // width of screen in columns
-  int promptPreviousLen;       // help erasing
-  int promptErrorCode;         // error code (invalid UTF-8) or zero
-
-  PromptBase() : promptPreviousInputLen(0) {}
-
-  bool write() {
-    if (write32(1, promptText.get(), promptBytes) == -1) return false;
-
-    return true;
-  }
-};
-
-struct PromptInfo : public PromptBase {
-  PromptInfo(const char* textPtr, int columns) {
-    promptExtraLines = 0;
-    promptLastLinePosition = 0;
-    promptPreviousLen = 0;
-    promptScreenColumns = columns;
-    Utf32String tempUnicode(textPtr);
-
-    // strip control characters from the prompt -- we do allow newline
-    char32_t* pIn = tempUnicode.get();
-    char32_t* pOut = pIn;
-
-    int len = 0;
-    int x = 0;
-
-    bool const strip = (isatty(1) == 0);
-
-    while (*pIn) {
-      char32_t c = *pIn;
-      if ('\n' == c || !isControlChar(c)) {
-        *pOut = c;
-        ++pOut;
-        ++pIn;
-        ++len;
-        if ('\n' == c || ++x >= promptScreenColumns) {
-          x = 0;
-          ++promptExtraLines;
-          promptLastLinePosition = len;
-        }
-      } else if (c == '\x1b') {
-        if (strip) {
-          // jump over control chars
-          ++pIn;
-          if (*pIn == '[') {
-            ++pIn;
-            while (*pIn && ((*pIn == ';') || ((*pIn >= '0' && *pIn <= '9')))) {
-              ++pIn;
-            }
-            if (*pIn == 'm') {
-              ++pIn;
-            }
-          }
-        } else {
-          // copy control chars
-          *pOut = *pIn;
-          ++pOut;
-          ++pIn;
-          if (*pIn == '[') {
-            *pOut = *pIn;
-            ++pOut;
-            ++pIn;
-            while (*pIn && ((*pIn == ';') || ((*pIn >= '0' && *pIn <= '9')))) {
-              *pOut = *pIn;
-              ++pOut;
-              ++pIn;
-            }
-            if (*pIn == 'm') {
-              *pOut = *pIn;
-              ++pOut;
-              ++pIn;
-            }
-          }
-        }
-      } else {
-        ++pIn;
-      }
-    }
-    *pOut = 0;
-    promptChars = len;
-    promptBytes = static_cast<int>(pOut - tempUnicode.get());
-    promptText = tempUnicode;
-
-    promptIndentation = len - promptLastLinePosition;
-    promptCursorRowOffset = promptExtraLines;
-  }
-};
-
-// Used with DynamicPrompt (history search)
-//
-static const Utf32String forwardSearchBasePrompt("(i-search)`");
-static const Utf32String reverseSearchBasePrompt("(reverse-i-search)`");
-static const Utf32String endSearchBasePrompt("': ");
-static Utf32String
-    previousSearchText;  // remembered across invocations of linenoise()
-
-// changing prompt for "(reverse-i-search)`text':" etc.
-//
-struct DynamicPrompt : public PromptBase {
-  Utf32String searchText;  // text we are searching for
-  char* searchCharWidths;  // character widths from mk_wcwidth()
-  int searchTextLen;       // chars in searchText
-  int direction;           // current search direction, 1=forward, -1=reverse
-
-  DynamicPrompt(PromptBase& pi, int initialDirection)
-      : searchTextLen(0), direction(initialDirection) {
-    promptScreenColumns = pi.promptScreenColumns;
-    promptCursorRowOffset = 0;
-    Utf32String emptyString(1);
-    searchText = emptyString;
-    const Utf32String* basePrompt =
-        (direction > 0) ? &forwardSearchBasePrompt : &reverseSearchBasePrompt;
-    size_t promptStartLength = basePrompt->length();
-    promptChars =
-        static_cast<int>(promptStartLength + endSearchBasePrompt.length());
-    promptBytes = promptChars;
-    promptLastLinePosition = promptChars;  // TODO fix this, we are asssuming
-                                           // that the history prompt won't wrap
-                                           // (!)
-    promptPreviousLen = promptChars;
-    Utf32String tempUnicode(promptChars + 1);
-    memcpy(tempUnicode.get(), basePrompt->get(),
-           sizeof(char32_t) * promptStartLength);
-    memcpy(&tempUnicode[promptStartLength], endSearchBasePrompt.get(),
-           sizeof(char32_t) * (endSearchBasePrompt.length() + 1));
-    tempUnicode.initFromBuffer();
-    promptText = tempUnicode;
-    calculateScreenPosition(0, 0, pi.promptScreenColumns, promptChars,
-                            promptIndentation, promptExtraLines);
-  }
-
-  void updateSearchPrompt(void) {
-    const Utf32String* basePrompt =
-        (direction > 0) ? &forwardSearchBasePrompt : &reverseSearchBasePrompt;
-    size_t promptStartLength = basePrompt->length();
-    promptChars = static_cast<int>(promptStartLength + searchTextLen +
-                                   endSearchBasePrompt.length());
-    promptBytes = promptChars;
-    Utf32String tempUnicode(promptChars + 1);
-    memcpy(tempUnicode.get(), basePrompt->get(),
-           sizeof(char32_t) * promptStartLength);
-    memcpy(&tempUnicode[promptStartLength], searchText.get(),
-           sizeof(char32_t) * searchTextLen);
-    size_t endIndex = promptStartLength + searchTextLen;
-    memcpy(&tempUnicode[endIndex], endSearchBasePrompt.get(),
-           sizeof(char32_t) * (endSearchBasePrompt.length() + 1));
-    tempUnicode.initFromBuffer();
-    promptText = tempUnicode;
-  }
-
-  void updateSearchText(const char32_t* textPtr) {
-    Utf32String tempUnicode(textPtr);
-    searchTextLen = static_cast<int>(tempUnicode.chars());
-    searchText = tempUnicode;
-    updateSearchPrompt();
-  }
-};
-
-class KillRing {
-  static const int capacity = 10;
-  int size;
-  int index;
-  char indexToSlot[10];
-  vector<Utf32String> theRing;
-
- public:
-  enum action { actionOther, actionKill, actionYank };
-  action lastAction;
-  size_t lastYankSize;
-
-  KillRing() : size(0), index(0), lastAction(actionOther) {
-    theRing.reserve(capacity);
-  }
-
-  void kill(const char32_t* text, int textLen, bool forward) {
-    if (textLen == 0) {
-      return;
-    }
-    Utf32String killedText(text, textLen);
-    if (lastAction == actionKill && size > 0) {
-      int slot = indexToSlot[0];
-      int currentLen = static_cast<int>(theRing[slot].length());
-      int resultLen = currentLen + textLen;
-      Utf32String temp(resultLen + 1);
-      if (forward) {
-        memcpy(temp.get(), theRing[slot].get(), currentLen * sizeof(char32_t));
-        memcpy(&temp[currentLen], killedText.get(), textLen * sizeof(char32_t));
-      } else {
-        memcpy(temp.get(), killedText.get(), textLen * sizeof(char32_t));
-        memcpy(&temp[textLen], theRing[slot].get(),
-               currentLen * sizeof(char32_t));
-      }
-      temp[resultLen] = 0;
-      temp.initFromBuffer();
-      theRing[slot] = temp;
-    } else {
-      if (size < capacity) {
-        if (size > 0) {
-          memmove(&indexToSlot[1], &indexToSlot[0], size);
-        }
-        indexToSlot[0] = size;
-        size++;
-        theRing.push_back(killedText);
-      } else {
-        int slot = indexToSlot[capacity - 1];
-        theRing[slot] = killedText;
-        memmove(&indexToSlot[1], &indexToSlot[0], capacity - 1);
-        indexToSlot[0] = slot;
-      }
-      index = 0;
-    }
-  }
-
-  Utf32String* yank() { return (size > 0) ? &theRing[indexToSlot[index]] : 0; }
-
-  Utf32String* yankPop() {
-    if (size == 0) {
-      return 0;
-    }
-    ++index;
-    if (index == size) {
-      index = 0;
-    }
-    return &theRing[indexToSlot[index]];
-  }
-};
-
-class InputBuffer {
-  char32_t* buf32;   // input buffer
-  char* charWidths;  // character widths from mk_wcwidth()
-  int buflen;        // buffer size in characters
-  int len;           // length of text in input buffer
-  int pos;           // character position in buffer ( 0 <= pos <= len )
-
-  void clearScreen(PromptBase& pi);
-  int incrementalHistorySearch(PromptBase& pi, int startChar);
-  int completeLine(PromptBase& pi);
-  void refreshLine(PromptBase& pi);
-
- public:
-  InputBuffer(char32_t* buffer, char* widthArray, int bufferLen)
-      : buf32(buffer),
-        charWidths(widthArray),
-        buflen(bufferLen - 1),
-        len(0),
-        pos(0) {
-    buf32[0] = 0;
-  }
-  void preloadBuffer(const char* preloadText) {
-    size_t ucharCount = 0;
-    copyString8to32(buf32, buflen + 1, ucharCount, preloadText);
-    recomputeCharacterWidths(buf32, charWidths, static_cast<int>(ucharCount));
-    len = static_cast<int>(ucharCount);
-    pos = static_cast<int>(ucharCount);
-  }
-  int getInputLine(PromptBase& pi);
-  int length(void) const { return len; }
-};
-
-// Special codes for keyboard input:
-//
-// Between Windows and the various Linux "terminal" programs, there is some
-// pretty diverse behavior in the "scan codes" and escape sequences we are
-// presented with.  So ... we'll translate them all into our own pidgin
-// pseudocode, trying to stay out of the way of UTF-8 and international
-// characters.  Here's the general plan.
-//
-// "User input keystrokes" (key chords, whatever) will be encoded as a single
-// value.
-// The low 21 bits are reserved for Unicode characters.  Popular function-type
-// keys
-// get their own codes in the range 0x10200000 to (if needed) 0x1FE00000,
-// currently
-// just arrow keys, Home, End and Delete.  Keypresses with Ctrl get ORed with
-// 0x20000000, with Alt get ORed with 0x40000000.  So, Ctrl+Alt+Home is encoded
-// as 0x20000000 + 0x40000000 + 0x10A00000 == 0x70A00000.  To keep things
-// complicated,
-// the Alt key is equivalent to prefixing the keystroke with ESC, so ESC
-// followed by
-// D is treated the same as Alt + D ... we'll just use Emacs terminology and
-// call
-// this "Meta".  So, we will encode both ESC followed by D and Alt held down
-// while D
-// is pressed the same, as Meta-D, encoded as 0x40000064.
-//
-// Here are the definitions of our component constants:
-//
-// Maximum unsigned 32-bit value    = 0xFFFFFFFF;   // For reference, max 32-bit
-// value
-// Highest allocated Unicode char   = 0x001FFFFF;   // For reference, max
-// Unicode value
-static const int META = 0x40000000;  // Meta key combination
-static const int CTRL = 0x20000000;  // Ctrl key combination
-// static const int SPECIAL_KEY = 0x10000000;   // Common bit for all special
-// keys
-static const int UP_ARROW_KEY = 0x10200000;  // Special keys
-static const int DOWN_ARROW_KEY = 0x10400000;
-static const int RIGHT_ARROW_KEY = 0x10600000;
-static const int LEFT_ARROW_KEY = 0x10800000;
-static const int HOME_KEY = 0x10A00000;
-static const int END_KEY = 0x10C00000;
-static const int DELETE_KEY = 0x10E00000;
-static const int PAGE_UP_KEY = 0x11000000;
-static const int PAGE_DOWN_KEY = 0x11200000;
-
-static const char* unsupported_term[] = {"dumb", "cons25", "emacs", NULL};
-static linenoiseCompletionCallback* completionCallback = NULL;
-
-#ifdef _WIN32
-static HANDLE console_in, console_out;
-static DWORD oldMode;
-static WORD oldDisplayAttribute;
-#else
-static struct termios orig_termios; /* in order to restore at exit */
-#endif
-
-static KillRing killRing;
-
-static int rawmode = 0; /* for atexit() function to check if restore is needed*/
-static int atexit_registered = 0; /* register atexit just 1 time */
-static int historyMaxLen = LINENOISE_DEFAULT_HISTORY_MAX_LEN;
-static int historyLen = 0;
-static int historyIndex = 0;
-static char8_t** history = NULL;
-
-// used to emulate Windows command prompt on down-arrow after a recall
-// we use -2 as our "not set" value because we add 1 to the previous index on
-// down-arrow,
-// and zero is a valid index (so -1 is a valid "previous index")
-static int historyPreviousIndex = -2;
-static bool historyRecallMostRecent = false;
-
-static void linenoiseAtExit(void);
-
-static bool isUnsupportedTerm(void) {
-  char* term = getenv("TERM");
-  if (term == NULL) return false;
-  for (int j = 0; unsupported_term[j]; ++j)
-    if (!strcasecmp(term, unsupported_term[j])) {
-      return true;
-    }
-  return false;
-}
-
-static void beep() {
-  fprintf(stderr, "\x7");  // ctrl-G == bell/beep
-  fflush(stderr);
-}
-
-void linenoiseHistoryFree(void) {
-  if (history) {
-    for (int j = 0; j < historyLen; ++j) free(history[j]);
-    historyLen = 0;
-    free(history);
-    history = 0;
-  }
-}
-
-static int enableRawMode(void) {
-#ifdef _WIN32
-  if (!console_in) {
-    console_in = GetStdHandle(STD_INPUT_HANDLE);
-    console_out = GetStdHandle(STD_OUTPUT_HANDLE);
-
-    GetConsoleMode(console_in, &oldMode);
-    SetConsoleMode(console_in, oldMode &
-                                   ~(ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT |
-                                     ENABLE_PROCESSED_INPUT));
-  }
-  return 0;
-#else
-  struct termios raw;
-
-  if (!isatty(STDIN_FILENO)) goto fatal;
-  if (!atexit_registered) {
-    atexit(linenoiseAtExit);
-    atexit_registered = 1;
-  }
-  if (tcgetattr(0, &orig_termios) == -1) goto fatal;
-
-  raw = orig_termios; /* modify the original mode */
-  /* input modes: no break, no CR to NL, no parity check, no strip char,
-   * no start/stop output control. */
-  raw.c_iflag &= ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON);
-  /* output modes - disable post processing */
-  // this is wrong, we don't want raw output, it turns newlines into straight
-  // linefeeds
-  // raw.c_oflag &= ~(OPOST);
-  /* control modes - set 8 bit chars */
-  raw.c_cflag |= (CS8);
-  /* local modes - echoing off, canonical off, no extended functions,
-   * no signal chars (^Z,^C) */
-  raw.c_lflag &= ~(ECHO | ICANON | IEXTEN | ISIG);
-  /* control chars - set return condition: min number of bytes and timer.
-   * We want read to return every single byte, without timeout. */
-  raw.c_cc[VMIN] = 1;
-  raw.c_cc[VTIME] = 0; /* 1 byte, no timer */
-
-  /* put terminal in raw mode after flushing */
-  if (tcsetattr(0, TCSADRAIN, &raw) < 0) goto fatal;
-  rawmode = 1;
-  return 0;
-
-fatal:
-  errno = ENOTTY;
-  return -1;
-#endif
-}
-
-static void disableRawMode(void) {
-#ifdef _WIN32
-  SetConsoleMode(console_in, oldMode);
-  console_in = 0;
-  console_out = 0;
-#else
-  if (rawmode && tcsetattr(0, TCSADRAIN, &orig_termios) != -1) rawmode = 0;
-#endif
-}
-
-// At exit we'll try to fix the terminal to the initial conditions
-static void linenoiseAtExit(void) { disableRawMode(); }
-
-static int getScreenColumns(void) {
-  int cols;
-#ifdef _WIN32
-  CONSOLE_SCREEN_BUFFER_INFO inf;
-  GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &inf);
-  cols = inf.dwSize.X;
-#else
-  struct winsize ws;
-  cols = (ioctl(1, TIOCGWINSZ, &ws) == -1) ? 80 : ws.ws_col;
-#endif
-  // cols is 0 in certain circumstances like inside debugger, which creates
-  // further issues
-  return (cols > 0) ? cols : 80;
-}
-
-static int getScreenRows(void) {
-  int rows;
-#ifdef _WIN32
-  CONSOLE_SCREEN_BUFFER_INFO inf;
-  GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &inf);
-  rows = 1 + inf.srWindow.Bottom - inf.srWindow.Top;
-#else
-  struct winsize ws;
-  rows = (ioctl(1, TIOCGWINSZ, &ws) == -1) ? 24 : ws.ws_row;
-#endif
-  return (rows > 0) ? rows : 24;
-}
-
-static void setDisplayAttribute(bool enhancedDisplay, bool error) {
-#ifdef _WIN32
-  if (enhancedDisplay) {
-    CONSOLE_SCREEN_BUFFER_INFO inf;
-    GetConsoleScreenBufferInfo(console_out, &inf);
-    oldDisplayAttribute = inf.wAttributes;
-    BYTE oldLowByte = oldDisplayAttribute & 0xFF;
-    BYTE newLowByte;
-    switch (oldLowByte) {
-      case 0x07:
-        // newLowByte = FOREGROUND_BLUE | FOREGROUND_INTENSITY;  // too dim
-        // newLowByte = FOREGROUND_BLUE;                         // even dimmer
-        newLowByte = FOREGROUND_BLUE |
-                     FOREGROUND_GREEN;  // most similar to xterm appearance
-        break;
-      case 0x70:
-        newLowByte = BACKGROUND_BLUE | BACKGROUND_INTENSITY;
-        break;
-      default:
-        newLowByte = oldLowByte ^ 0xFF;  // default to inverse video
-        break;
-    }
-    inf.wAttributes = (inf.wAttributes & 0xFF00) | newLowByte;
-    SetConsoleTextAttribute(console_out, inf.wAttributes);
-  } else {
-    SetConsoleTextAttribute(console_out, oldDisplayAttribute);
-  }
-#else
-  if (enhancedDisplay) {
-    char const* p = (error ? "\x1b[1;31m" : "\x1b[1;34m");
-    if (write(1, p, 7) == -1)
-      return; /* bright blue (visible with both B&W bg) */
-  } else {
-    if (write(1, "\x1b[0m", 4) == -1) return; /* reset */
-  }
-#endif
-}
-
-/**
- * Display the dynamic incremental search prompt and the current user input
- * line.
- * @param pi   PromptBase struct holding information about the prompt and our
- * screen position
- * @param buf32  input buffer to be displayed
- * @param len  count of characters in the buffer
- * @param pos  current cursor position within the buffer (0 <= pos <= len)
- */
-static void dynamicRefresh(PromptBase& pi, char32_t* buf32, int len, int pos) {
-  // calculate the position of the end of the prompt
-  int xEndOfPrompt, yEndOfPrompt;
-  calculateScreenPosition(0, 0, pi.promptScreenColumns, pi.promptChars,
-                          xEndOfPrompt, yEndOfPrompt);
-  pi.promptIndentation = xEndOfPrompt;
-
-  // calculate the position of the end of the input line
-  int xEndOfInput, yEndOfInput;
-  calculateScreenPosition(xEndOfPrompt, yEndOfPrompt, pi.promptScreenColumns,
-                          calculateColumnPosition(buf32, len), xEndOfInput,
-                          yEndOfInput);
-
-  // calculate the desired position of the cursor
-  int xCursorPos, yCursorPos;
-  calculateScreenPosition(xEndOfPrompt, yEndOfPrompt, pi.promptScreenColumns,
-                          calculateColumnPosition(buf32, pos), xCursorPos,
-                          yCursorPos);
-
-#ifdef _WIN32
-  // position at the start of the prompt, clear to end of previous input
-  CONSOLE_SCREEN_BUFFER_INFO inf;
-  GetConsoleScreenBufferInfo(console_out, &inf);
-  inf.dwCursorPosition.X = 0;
-  inf.dwCursorPosition.Y -= pi.promptCursorRowOffset /*- pi.promptExtraLines*/;
-  SetConsoleCursorPosition(console_out, inf.dwCursorPosition);
-  DWORD count;
-  FillConsoleOutputCharacterA(console_out, ' ',
-                              pi.promptPreviousLen + pi.promptPreviousInputLen,
-                              inf.dwCursorPosition, &count);
-  pi.promptPreviousLen = pi.promptIndentation;
-  pi.promptPreviousInputLen = len;
-
-  // display the prompt
-  if (!pi.write()) return;
-
-  // display the input line
-  if (write32(1, buf32, len) == -1) return;
-
-  // position the cursor
-  GetConsoleScreenBufferInfo(console_out, &inf);
-  inf.dwCursorPosition.X = xCursorPos;  // 0-based on Win32
-  inf.dwCursorPosition.Y -= yEndOfInput - yCursorPos;
-  SetConsoleCursorPosition(console_out, inf.dwCursorPosition);
-#else  // _WIN32
-  char seq[64];
-  int cursorRowMovement = pi.promptCursorRowOffset - pi.promptExtraLines;
-  if (cursorRowMovement > 0) {  // move the cursor up as required
-    snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement);
-    if (write(1, seq, strlen(seq)) == -1) return;
-  }
-  // position at the start of the prompt, clear to end of screen
-  snprintf(seq, sizeof seq, "\x1b[1G\x1b[J");  // 1-based on VT100
-  if (write(1, seq, strlen(seq)) == -1) return;
-
-  // display the prompt
-  if (!pi.write()) return;
-
-  // display the input line
-  if (write32(1, buf32, len) == -1) return;
-
-  // we have to generate our own newline on line wrap
-  if (xEndOfInput == 0 && yEndOfInput > 0)
-    if (write(1, "\n", 1) == -1) return;
-
-  // position the cursor
-  cursorRowMovement = yEndOfInput - yCursorPos;
-  if (cursorRowMovement > 0) {  // move the cursor up as required
-    snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement);
-    if (write(1, seq, strlen(seq)) == -1) return;
-  }
-  // position the cursor within the line
-  snprintf(seq, sizeof seq, "\x1b[%dG", xCursorPos + 1);  // 1-based on VT100
-  if (write(1, seq, strlen(seq)) == -1) return;
-#endif
-
-  pi.promptCursorRowOffset =
-      pi.promptExtraLines + yCursorPos;  // remember row for next pass
-}
-
-/**
- * Refresh the user's input line: the prompt is already onscreen and is not
- * redrawn here
- * @param pi   PromptBase struct holding information about the prompt and our
- * screen position
- */
-void InputBuffer::refreshLine(PromptBase& pi) {
-  // check for a matching brace/bracket/paren, remember its position if found
-  int highlight = -1;
-  bool indicateError = false;
-  if (pos < len) {
-    /* this scans for a brace matching buf32[pos] to highlight */
-    unsigned char part1, part2;
-    int scanDirection = 0;
-    if (strchr("}])", buf32[pos])) {
-      scanDirection = -1; /* backwards */
-      if (buf32[pos] == '}') {
-        part1 = '}'; part2 = '{';
-      } else if (buf32[pos] == ']') {
-        part1 = ']'; part2 = '[';
-      } else {
-        part1 = ')'; part2 = '(';
-      }
-    }
-    else if (strchr("{[(", buf32[pos])) {
-      scanDirection = 1; /* forwards */
-      if (buf32[pos] == '{') {
-        //part1 = '{'; part2 = '}';
-        part1 = '}'; part2 = '{';
-      } else if (buf32[pos] == '[') {
-        //part1 = '['; part2 = ']';
-        part1 = ']'; part2 = '[';
-      } else {
-        //part1 = '('; part2 = ')';
-        part1 = ')'; part2 = '(';
-      }
-    }
-
-    if (scanDirection) {
-      int unmatched = scanDirection;
-      int unmatchedOther = 0;
-      for (int i = pos + scanDirection; i >= 0 && i < len; i += scanDirection) {
-        /* TODO: the right thing when inside a string */
-        if (strchr("}])", buf32[i])) {
-          if (buf32[i] == part1) {
-            --unmatched;
-          } else {
-            --unmatchedOther;
-          }
-        } else if (strchr("{[(", buf32[i])) {
-          if (buf32[i] == part2) {
-            ++unmatched;
-          } else {
-            ++unmatchedOther;
-          }
-        }
-/*
-        if (strchr("}])", buf32[i]))
-          --unmatched;
-        else if (strchr("{[(", buf32[i]))
-          ++unmatched;
-*/
-        if (unmatched == 0) {
-          highlight = i;
-          indicateError = (unmatchedOther != 0);
-          break;
-        }
-      }
-    }
-  }
-
-  // calculate the position of the end of the input line
-  int xEndOfInput, yEndOfInput;
-  calculateScreenPosition(pi.promptIndentation, 0, pi.promptScreenColumns,
-                          calculateColumnPosition(buf32, len), xEndOfInput,
-                          yEndOfInput);
-
-  // calculate the desired position of the cursor
-  int xCursorPos, yCursorPos;
-  calculateScreenPosition(pi.promptIndentation, 0, pi.promptScreenColumns,
-                          calculateColumnPosition(buf32, pos), xCursorPos,
-                          yCursorPos);
-
-#ifdef _WIN32
-  // position at the end of the prompt, clear to end of previous input
-  CONSOLE_SCREEN_BUFFER_INFO inf;
-  GetConsoleScreenBufferInfo(console_out, &inf);
-  inf.dwCursorPosition.X = pi.promptIndentation;  // 0-based on Win32
-  inf.dwCursorPosition.Y -= pi.promptCursorRowOffset - pi.promptExtraLines;
-  SetConsoleCursorPosition(console_out, inf.dwCursorPosition);
-  DWORD count;
-  if (len < pi.promptPreviousInputLen)
-    FillConsoleOutputCharacterA(console_out, ' ', pi.promptPreviousInputLen,
-                                inf.dwCursorPosition, &count);
-  pi.promptPreviousInputLen = len;
-
-  // display the input line
-  if (highlight == -1) {
-    if (write32(1, buf32, len) == -1) return;
-  } else {
-    if (write32(1, buf32, highlight) == -1) return;
-    setDisplayAttribute(true, indicateError); /* bright blue (visible with both B&W bg) */
-    if (write32(1, &buf32[highlight], 1) == -1) return;
-    setDisplayAttribute(false, indicateError);
-    if (write32(1, buf32 + highlight + 1, len - highlight - 1) == -1) return;
-  }
-
-  // position the cursor
-  GetConsoleScreenBufferInfo(console_out, &inf);
-  inf.dwCursorPosition.X = xCursorPos;  // 0-based on Win32
-  inf.dwCursorPosition.Y -= yEndOfInput - yCursorPos;
-  SetConsoleCursorPosition(console_out, inf.dwCursorPosition);
-#else  // _WIN32
-  char seq[64];
-  int cursorRowMovement = pi.promptCursorRowOffset - pi.promptExtraLines;
-  if (cursorRowMovement > 0) {  // move the cursor up as required
-    snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement);
-    if (write(1, seq, strlen(seq)) == -1) return;
-  }
-  // position at the end of the prompt, clear to end of screen
-  snprintf(seq, sizeof seq, "\x1b[%dG\x1b[J",
-           pi.promptIndentation + 1);  // 1-based on VT100
-  if (write(1, seq, strlen(seq)) == -1) return;
-
-  if (highlight == -1) {  // write unhighlighted text
-    if (write32(1, buf32, len) == -1) return;
-  } else {  // highlight the matching brace/bracket/parenthesis
-    if (write32(1, buf32, highlight) == -1) return;
-    setDisplayAttribute(true, indicateError);
-    if (write32(1, &buf32[highlight], 1) == -1) return;
-    setDisplayAttribute(false, indicateError);
-    if (write32(1, buf32 + highlight + 1, len - highlight - 1) == -1) return;
-  }
-
-  // we have to generate our own newline on line wrap
-  if (xEndOfInput == 0 && yEndOfInput > 0)
-    if (write(1, "\n", 1) == -1) return;
-
-  // position the cursor
-  cursorRowMovement = yEndOfInput - yCursorPos;
-  if (cursorRowMovement > 0) {  // move the cursor up as required
-    snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement);
-    if (write(1, seq, strlen(seq)) == -1) return;
-  }
-  // position the cursor within the line
-  snprintf(seq, sizeof seq, "\x1b[%dG", xCursorPos + 1);  // 1-based on VT100
-  if (write(1, seq, strlen(seq)) == -1) return;
-#endif
-
-  pi.promptCursorRowOffset =
-      pi.promptExtraLines + yCursorPos;  // remember row for next pass
-}
-
-#ifndef _WIN32
-
-/**
- * Read a UTF-8 sequence from the non-Windows keyboard and return the Unicode
- * (char32_t) character it
- * encodes
- *
- * @return  char32_t Unicode character
- */
-static char32_t readUnicodeCharacter(void) {
-  static char8_t utf8String[5];
-  static size_t utf8Count = 0;
-  while (true) {
-    char8_t c;
-
-    /* Continue reading if interrupted by signal. */
-    ssize_t nread;
-    do {
-      nread = read(0, &c, 1);
-    } while ((nread == -1) && (errno == EINTR));
-
-    if (nread <= 0) return 0;
-    if (c <= 0x7F) {  // short circuit ASCII
-      utf8Count = 0;
-      return c;
-    } else if (utf8Count < sizeof(utf8String) - 1) {
-      utf8String[utf8Count++] = c;
-      utf8String[utf8Count] = 0;
-      char32_t unicodeChar[2];
-      size_t ucharCount;
-      ConversionResult res =
-          copyString8to32(unicodeChar, 2, ucharCount, utf8String);
-      if (res == conversionOK && ucharCount) {
-        utf8Count = 0;
-        return unicodeChar[0];
-      }
-    } else {
-      utf8Count =
-          0;  // this shouldn't happen: got four bytes but no UTF-8 character
-    }
-  }
-}
-
-namespace EscapeSequenceProcessing {  // move these out of global namespace
-
-// This chunk of code does parsing of the escape sequences sent by various Linux
-// terminals.
-//
-// It handles arrow keys, Home, End and Delete keys by interpreting the
-// sequences sent by
-// gnome terminal, xterm, rxvt, konsole, aterm and yakuake including the Alt and
-// Ctrl key
-// combinations that are understood by linenoise.
-//
-// The parsing uses tables, a bunch of intermediate dispatch routines and a
-// doDispatch
-// loop that reads the tables and sends control to "deeper" routines to continue
-// the
-// parsing.  The starting call to doDispatch( c, initialDispatch ) will
-// eventually return
-// either a character (with optional CTRL and META bits set), or -1 if parsing
-// fails, or
-// zero if an attempt to read from the keyboard fails.
-//
-// This is rather sloppy escape sequence processing, since we're not paying
-// attention to what the
-// actual TERM is set to and are processing all key sequences for all terminals,
-// but it works with
-// the most common keystrokes on the most common terminals.  It's intricate, but
-// the nested 'if'
-// statements required to do it directly would be worse.  This way has the
-// advantage of allowing
-// changes and extensions without having to touch a lot of code.
-
-// This is a typedef for the routine called by doDispatch().  It takes the
-// current character
-// as input, does any required processing including reading more characters and
-// calling other
-// dispatch routines, then eventually returns the final (possibly extended or
-// special) character.
-//
-typedef char32_t (*CharacterDispatchRoutine)(char32_t);
-
-// This structure is used by doDispatch() to hold a list of characters to test
-// for and
-// a list of routines to call if the character matches.  The dispatch routine
-// list is one
-// longer than the character list; the final entry is used if no character
-// matches.
-//
-struct CharacterDispatch {
-  unsigned int len;                    // length of the chars list
-  const char* chars;                   // chars to test
-  CharacterDispatchRoutine* dispatch;  // array of routines to call
-};
-
-// This dispatch routine is given a dispatch table and then farms work out to
-// routines
-// listed in the table based on the character it is called with.  The dispatch
-// routines can
-// read more input characters to decide what should eventually be returned.
-// Eventually,
-// a called routine returns either a character or -1 to indicate parsing
-// failure.
-//
-static char32_t doDispatch(char32_t c, CharacterDispatch& dispatchTable) {
-  for (unsigned int i = 0; i < dispatchTable.len; ++i) {
-    if (static_cast<unsigned char>(dispatchTable.chars[i]) == c) {
-      return dispatchTable.dispatch[i](c);
-    }
-  }
-  return dispatchTable.dispatch[dispatchTable.len](c);
-}
-
-static char32_t thisKeyMetaCtrl =
-    0;  // holds pre-set Meta and/or Ctrl modifiers
-
-// Final dispatch routines -- return something
-//
-static char32_t normalKeyRoutine(char32_t c) { return thisKeyMetaCtrl | c; }
-static char32_t upArrowKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | UP_ARROW_KEY;
-}
-static char32_t downArrowKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | DOWN_ARROW_KEY;
-}
-static char32_t rightArrowKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | RIGHT_ARROW_KEY;
-}
-static char32_t leftArrowKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | LEFT_ARROW_KEY;
-}
-static char32_t homeKeyRoutine(char32_t) { return thisKeyMetaCtrl | HOME_KEY; }
-static char32_t endKeyRoutine(char32_t) { return thisKeyMetaCtrl | END_KEY; }
-static char32_t pageUpKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | PAGE_UP_KEY;
-}
-static char32_t pageDownKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | PAGE_DOWN_KEY;
-}
-static char32_t deleteCharRoutine(char32_t) {
-  return thisKeyMetaCtrl | ctrlChar('H');
-}  // key labeled Backspace
-static char32_t deleteKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | DELETE_KEY;
-}  // key labeled Delete
-static char32_t ctrlUpArrowKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | CTRL | UP_ARROW_KEY;
-}
-static char32_t ctrlDownArrowKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | CTRL | DOWN_ARROW_KEY;
-}
-static char32_t ctrlRightArrowKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | CTRL | RIGHT_ARROW_KEY;
-}
-static char32_t ctrlLeftArrowKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | CTRL | LEFT_ARROW_KEY;
-}
-static char32_t escFailureRoutine(char32_t) {
-  beep();
-  return -1;
-}
-
-// Handle ESC [ 1 ; 3 (or 5) <more stuff> escape sequences
-//
-static CharacterDispatchRoutine escLeftBracket1Semicolon3or5Routines[] = {
-    upArrowKeyRoutine, downArrowKeyRoutine, rightArrowKeyRoutine,
-    leftArrowKeyRoutine, escFailureRoutine};
-static CharacterDispatch escLeftBracket1Semicolon3or5Dispatch = {
-    4, "ABCD", escLeftBracket1Semicolon3or5Routines};
-
-// Handle ESC [ 1 ; <more stuff> escape sequences
-//
-static char32_t escLeftBracket1Semicolon3Routine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  thisKeyMetaCtrl |= META;
-  return doDispatch(c, escLeftBracket1Semicolon3or5Dispatch);
-}
-static char32_t escLeftBracket1Semicolon5Routine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  thisKeyMetaCtrl |= CTRL;
-  return doDispatch(c, escLeftBracket1Semicolon3or5Dispatch);
-}
-static CharacterDispatchRoutine escLeftBracket1SemicolonRoutines[] = {
-    escLeftBracket1Semicolon3Routine, escLeftBracket1Semicolon5Routine,
-    escFailureRoutine};
-static CharacterDispatch escLeftBracket1SemicolonDispatch = {
-    2, "35", escLeftBracket1SemicolonRoutines};
-
-// Handle ESC [ 1 <more stuff> escape sequences
-//
-static char32_t escLeftBracket1SemicolonRoutine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escLeftBracket1SemicolonDispatch);
-}
-static CharacterDispatchRoutine escLeftBracket1Routines[] = {
-    homeKeyRoutine, escLeftBracket1SemicolonRoutine, escFailureRoutine};
-static CharacterDispatch escLeftBracket1Dispatch = {2, "~;",
-                                                    escLeftBracket1Routines};
-
-// Handle ESC [ 3 <more stuff> escape sequences
-//
-static CharacterDispatchRoutine escLeftBracket3Routines[] = {deleteKeyRoutine,
-                                                             escFailureRoutine};
-static CharacterDispatch escLeftBracket3Dispatch = {1, "~",
-                                                    escLeftBracket3Routines};
-
-// Handle ESC [ 4 <more stuff> escape sequences
-//
-static CharacterDispatchRoutine escLeftBracket4Routines[] = {endKeyRoutine,
-                                                             escFailureRoutine};
-static CharacterDispatch escLeftBracket4Dispatch = {1, "~",
-                                                    escLeftBracket4Routines};
-
-// Handle ESC [ 5 <more stuff> escape sequences
-//
-static CharacterDispatchRoutine escLeftBracket5Routines[] = {pageUpKeyRoutine,
-                                                             escFailureRoutine};
-static CharacterDispatch escLeftBracket5Dispatch = {1, "~",
-                                                    escLeftBracket5Routines};
-
-// Handle ESC [ 6 <more stuff> escape sequences
-//
-static CharacterDispatchRoutine escLeftBracket6Routines[] = {pageDownKeyRoutine,
-                                                             escFailureRoutine};
-static CharacterDispatch escLeftBracket6Dispatch = {1, "~",
-                                                    escLeftBracket6Routines};
-
-// Handle ESC [ 7 <more stuff> escape sequences
-//
-static CharacterDispatchRoutine escLeftBracket7Routines[] = {homeKeyRoutine,
-                                                             escFailureRoutine};
-static CharacterDispatch escLeftBracket7Dispatch = {1, "~",
-                                                    escLeftBracket7Routines};
-
-// Handle ESC [ 8 <more stuff> escape sequences
-//
-static CharacterDispatchRoutine escLeftBracket8Routines[] = {endKeyRoutine,
-                                                             escFailureRoutine};
-static CharacterDispatch escLeftBracket8Dispatch = {1, "~",
-                                                    escLeftBracket8Routines};
-
-// Handle ESC [ <digit> escape sequences
-//
-static char32_t escLeftBracket0Routine(char32_t c) {
-  return escFailureRoutine(c);
-}
-static char32_t escLeftBracket1Routine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escLeftBracket1Dispatch);
-}
-static char32_t escLeftBracket2Routine(char32_t c) {
-  return escFailureRoutine(c);  // Insert key, unused
-}
-static char32_t escLeftBracket3Routine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escLeftBracket3Dispatch);
-}
-static char32_t escLeftBracket4Routine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escLeftBracket4Dispatch);
-}
-static char32_t escLeftBracket5Routine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escLeftBracket5Dispatch);
-}
-static char32_t escLeftBracket6Routine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escLeftBracket6Dispatch);
-}
-static char32_t escLeftBracket7Routine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escLeftBracket7Dispatch);
-}
-static char32_t escLeftBracket8Routine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escLeftBracket8Dispatch);
-}
-static char32_t escLeftBracket9Routine(char32_t c) {
-  return escFailureRoutine(c);
-}
-
-// Handle ESC [ <more stuff> escape sequences
-//
-static CharacterDispatchRoutine escLeftBracketRoutines[] = {
-    upArrowKeyRoutine,      downArrowKeyRoutine,    rightArrowKeyRoutine,
-    leftArrowKeyRoutine,    homeKeyRoutine,         endKeyRoutine,
-    escLeftBracket0Routine, escLeftBracket1Routine, escLeftBracket2Routine,
-    escLeftBracket3Routine, escLeftBracket4Routine, escLeftBracket5Routine,
-    escLeftBracket6Routine, escLeftBracket7Routine, escLeftBracket8Routine,
-    escLeftBracket9Routine, escFailureRoutine};
-static CharacterDispatch escLeftBracketDispatch = {16, "ABCDHF0123456789",
-                                                   escLeftBracketRoutines};
-
-// Handle ESC O <char> escape sequences
-//
-static CharacterDispatchRoutine escORoutines[] = {
-    upArrowKeyRoutine,       downArrowKeyRoutine,     rightArrowKeyRoutine,
-    leftArrowKeyRoutine,     homeKeyRoutine,          endKeyRoutine,
-    ctrlUpArrowKeyRoutine,   ctrlDownArrowKeyRoutine, ctrlRightArrowKeyRoutine,
-    ctrlLeftArrowKeyRoutine, escFailureRoutine};
-static CharacterDispatch escODispatch = {10, "ABCDHFabcd", escORoutines};
-
-// Initial ESC dispatch -- could be a Meta prefix or the start of an escape
-// sequence
-//
-static char32_t escLeftBracketRoutine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escLeftBracketDispatch);
-}
-static char32_t escORoutine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escODispatch);
-}
-static char32_t setMetaRoutine(char32_t c);  // need forward reference
-static CharacterDispatchRoutine escRoutines[] = {escLeftBracketRoutine,
-                                                 escORoutine, setMetaRoutine};
-static CharacterDispatch escDispatch = {2, "[O", escRoutines};
-
-// Initial dispatch -- we are not in the middle of anything yet
-//
-static char32_t escRoutine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escDispatch);
-}
-static CharacterDispatchRoutine initialRoutines[] = {
-    escRoutine, deleteCharRoutine, normalKeyRoutine};
-static CharacterDispatch initialDispatch = {2, "\x1B\x7F", initialRoutines};
-
-// Special handling for the ESC key because it does double duty
-//
-static char32_t setMetaRoutine(char32_t c) {
-  thisKeyMetaCtrl = META;
-  if (c == 0x1B) {  // another ESC, stay in ESC processing mode
-    c = readUnicodeCharacter();
-    if (c == 0) return 0;
-    return doDispatch(c, escDispatch);
-  }
-  return doDispatch(c, initialDispatch);
-}
-
-}  // namespace EscapeSequenceProcessing // move these out of global namespace
-
-#endif  // #ifndef _WIN32
-
-// linenoiseReadChar -- read a keystroke or keychord from the keyboard, and
-// translate it
-// into an encoded "keystroke".  When convenient, extended keys are translated
-// into their
-// simpler Emacs keystrokes, so an unmodified "left arrow" becomes Ctrl-B.
-//
-// A return value of zero means "no input available", and a return value of -1
-// means "invalid key".
-//
-static char32_t linenoiseReadChar(void) {
-#ifdef _WIN32
-
-  INPUT_RECORD rec;
-  DWORD count;
-  int modifierKeys = 0;
-  bool escSeen = false;
-  while (true) {
-    ReadConsoleInputW(console_in, &rec, 1, &count);
-#if 0  // helper for debugging keystrokes, display info in the debug "Output"
-       // window in the debugger
-        {
-            if ( rec.EventType == KEY_EVENT ) {
-                //if ( rec.Event.KeyEvent.uChar.UnicodeChar ) {
-                    char buf[1024];
-                    sprintf(
-                            buf,
-                            "Unicode character 0x%04X, repeat count %d, virtual keycode 0x%04X, "
-                            "virtual scancode 0x%04X, key %s%s%s%s%s\n",
-                            rec.Event.KeyEvent.uChar.UnicodeChar,
-                            rec.Event.KeyEvent.wRepeatCount,
-                            rec.Event.KeyEvent.wVirtualKeyCode,
-                            rec.Event.KeyEvent.wVirtualScanCode,
-                            rec.Event.KeyEvent.bKeyDown ? "down" : "up",
-                                (rec.Event.KeyEvent.dwControlKeyState & LEFT_CTRL_PRESSED)  ?
-                                    " L-Ctrl" : "",
-                                (rec.Event.KeyEvent.dwControlKeyState & RIGHT_CTRL_PRESSED) ?
-                                    " R-Ctrl" : "",
-                                (rec.Event.KeyEvent.dwControlKeyState & LEFT_ALT_PRESSED)   ?
-                                    " L-Alt"  : "",
-                                (rec.Event.KeyEvent.dwControlKeyState & RIGHT_ALT_PRESSED)  ?
-                                    " R-Alt"  : ""
-                           );
-                    OutputDebugStringA( buf );
-                //}
-            }
-        }
-#endif
-    if (rec.EventType != KEY_EVENT) {
-      continue;
-    }
-    // Windows provides for entry of characters that are not on your keyboard by
-    // sending the
-    // Unicode characters as a "key up" with virtual keycode 0x12 (VK_MENU ==
-    // Alt key) ...
-    // accept these characters, otherwise only process characters on "key down"
-    if (!rec.Event.KeyEvent.bKeyDown &&
-        rec.Event.KeyEvent.wVirtualKeyCode != VK_MENU) {
-      continue;
-    }
-    modifierKeys = 0;
-    // AltGr is encoded as ( LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED ), so don't
-    // treat this
-    // combination as either CTRL or META we just turn off those two bits, so it
-    // is still
-    // possible to combine CTRL and/or META with an AltGr key by using
-    // right-Ctrl and/or
-    // left-Alt
-    if ((rec.Event.KeyEvent.dwControlKeyState &
-         (LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED)) ==
-        (LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED)) {
-      rec.Event.KeyEvent.dwControlKeyState &=
-          ~(LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED);
-    }
-    if (rec.Event.KeyEvent.dwControlKeyState &
-        (RIGHT_CTRL_PRESSED | LEFT_CTRL_PRESSED)) {
-      modifierKeys |= CTRL;
-    }
-    if (rec.Event.KeyEvent.dwControlKeyState &
-        (RIGHT_ALT_PRESSED | LEFT_ALT_PRESSED)) {
-      modifierKeys |= META;
-    }
-    if (escSeen) {
-      modifierKeys |= META;
-    }
-    if (rec.Event.KeyEvent.uChar.UnicodeChar == 0) {
-      switch (rec.Event.KeyEvent.wVirtualKeyCode) {
-        case VK_LEFT:
-          return modifierKeys | LEFT_ARROW_KEY;
-        case VK_RIGHT:
-          return modifierKeys | RIGHT_ARROW_KEY;
-        case VK_UP:
-          return modifierKeys | UP_ARROW_KEY;
-        case VK_DOWN:
-          return modifierKeys | DOWN_ARROW_KEY;
-        case VK_DELETE:
-          return modifierKeys | DELETE_KEY;
-        case VK_HOME:
-          return modifierKeys | HOME_KEY;
-        case VK_END:
-          return modifierKeys | END_KEY;
-        case VK_PRIOR:
-          return modifierKeys | PAGE_UP_KEY;
-        case VK_NEXT:
-          return modifierKeys | PAGE_DOWN_KEY;
-        default:
-          continue;  // in raw mode, ReadConsoleInput shows shift, ctrl ...
-      }              //  ... ignore them
-    } else if (rec.Event.KeyEvent.uChar.UnicodeChar ==
-               ctrlChar('[')) {  // ESC, set flag for later
-      escSeen = true;
-      continue;
-    } else {
-      // we got a real character, return it
-      return modifierKeys | rec.Event.KeyEvent.uChar.UnicodeChar;
-    }
-  }
-
-#else
-  char32_t c;
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-
-// If _DEBUG_LINUX_KEYBOARD is set, then ctrl-^ puts us into a keyboard
-// debugging mode
-// where we print out decimal and decoded values for whatever the "terminal"
-// program
-// gives us on different keystrokes.  Hit ctrl-C to exit this mode.
-//
-#define _DEBUG_LINUX_KEYBOARD
-#if defined(_DEBUG_LINUX_KEYBOARD)
-  if (c == ctrlChar('^')) {  // ctrl-^, special debug mode, prints all keys hit,
-                             // ctrl-C to get out
-    printf(
-        "\nEntering keyboard debugging mode (on ctrl-^), press ctrl-C to exit "
-        "this mode\n");
-    while (true) {
-      unsigned char keys[10];
-      int ret = read(0, keys, 10);
-
-      if (ret <= 0) {
-        printf("\nret: %d\n", ret);
-      }
-      for (int i = 0; i < ret; ++i) {
-        char32_t key = static_cast<char32_t>(keys[i]);
-        char* friendlyTextPtr;
-        char friendlyTextBuf[10];
-        const char* prefixText = (key < 0x80) ? "" : "0x80+";
-        char32_t keyCopy = (key < 0x80) ? key : key - 0x80;
-        if (keyCopy >= '!' && keyCopy <= '~') {  // printable
-          friendlyTextBuf[0] = '\'';
-          friendlyTextBuf[1] = keyCopy;
-          friendlyTextBuf[2] = '\'';
-          friendlyTextBuf[3] = 0;
-          friendlyTextPtr = friendlyTextBuf;
-        } else if (keyCopy == ' ') {
-          friendlyTextPtr = const_cast<char*>("space");
-        } else if (keyCopy == 27) {
-          friendlyTextPtr = const_cast<char*>("ESC");
-        } else if (keyCopy == 0) {
-          friendlyTextPtr = const_cast<char*>("NUL");
-        } else if (keyCopy == 127) {
-          friendlyTextPtr = const_cast<char*>("DEL");
-        } else {
-          friendlyTextBuf[0] = '^';
-          friendlyTextBuf[1] = keyCopy + 0x40;
-          friendlyTextBuf[2] = 0;
-          friendlyTextPtr = friendlyTextBuf;
-        }
-        printf("%d x%02X (%s%s)  ", key, key, prefixText, friendlyTextPtr);
-      }
-      printf("\x1b[1G\n");  // go to first column of new line
-
-      // drop out of this loop on ctrl-C
-      if (keys[0] == ctrlChar('C')) {
-        printf("Leaving keyboard debugging mode (on ctrl-C)\n");
-        fflush(stdout);
-        return -2;
-      }
-    }
-  }
-#endif  // _DEBUG_LINUX_KEYBOARD
-
-  EscapeSequenceProcessing::thisKeyMetaCtrl =
-      0;  // no modifiers yet at initialDispatch
-  return EscapeSequenceProcessing::doDispatch(
-      c, EscapeSequenceProcessing::initialDispatch);
-#endif  // #_WIN32
-}
-
-/**
- * Free memory used in a recent command completion session
- *
- * @param lc pointer to a linenoiseCompletions struct
- */
-static void freeCompletions(linenoiseCompletions* lc) {
-  lc->completionStrings.clear();
-}
-
-/**
- * convert {CTRL + 'A'}, {CTRL + 'a'} and {CTRL + ctrlChar( 'A' )} into
- * ctrlChar( 'A' )
- * leave META alone
- *
- * @param c character to clean up
- * @return cleaned-up character
- */
-static int cleanupCtrl(int c) {
-  if (c & CTRL) {
-    int d = c & 0x1FF;
-    if (d >= 'a' && d <= 'z') {
-      c = (c + ('a' - ctrlChar('A'))) & ~CTRL;
-    }
-    if (d >= 'A' && d <= 'Z') {
-      c = (c + ('A' - ctrlChar('A'))) & ~CTRL;
-    }
-    if (d >= ctrlChar('A') && d <= ctrlChar('Z')) {
-      c = c & ~CTRL;
-    }
-  }
-  return c;
-}
-
-// break characters that may precede items to be completed
-static const char breakChars[] = " =+-/\\*?\"'`&<>;|@{([])}";
-
-// maximum number of completions to display without asking
-static const size_t completionCountCutoff = 100;
-
-/**
- * Handle command completion, using a completionCallback() routine to provide
- * possible substitutions
- * This routine handles the mechanics of updating the user's input buffer with
- * possible replacement
- * of text as the user selects a proposed completion string, or cancels the
- * completion attempt.
- * @param pi     PromptBase struct holding information about the prompt and our
- * screen position
- */
-int InputBuffer::completeLine(PromptBase& pi) {
-  linenoiseCompletions lc;
-  char32_t c = 0;
-
-  // completionCallback() expects a parsable entity, so find the previous break
-  // character and
-  // extract a copy to parse.  we also handle the case where tab is hit while
-  // not at end-of-line.
-  int startIndex = pos;
-  while (--startIndex >= 0) {
-    if (strchr(breakChars, buf32[startIndex])) {
-      break;
-    }
-  }
-  ++startIndex;
-  int itemLength = pos - startIndex;
-  Utf32String unicodeCopy(&buf32[startIndex], itemLength);
-  Utf8String parseItem(unicodeCopy);
-
-  // get a list of completions
-  completionCallback(parseItem.get(), &lc);
-
-  // if no completions, we are done
-  if (lc.completionStrings.size() == 0) {
-    beep();
-    freeCompletions(&lc);
-    return 0;
-  }
-
-  // at least one completion
-  int longestCommonPrefix = 0;
-  int displayLength = 0;
-  if (lc.completionStrings.size() == 1) {
-    longestCommonPrefix = static_cast<int>(lc.completionStrings[0].length());
-  } else {
-    bool keepGoing = true;
-    while (keepGoing) {
-      for (size_t j = 0; j < lc.completionStrings.size() - 1; ++j) {
-        char32_t c1 = lc.completionStrings[j][longestCommonPrefix];
-        char32_t c2 = lc.completionStrings[j + 1][longestCommonPrefix];
-        if ((0 == c1) || (0 == c2) || (c1 != c2)) {
-          keepGoing = false;
-          break;
-        }
-      }
-      if (keepGoing) {
-        ++longestCommonPrefix;
-      }
-    }
-  }
-  if (lc.completionStrings.size() != 1) {  // beep if ambiguous
-    beep();
-  }
-
-  // if we can extend the item, extend it and return to main loop
-  if (longestCommonPrefix > itemLength) {
-    displayLength = len + longestCommonPrefix - itemLength;
-    if (displayLength > buflen) {
-      longestCommonPrefix -= displayLength - buflen;  // don't overflow buffer
-      displayLength = buflen;                         // truncate the insertion
-      beep();                                         // and make a noise
-    }
-    Utf32String displayText(displayLength + 1);
-    memcpy(displayText.get(), buf32, sizeof(char32_t) * startIndex);
-    memcpy(&displayText[startIndex], &lc.completionStrings[0][0],
-           sizeof(char32_t) * longestCommonPrefix);
-    int tailIndex = startIndex + longestCommonPrefix;
-    memcpy(&displayText[tailIndex], &buf32[pos],
-           sizeof(char32_t) * (displayLength - tailIndex + 1));
-    copyString32(buf32, displayText.get(), displayLength);
-    pos = startIndex + longestCommonPrefix;
-    len = displayLength;
-    refreshLine(pi);
-    return 0;
-  }
-
-  // we can't complete any further, wait for second tab
-  do {
-    c = linenoiseReadChar();
-    c = cleanupCtrl(c);
-  } while (c == static_cast<char32_t>(-1));
-
-  // if any character other than tab, pass it to the main loop
-  if (c != ctrlChar('I')) {
-    freeCompletions(&lc);
-    return c;
-  }
-
-  // we got a second tab, maybe show list of possible completions
-  bool showCompletions = true;
-  bool onNewLine = false;
-  if (lc.completionStrings.size() > completionCountCutoff) {
-    int savePos =
-        pos;  // move cursor to EOL to avoid overwriting the command line
-    pos = len;
-    refreshLine(pi);
-    pos = savePos;
-    printf("\nDisplay all %u possibilities? (y or n)",
-           static_cast<unsigned int>(lc.completionStrings.size()));
-    fflush(stdout);
-    onNewLine = true;
-    while (c != 'y' && c != 'Y' && c != 'n' && c != 'N' && c != ctrlChar('C')) {
-      do {
-        c = linenoiseReadChar();
-        c = cleanupCtrl(c);
-      } while (c == static_cast<char32_t>(-1));
-    }
-    switch (c) {
-      case 'n':
-      case 'N':
-        showCompletions = false;
-        freeCompletions(&lc);
-        break;
-      case ctrlChar('C'):
-        showCompletions = false;
-        freeCompletions(&lc);
-        if (write(1, "^C", 2) == -1) return -1;  // Display the ^C we got
-        c = 0;
-        break;
-    }
-  }
-
-  // if showing the list, do it the way readline does it
-  bool stopList = false;
-  if (showCompletions) {
-    int longestCompletion = 0;
-    for (size_t j = 0; j < lc.completionStrings.size(); ++j) {
-      itemLength = static_cast<int>(lc.completionStrings[j].length());
-      if (itemLength > longestCompletion) {
-        longestCompletion = itemLength;
-      }
-    }
-    longestCompletion += 2;
-    int columnCount = pi.promptScreenColumns / longestCompletion;
-    if (columnCount < 1) {
-      columnCount = 1;
-    }
-    if (!onNewLine) {  // skip this if we showed "Display all %d possibilities?"
-      int savePos =
-          pos;  // move cursor to EOL to avoid overwriting the command line
-      pos = len;
-      refreshLine(pi);
-      pos = savePos;
-    }
-    size_t pauseRow = getScreenRows() - 1;
-    size_t rowCount =
-        (lc.completionStrings.size() + columnCount - 1) / columnCount;
-    for (size_t row = 0; row < rowCount; ++row) {
-      if (row == pauseRow) {
-        printf("\n--More--");
-        fflush(stdout);
-        c = 0;
-        bool doBeep = false;
-        while (c != ' ' && c != '\r' && c != '\n' && c != 'y' && c != 'Y' &&
-               c != 'n' && c != 'N' && c != 'q' && c != 'Q' &&
-               c != ctrlChar('C')) {
-          if (doBeep) {
-            beep();
-          }
-          doBeep = true;
-          do {
-            c = linenoiseReadChar();
-            c = cleanupCtrl(c);
-          } while (c == static_cast<char32_t>(-1));
-        }
-        switch (c) {
-          case ' ':
-          case 'y':
-          case 'Y':
-            printf("\r        \r");
-            pauseRow += getScreenRows() - 1;
-            break;
-          case '\r':
-          case '\n':
-            printf("\r        \r");
-            ++pauseRow;
-            break;
-          case 'n':
-          case 'N':
-          case 'q':
-          case 'Q':
-            printf("\r        \r");
-            stopList = true;
-            break;
-          case ctrlChar('C'):
-            if (write(1, "^C", 2) == -1) return -1;  // Display the ^C we got
-            stopList = true;
-            break;
-        }
-      } else {
-        printf("\n");
-      }
-      if (stopList) {
-        break;
-      }
-      for (int column = 0; column < columnCount; ++column) {
-        size_t index = (column * rowCount) + row;
-        if (index < lc.completionStrings.size()) {
-          itemLength = static_cast<int>(lc.completionStrings[index].length());
-          fflush(stdout);
-          if (write32(1, lc.completionStrings[index].get(), itemLength) == -1)
-            return -1;
-          if (((column + 1) * rowCount) + row < lc.completionStrings.size()) {
-            for (int k = itemLength; k < longestCompletion; ++k) {
-              printf(" ");
-            }
-          }
-        }
-      }
-    }
-    fflush(stdout);
-    freeCompletions(&lc);
-  }
-
-  // display the prompt on a new line, then redisplay the input buffer
-  if (!stopList || c == ctrlChar('C')) {
-    if (write(1, "\n", 1) == -1) return 0;
-  }
-  if (!pi.write()) return 0;
-#ifndef _WIN32
-  // we have to generate our own newline on line wrap on Linux
-  if (pi.promptIndentation == 0 && pi.promptExtraLines > 0)
-    if (write(1, "\n", 1) == -1) return 0;
-#endif
-  pi.promptCursorRowOffset = pi.promptExtraLines;
-  refreshLine(pi);
-  return 0;
-}
-
-/**
- * Clear the screen ONLY (no redisplay of anything)
- */
-void linenoiseClearScreen(void) {
-#ifdef _WIN32
-  COORD coord = {0, 0};
-  CONSOLE_SCREEN_BUFFER_INFO inf;
-  HANDLE screenHandle = GetStdHandle(STD_OUTPUT_HANDLE);
-  GetConsoleScreenBufferInfo(screenHandle, &inf);
-  SetConsoleCursorPosition(screenHandle, coord);
-  DWORD count;
-  FillConsoleOutputCharacterA(screenHandle, ' ', inf.dwSize.X * inf.dwSize.Y,
-                              coord, &count);
-#else
-  if (write(1, "\x1b[H\x1b[2J", 7) <= 0) return;
-#endif
-}
-
-void InputBuffer::clearScreen(PromptBase& pi) {
-  linenoiseClearScreen();
-  if (!pi.write()) return;
-#ifndef _WIN32
-  // we have to generate our own newline on line wrap on Linux
-  if (pi.promptIndentation == 0 && pi.promptExtraLines > 0)
-    if (write(1, "\n", 1) == -1) return;
-#endif
-  pi.promptCursorRowOffset = pi.promptExtraLines;
-  refreshLine(pi);
-}
-
-/**
- * Incremental history search -- take over the prompt and keyboard as the user
- * types a search
- * string, deletes characters from it, changes direction, and either accepts the
- * found line (for
- * execution orediting) or cancels.
- * @param pi        PromptBase struct holding information about the (old,
- * static) prompt and our
- *                  screen position
- * @param startChar the character that began the search, used to set the initial
- * direction
- */
-int InputBuffer::incrementalHistorySearch(PromptBase& pi, int startChar) {
-  size_t bufferSize;
-  size_t ucharCount = 0;
-
-  // if not already recalling, add the current line to the history list so we
-  // don't have to
-  // special case it
-  if (historyIndex == historyLen - 1) {
-    free(history[historyLen - 1]);
-    bufferSize = sizeof(char32_t) * len + 1;
-    unique_ptr<char[]> tempBuffer(new char[bufferSize]);
-    copyString32to8(tempBuffer.get(), bufferSize, buf32);
-    history[historyLen - 1] = strdup8(tempBuffer.get());
-  }
-  int historyLineLength = len;
-  int historyLinePosition = pos;
-  char32_t emptyBuffer[1];
-  char emptyWidths[1];
-  InputBuffer empty(emptyBuffer, emptyWidths, 1);
-  empty.refreshLine(pi);  // erase the old input first
-  DynamicPrompt dp(pi, (startChar == ctrlChar('R')) ? -1 : 1);
-
-  dp.promptPreviousLen = pi.promptPreviousLen;
-  dp.promptPreviousInputLen = pi.promptPreviousInputLen;
-  dynamicRefresh(dp, buf32, historyLineLength,
-                 historyLinePosition);  // draw user's text with our prompt
-
-  // loop until we get an exit character
-  int c = 0;
-  bool keepLooping = true;
-  bool useSearchedLine = true;
-  bool searchAgain = false;
-  char32_t* activeHistoryLine = 0;
-  while (keepLooping) {
-    c = linenoiseReadChar();
-    c = cleanupCtrl(c);  // convert CTRL + <char> into normal ctrl
-
-    switch (c) {
-      // these characters keep the selected text but do not execute it
-      case ctrlChar('A'):  // ctrl-A, move cursor to start of line
-      case HOME_KEY:
-      case ctrlChar('B'):  // ctrl-B, move cursor left by one character
-      case LEFT_ARROW_KEY:
-      case META + 'b':  // meta-B, move cursor left by one word
-      case META + 'B':
-      case CTRL + LEFT_ARROW_KEY:
-      case META + LEFT_ARROW_KEY:  // Emacs allows Meta, bash & readline don't
-      case ctrlChar('D'):
-      case META + 'd':  // meta-D, kill word to right of cursor
-      case META + 'D':
-      case ctrlChar('E'):  // ctrl-E, move cursor to end of line
-      case END_KEY:
-      case ctrlChar('F'):  // ctrl-F, move cursor right by one character
-      case RIGHT_ARROW_KEY:
-      case META + 'f':  // meta-F, move cursor right by one word
-      case META + 'F':
-      case CTRL + RIGHT_ARROW_KEY:
-      case META + RIGHT_ARROW_KEY:  // Emacs allows Meta, bash & readline don't
-      case META + ctrlChar('H'):
-      case ctrlChar('J'):
-      case ctrlChar('K'):  // ctrl-K, kill from cursor to end of line
-      case ctrlChar('M'):
-      case ctrlChar('N'):  // ctrl-N, recall next line in history
-      case ctrlChar('P'):  // ctrl-P, recall previous line in history
-      case DOWN_ARROW_KEY:
-      case UP_ARROW_KEY:
-      case ctrlChar('T'):  // ctrl-T, transpose characters
-      case ctrlChar(
-          'U'):  // ctrl-U, kill all characters to the left of the cursor
-      case ctrlChar('W'):
-      case META + 'y':  // meta-Y, "yank-pop", rotate popped text
-      case META + 'Y':
-      case 127:
-      case DELETE_KEY:
-      case META + '<':  // start of history
-      case PAGE_UP_KEY:
-      case META + '>':  // end of history
-      case PAGE_DOWN_KEY:
-        keepLooping = false;
-        break;
-
-      // these characters revert the input line to its previous state
-      case ctrlChar('C'):  // ctrl-C, abort this line
-      case ctrlChar('G'):
-      case ctrlChar('L'):  // ctrl-L, clear screen and redisplay line
-        keepLooping = false;
-        useSearchedLine = false;
-        if (c != ctrlChar('L')) {
-          c = -1;  // ctrl-C and ctrl-G just abort the search and do nothing
-                   // else
-        }
-        break;
-
-      // these characters stay in search mode and update the display
-      case ctrlChar('S'):
-      case ctrlChar('R'):
-        if (dp.searchTextLen ==
-            0) {  // if no current search text, recall previous text
-          if (previousSearchText.length()) {
-            dp.updateSearchText(previousSearchText.get());
-          }
-        }
-        if ((dp.direction == 1 && c == ctrlChar('R')) ||
-            (dp.direction == -1 && c == ctrlChar('S'))) {
-          dp.direction = 0 - dp.direction;  // reverse direction
-          dp.updateSearchPrompt();          // change the prompt
-        } else {
-          searchAgain = true;  // same direction, search again
-        }
-        break;
-
-// job control is its own thing
-#ifndef _WIN32
-      case ctrlChar('Z'):  // ctrl-Z, job control
-        disableRawMode();  // Returning to Linux (whatever) shell, leave raw
-                           // mode
-        raise(SIGSTOP);    // Break out in mid-line
-        enableRawMode();   // Back from Linux shell, re-enter raw mode
-        {
-          bufferSize = historyLineLength + 1;
-          unique_ptr<char32_t[]> tempUnicode(new char32_t[bufferSize]);
-          copyString8to32(tempUnicode.get(), bufferSize, ucharCount,
-                          history[historyIndex]);
-          dynamicRefresh(dp, tempUnicode.get(), historyLineLength,
-                         historyLinePosition);
-        }
-        continue;
-        break;
-#endif
-
-      // these characters update the search string, and hence the selected input
-      // line
-      case ctrlChar('H'):  // backspace/ctrl-H, delete char to left of cursor
-        if (dp.searchTextLen > 0) {
-          unique_ptr<char32_t[]> tempUnicode(new char32_t[dp.searchTextLen]);
-          --dp.searchTextLen;
-          dp.searchText[dp.searchTextLen] = 0;
-          copyString32(tempUnicode.get(), dp.searchText.get(),
-                       dp.searchTextLen);
-          dp.updateSearchText(tempUnicode.get());
-        } else {
-          beep();
-        }
-        break;
-
-      case ctrlChar('Y'):  // ctrl-Y, yank killed text
-        break;
-
-      default:
-        if (!isControlChar(c) && c <= 0x0010FFFF) {  // not an action character
-          unique_ptr<char32_t[]> tempUnicode(
-              new char32_t[dp.searchTextLen + 2]);
-          copyString32(tempUnicode.get(), dp.searchText.get(),
-                       dp.searchTextLen);
-          tempUnicode[dp.searchTextLen] = c;
-          tempUnicode[dp.searchTextLen + 1] = 0;
-          dp.updateSearchText(tempUnicode.get());
-        } else {
-          beep();
-        }
-    }  // switch
-
-    // if we are staying in search mode, search now
-    if (keepLooping) {
-      bufferSize = historyLineLength + 1;
-      if (activeHistoryLine) {
-        delete[] activeHistoryLine;
-        activeHistoryLine = nullptr;
-      }
-      activeHistoryLine = new char32_t[bufferSize];
-      copyString8to32(activeHistoryLine, bufferSize, ucharCount,
-                      history[historyIndex]);
-      if (dp.searchTextLen > 0) {
-        bool found = false;
-        int historySearchIndex = historyIndex;
-        int lineLength = static_cast<int>(ucharCount);
-        int lineSearchPos = historyLinePosition;
-        if (searchAgain) {
-          lineSearchPos += dp.direction;
-        }
-        searchAgain = false;
-        while (true) {
-          while ((dp.direction > 0) ? (lineSearchPos < lineLength)
-                                    : (lineSearchPos >= 0)) {
-            if (strncmp32(dp.searchText.get(),
-                          &activeHistoryLine[lineSearchPos],
-                          dp.searchTextLen) == 0) {
-              found = true;
-              break;
-            }
-            lineSearchPos += dp.direction;
-          }
-          if (found) {
-            historyIndex = historySearchIndex;
-            historyLineLength = lineLength;
-            historyLinePosition = lineSearchPos;
-            break;
-          } else if ((dp.direction > 0) ? (historySearchIndex < historyLen - 1)
-                                        : (historySearchIndex > 0)) {
-            historySearchIndex += dp.direction;
-            bufferSize = strlen8(history[historySearchIndex]) + 1;
-            delete[] activeHistoryLine;
-            activeHistoryLine = nullptr;
-            activeHistoryLine = new char32_t[bufferSize];
-            copyString8to32(activeHistoryLine, bufferSize, ucharCount,
-                            history[historySearchIndex]);
-            lineLength = static_cast<int>(ucharCount);
-            lineSearchPos =
-                (dp.direction > 0) ? 0 : (lineLength - dp.searchTextLen);
-          } else {
-            beep();
-            break;
-          }
-        };  // while
-      }
-      if (activeHistoryLine) {
-        delete[] activeHistoryLine;
-        activeHistoryLine = nullptr;
-      }
-      bufferSize = historyLineLength + 1;
-      activeHistoryLine = new char32_t[bufferSize];
-      copyString8to32(activeHistoryLine, bufferSize, ucharCount,
-                      history[historyIndex]);
-      dynamicRefresh(dp, activeHistoryLine, historyLineLength,
-                     historyLinePosition);  // draw user's text with our prompt
-    }
-  }  // while
-
-  // leaving history search, restore previous prompt, maybe make searched line
-  // current
-  PromptBase pb;
-  pb.promptChars = pi.promptIndentation;
-  pb.promptBytes = pi.promptBytes;
-  Utf32String tempUnicode(pb.promptBytes + 1);
-
-  copyString32(tempUnicode.get(), &pi.promptText[pi.promptLastLinePosition],
-               pb.promptBytes - pi.promptLastLinePosition);
-  tempUnicode.initFromBuffer();
-  pb.promptText = tempUnicode;
-  pb.promptExtraLines = 0;
-  pb.promptIndentation = pi.promptIndentation;
-  pb.promptLastLinePosition = 0;
-  pb.promptPreviousInputLen = historyLineLength;
-  pb.promptCursorRowOffset = dp.promptCursorRowOffset;
-  pb.promptScreenColumns = pi.promptScreenColumns;
-  pb.promptPreviousLen = dp.promptChars;
-  if (useSearchedLine && activeHistoryLine) {
-    historyRecallMostRecent = true;
-    copyString32(buf32, activeHistoryLine, buflen + 1);
-    len = historyLineLength;
-    pos = historyLinePosition;
-  }
-  if (activeHistoryLine) {
-    delete[] activeHistoryLine;
-    activeHistoryLine = nullptr;
-  }
-  dynamicRefresh(pb, buf32, len,
-                 pos);  // redraw the original prompt with current input
-  pi.promptPreviousInputLen = len;
-  pi.promptCursorRowOffset = pi.promptExtraLines + pb.promptCursorRowOffset;
-  previousSearchText =
-      dp.searchText;  // save search text for possible reuse on ctrl-R ctrl-R
-  return c;           // pass a character or -1 back to main loop
-}
-
-static bool isCharacterAlphanumeric(char32_t testChar) {
-#ifdef _WIN32
-  return (iswalnum((wint_t)testChar) != 0 ? true : false);
-#else
-  return (iswalnum(testChar) != 0 ? true : false);
-#endif
-}
-
-#ifndef _WIN32
-static bool gotResize = false;
-#endif
-static int keyType = 0;
-
-int InputBuffer::getInputLine(PromptBase& pi) {
-  keyType = 0;
-
-  // The latest history entry is always our current buffer
-  if (len > 0) {
-    size_t bufferSize = sizeof(char32_t) * len + 1;
-    unique_ptr<char[]> tempBuffer(new char[bufferSize]);
-    copyString32to8(tempBuffer.get(), bufferSize, buf32);
-    linenoiseHistoryAdd(tempBuffer.get());
-  } else {
-    linenoiseHistoryAdd("");
-  }
-  historyIndex = historyLen - 1;
-  historyRecallMostRecent = false;
-
-  // display the prompt
-  if (!pi.write()) return -1;
-
-#ifndef _WIN32
-  // we have to generate our own newline on line wrap on Linux
-  if (pi.promptIndentation == 0 && pi.promptExtraLines > 0)
-    if (write(1, "\n", 1) == -1) return -1;
-#endif
-
-  // the cursor starts out at the end of the prompt
-  pi.promptCursorRowOffset = pi.promptExtraLines;
-
-  // kill and yank start in "other" mode
-  killRing.lastAction = KillRing::actionOther;
-
-  // when history search returns control to us, we execute its terminating
-  // keystroke
-  int terminatingKeystroke = -1;
-
-  // if there is already text in the buffer, display it first
-  if (len > 0) {
-    refreshLine(pi);
-  }
-
-  // loop collecting characters, respond to line editing characters
-  while (true) {
-    int c;
-    if (terminatingKeystroke == -1) {
-      c = linenoiseReadChar();  // get a new keystroke
-
-      keyType = 0; 
-      if (c != 0) {
-        // set flag that we got some input
-        if (c == ctrlChar('C')) {
-          keyType = 1;
-        } else if (c == ctrlChar('D')) {
-          keyType = 2;
-        }
-      }
-
-#ifndef _WIN32
-      if (c == 0 && gotResize) {
-        // caught a window resize event
-        // now redraw the prompt and line
-        gotResize = false;
-        pi.promptScreenColumns = getScreenColumns();
-        dynamicRefresh(pi, buf32, len,
-                       pos);  // redraw the original prompt with current input
-        continue;
-      }
-#endif
-    } else {
-      c = terminatingKeystroke;   // use the terminating keystroke from search
-      terminatingKeystroke = -1;  // clear it once we've used it
-    }
-
-    c = cleanupCtrl(c);  // convert CTRL + <char> into normal ctrl
-
-    if (c == 0) {
-      return len;
-    }
-
-    if (c == -1) {
-      refreshLine(pi);
-      continue;
-    }
-
-    if (c == -2) {
-      if (!pi.write()) return -1;
-      refreshLine(pi);
-      continue;
-    }
-
-    // ctrl-I/tab, command completion, needs to be before switch statement
-    if (c == ctrlChar('I') && completionCallback) {
-      killRing.lastAction = KillRing::actionOther;
-      historyRecallMostRecent = false;
-
-      // completeLine does the actual completion and replacement
-      c = completeLine(pi);
-
-      if (c < 0)  // return on error
-        return len;
-
-      if (c == 0)  // read next character when 0
-        continue;
-
-      // deliberate fall-through here, so we use the terminating character
-    }
-
-    switch (c) {
-      case ctrlChar('A'):  // ctrl-A, move cursor to start of line
-      case HOME_KEY:
-        killRing.lastAction = KillRing::actionOther;
-        pos = 0;
-        refreshLine(pi);
-        break;
-
-      case ctrlChar('B'):  // ctrl-B, move cursor left by one character
-      case LEFT_ARROW_KEY:
-        killRing.lastAction = KillRing::actionOther;
-        if (pos > 0) {
-          --pos;
-          refreshLine(pi);
-        }
-        break;
-
-      case META + 'b':  // meta-B, move cursor left by one word
-      case META + 'B':
-      case CTRL + LEFT_ARROW_KEY:
-      case META + LEFT_ARROW_KEY:  // Emacs allows Meta, bash & readline don't
-        killRing.lastAction = KillRing::actionOther;
-        if (pos > 0) {
-          while (pos > 0 && !isCharacterAlphanumeric(buf32[pos - 1])) {
-            --pos;
-          }
-          while (pos > 0 && isCharacterAlphanumeric(buf32[pos - 1])) {
-            --pos;
-          }
-          refreshLine(pi);
-        }
-        break;
-
-      case ctrlChar('C'):  // ctrl-C, abort this line
-        killRing.lastAction = KillRing::actionOther;
-        historyRecallMostRecent = false;
-        errno = EAGAIN;
-        --historyLen;
-        free(history[historyLen]);
-        // we need one last refresh with the cursor at the end of the line
-        // so we don't display the next prompt over the previous input line
-        pos = len;  // pass len as pos for EOL
-        refreshLine(pi);
-        if (write(1, "^C", 2) == -1) return -1;  // Display the ^C we got
-        return -1;
-
-      case META + 'c':  // meta-C, give word initial Cap
-      case META + 'C':
-        killRing.lastAction = KillRing::actionOther;
-        historyRecallMostRecent = false;
-        if (pos < len) {
-          while (pos < len && !isCharacterAlphanumeric(buf32[pos])) {
-            ++pos;
-          }
-          if (pos < len && isCharacterAlphanumeric(buf32[pos])) {
-            if (buf32[pos] >= 'a' && buf32[pos] <= 'z') {
-              buf32[pos] += 'A' - 'a';
-            }
-            ++pos;
-          }
-          while (pos < len && isCharacterAlphanumeric(buf32[pos])) {
-            if (buf32[pos] >= 'A' && buf32[pos] <= 'Z') {
-              buf32[pos] += 'a' - 'A';
-            }
-            ++pos;
-          }
-          refreshLine(pi);
-        }
-        break;
-
-      // ctrl-D, delete the character under the cursor
-      // on an empty line, exit the shell
-      case ctrlChar('D'):
-        killRing.lastAction = KillRing::actionOther;
-        if (len > 0 && pos < len) {
-          historyRecallMostRecent = false;
-          memmove(buf32 + pos, buf32 + pos + 1, sizeof(char32_t) * (len - pos));
-          --len;
-          refreshLine(pi);
-        } else if (len == 0) {
-          --historyLen;
-          free(history[historyLen]);
-          return -1;
-        }
-        break;
-
-      case META + 'd':  // meta-D, kill word to right of cursor
-      case META + 'D':
-        if (pos < len) {
-          historyRecallMostRecent = false;
-          int endingPos = pos;
-          while (endingPos < len &&
-                 !isCharacterAlphanumeric(buf32[endingPos])) {
-            ++endingPos;
-          }
-          while (endingPos < len && isCharacterAlphanumeric(buf32[endingPos])) {
-            ++endingPos;
-          }
-          killRing.kill(&buf32[pos], endingPos - pos, true);
-          memmove(buf32 + pos, buf32 + endingPos,
-                  sizeof(char32_t) * (len - endingPos + 1));
-          len -= endingPos - pos;
-          refreshLine(pi);
-        }
-        killRing.lastAction = KillRing::actionKill;
-        break;
-
-      case ctrlChar('E'):  // ctrl-E, move cursor to end of line
-      case END_KEY:
-        killRing.lastAction = KillRing::actionOther;
-        pos = len;
-        refreshLine(pi);
-        break;
-
-      case ctrlChar('F'):  // ctrl-F, move cursor right by one character
-      case RIGHT_ARROW_KEY:
-        killRing.lastAction = KillRing::actionOther;
-        if (pos < len) {
-          ++pos;
-          refreshLine(pi);
-        }
-        break;
-
-      case META + 'f':  // meta-F, move cursor right by one word
-      case META + 'F':
-      case CTRL + RIGHT_ARROW_KEY:
-      case META + RIGHT_ARROW_KEY:  // Emacs allows Meta, bash & readline don't
-        killRing.lastAction = KillRing::actionOther;
-        if (pos < len) {
-          while (pos < len && !isCharacterAlphanumeric(buf32[pos])) {
-            ++pos;
-          }
-          while (pos < len && isCharacterAlphanumeric(buf32[pos])) {
-            ++pos;
-          }
-          refreshLine(pi);
-        }
-        break;
-
-      case ctrlChar('H'):  // backspace/ctrl-H, delete char to left of cursor
-        killRing.lastAction = KillRing::actionOther;
-        if (pos > 0) {
-          historyRecallMostRecent = false;
-          memmove(buf32 + pos - 1, buf32 + pos,
-                  sizeof(char32_t) * (1 + len - pos));
-          --pos;
-          --len;
-          refreshLine(pi);
-        }
-        break;
-
-      // meta-Backspace, kill word to left of cursor
-      case META + ctrlChar('H'):
-        if (pos > 0) {
-          historyRecallMostRecent = false;
-          int startingPos = pos;
-          while (pos > 0 && !isCharacterAlphanumeric(buf32[pos - 1])) {
-            --pos;
-          }
-          while (pos > 0 && isCharacterAlphanumeric(buf32[pos - 1])) {
-            --pos;
-          }
-          killRing.kill(&buf32[pos], startingPos - pos, false);
-          memmove(buf32 + pos, buf32 + startingPos,
-                  sizeof(char32_t) * (len - startingPos + 1));
-          len -= startingPos - pos;
-          refreshLine(pi);
-        }
-        killRing.lastAction = KillRing::actionKill;
-        break;
-
-      case ctrlChar('J'):  // ctrl-J/linefeed/newline, accept line
-      case ctrlChar('M'):  // ctrl-M/return/enter
-        killRing.lastAction = KillRing::actionOther;
-        // we need one last refresh with the cursor at the end of the line
-        // so we don't display the next prompt over the previous input line
-        pos = len;  // pass len as pos for EOL
-        refreshLine(pi);
-        historyPreviousIndex = historyRecallMostRecent ? historyIndex : -2;
-        --historyLen;
-        free(history[historyLen]);
-        return len;
-
-      case ctrlChar('K'):  // ctrl-K, kill from cursor to end of line
-        killRing.kill(&buf32[pos], len - pos, true);
-        buf32[pos] = '\0';
-        len = pos;
-        refreshLine(pi);
-        killRing.lastAction = KillRing::actionKill;
-        historyRecallMostRecent = false;
-        break;
-
-      case ctrlChar('L'):  // ctrl-L, clear screen and redisplay line
-        clearScreen(pi);
-        break;
-
-      case META + 'l':  // meta-L, lowercase word
-      case META + 'L':
-        killRing.lastAction = KillRing::actionOther;
-        if (pos < len) {
-          historyRecallMostRecent = false;
-          while (pos < len && !isCharacterAlphanumeric(buf32[pos])) {
-            ++pos;
-          }
-          while (pos < len && isCharacterAlphanumeric(buf32[pos])) {
-            if (buf32[pos] >= 'A' && buf32[pos] <= 'Z') {
-              buf32[pos] += 'a' - 'A';
-            }
-            ++pos;
-          }
-          refreshLine(pi);
-        }
-        break;
-
-      case ctrlChar('N'):  // ctrl-N, recall next line in history
-      case ctrlChar('P'):  // ctrl-P, recall previous line in history
-      case DOWN_ARROW_KEY:
-      case UP_ARROW_KEY:
-        killRing.lastAction = KillRing::actionOther;
-        // if not already recalling, add the current line to the history list so
-        // we don't
-        // have to special case it
-        if (historyIndex == historyLen - 1) {
-          free(history[historyLen - 1]);
-          size_t tempBufferSize = sizeof(char32_t) * len + 1;
-          unique_ptr<char[]> tempBuffer(new char[tempBufferSize]);
-          copyString32to8(tempBuffer.get(), tempBufferSize, buf32);
-          history[historyLen - 1] = strdup8(tempBuffer.get());
-        }
-        if (historyLen > 1) {
-          if (c == UP_ARROW_KEY) {
-            c = ctrlChar('P');
-          }
-          if (historyPreviousIndex != -2 && c != ctrlChar('P')) {
-            historyIndex =
-                1 + historyPreviousIndex;  // emulate Windows down-arrow
-          } else {
-            historyIndex += (c == ctrlChar('P')) ? -1 : 1;
-          }
-          historyPreviousIndex = -2;
-          if (historyIndex < 0) {
-            historyIndex = 0;
-            break;
-          } else if (historyIndex >= historyLen) {
-            historyIndex = historyLen - 1;
-            break;
-          }
-          historyRecallMostRecent = true;
-          size_t ucharCount = 0;
-          copyString8to32(buf32, buflen, ucharCount, history[historyIndex]);
-          len = pos = static_cast<int>(ucharCount);
-          refreshLine(pi);
-        }
-        break;
-
-      case ctrlChar('R'):  // ctrl-R, reverse history search
-      case ctrlChar('S'):  // ctrl-S, forward history search
-        terminatingKeystroke = incrementalHistorySearch(pi, c);
-        break;
-
-      case ctrlChar('T'):  // ctrl-T, transpose characters
-        killRing.lastAction = KillRing::actionOther;
-        if (pos > 0 && len > 1) {
-          historyRecallMostRecent = false;
-          size_t leftCharPos = (pos == len) ? pos - 2 : pos - 1;
-          char32_t aux = buf32[leftCharPos];
-          buf32[leftCharPos] = buf32[leftCharPos + 1];
-          buf32[leftCharPos + 1] = aux;
-          if (pos != len) ++pos;
-          refreshLine(pi);
-        }
-        break;
-
-      case ctrlChar(
-          'U'):  // ctrl-U, kill all characters to the left of the cursor
-        if (pos > 0) {
-          historyRecallMostRecent = false;
-          killRing.kill(&buf32[0], pos, false);
-          len -= pos;
-          memmove(buf32, buf32 + pos, sizeof(char32_t) * (len + 1));
-          pos = 0;
-          refreshLine(pi);
-        }
-        killRing.lastAction = KillRing::actionKill;
-        break;
-
-      case META + 'u':  // meta-U, uppercase word
-      case META + 'U':
-        killRing.lastAction = KillRing::actionOther;
-        if (pos < len) {
-          historyRecallMostRecent = false;
-          while (pos < len && !isCharacterAlphanumeric(buf32[pos])) {
-            ++pos;
-          }
-          while (pos < len && isCharacterAlphanumeric(buf32[pos])) {
-            if (buf32[pos] >= 'a' && buf32[pos] <= 'z') {
-              buf32[pos] += 'A' - 'a';
-            }
-            ++pos;
-          }
-          refreshLine(pi);
-        }
-        break;
-
-      // ctrl-W, kill to whitespace (not word) to left of cursor
-      case ctrlChar('W'):
-        if (pos > 0) {
-          historyRecallMostRecent = false;
-          int startingPos = pos;
-          while (pos > 0 && buf32[pos - 1] == ' ') {
-            --pos;
-          }
-          while (pos > 0 && buf32[pos - 1] != ' ') {
-            --pos;
-          }
-          killRing.kill(&buf32[pos], startingPos - pos, false);
-          memmove(buf32 + pos, buf32 + startingPos,
-                  sizeof(char32_t) * (len - startingPos + 1));
-          len -= startingPos - pos;
-          refreshLine(pi);
-        }
-        killRing.lastAction = KillRing::actionKill;
-        break;
-
-      case ctrlChar('Y'):  // ctrl-Y, yank killed text
-        historyRecallMostRecent = false;
-        {
-          Utf32String* restoredText = killRing.yank();
-          if (restoredText) {
-            bool truncated = false;
-            size_t ucharCount = restoredText->length();
-            if (ucharCount > static_cast<size_t>(buflen - len)) {
-              ucharCount = buflen - len;
-              truncated = true;
-            }
-            memmove(buf32 + pos + ucharCount, buf32 + pos,
-                    sizeof(char32_t) * (len - pos + 1));
-            memmove(buf32 + pos, restoredText->get(),
-                    sizeof(char32_t) * ucharCount);
-            pos += static_cast<int>(ucharCount);
-            len += static_cast<int>(ucharCount);
-            refreshLine(pi);
-            killRing.lastAction = KillRing::actionYank;
-            killRing.lastYankSize = ucharCount;
-            if (truncated) {
-              beep();
-            }
-          } else {
-            beep();
-          }
-        }
-        break;
-
-      case META + 'y':  // meta-Y, "yank-pop", rotate popped text
-      case META + 'Y':
-        if (killRing.lastAction == KillRing::actionYank) {
-          historyRecallMostRecent = false;
-          Utf32String* restoredText = killRing.yankPop();
-          if (restoredText) {
-            bool truncated = false;
-            size_t ucharCount = restoredText->length();
-            if (ucharCount >
-                static_cast<size_t>(killRing.lastYankSize + buflen - len)) {
-              ucharCount = killRing.lastYankSize + buflen - len;
-              truncated = true;
-            }
-            if (ucharCount > killRing.lastYankSize) {
-              memmove(buf32 + pos + ucharCount - killRing.lastYankSize,
-                      buf32 + pos, sizeof(char32_t) * (len - pos + 1));
-              memmove(buf32 + pos - killRing.lastYankSize, restoredText->get(),
-                      sizeof(char32_t) * ucharCount);
-            } else {
-              memmove(buf32 + pos - killRing.lastYankSize, restoredText->get(),
-                      sizeof(char32_t) * ucharCount);
-              memmove(buf32 + pos + ucharCount - killRing.lastYankSize,
-                      buf32 + pos, sizeof(char32_t) * (len - pos + 1));
-            }
-            pos += static_cast<int>(ucharCount - killRing.lastYankSize);
-            len += static_cast<int>(ucharCount - killRing.lastYankSize);
-            killRing.lastYankSize = ucharCount;
-            refreshLine(pi);
-            if (truncated) {
-              beep();
-            }
-            break;
-          }
-        }
-        beep();
-        break;
-
-#ifndef _WIN32
-      case ctrlChar('Z'):  // ctrl-Z, job control
-        disableRawMode();  // Returning to Linux (whatever) shell, leave raw
-                           // mode
-        raise(SIGSTOP);    // Break out in mid-line
-        enableRawMode();   // Back from Linux shell, re-enter raw mode
-        if (!pi.write()) break;  // Redraw prompt
-        refreshLine(pi);         // Refresh the line
-        break;
-#endif
-
-      // DEL, delete the character under the cursor
-      case 127:
-      case DELETE_KEY:
-        killRing.lastAction = KillRing::actionOther;
-        if (len > 0 && pos < len) {
-          historyRecallMostRecent = false;
-          memmove(buf32 + pos, buf32 + pos + 1, sizeof(char32_t) * (len - pos));
-          --len;
-          refreshLine(pi);
-        }
-        break;
-
-      case META + '<':     // meta-<, beginning of history
-      case PAGE_UP_KEY:    // Page Up, beginning of history
-      case META + '>':     // meta->, end of history
-      case PAGE_DOWN_KEY:  // Page Down, end of history
-        killRing.lastAction = KillRing::actionOther;
-        // if not already recalling, add the current line to the history list so
-        // we don't
-        // have to special case it
-        if (historyIndex == historyLen - 1) {
-          free(history[historyLen - 1]);
-          size_t tempBufferSize = sizeof(char32_t) * len + 1;
-          unique_ptr<char[]> tempBuffer(new char[tempBufferSize]);
-          copyString32to8(tempBuffer.get(), tempBufferSize, buf32);
-          history[historyLen - 1] = strdup8(tempBuffer.get());
-        }
-        if (historyLen > 1) {
-          historyIndex =
-              (c == META + '<' || c == PAGE_UP_KEY) ? 0 : historyLen - 1;
-          historyPreviousIndex = -2;
-          historyRecallMostRecent = true;
-          size_t ucharCount = 0;
-          copyString8to32(buf32, buflen, ucharCount, history[historyIndex]);
-          len = pos = static_cast<int>(ucharCount);
-          refreshLine(pi);
-        }
-        break;
-
-      // not one of our special characters, maybe insert it in the buffer
-      default:
-        killRing.lastAction = KillRing::actionOther;
-        historyRecallMostRecent = false;
-        if (c & (META | CTRL)) {  // beep on unknown Ctrl and/or Meta keys
-          beep();
-          break;
-        }
-        if (len < buflen) {
-          if (isControlChar(c)) {  // don't insert control characters
-            beep();
-            break;
-          }
-          if (len == pos) {  // at end of buffer
-            buf32[pos] = c;
-            ++pos;
-            ++len;
-            buf32[len] = '\0';
-            int inputLen = calculateColumnPosition(buf32, len);
-            if (pi.promptIndentation + inputLen < pi.promptScreenColumns) {
-              if (inputLen > pi.promptPreviousInputLen)
-                pi.promptPreviousInputLen = inputLen;
-              /* Avoid a full update of the line in the
-               * trivial case. */
-              if (write32(1, reinterpret_cast<char32_t*>(&c), 1) == -1)
-                return -1;
-            } else {
-              refreshLine(pi);
-            }
-          } else {  // not at end of buffer, have to move characters to our
-                    // right
-            memmove(buf32 + pos + 1, buf32 + pos,
-                    sizeof(char32_t) * (len - pos));
-            buf32[pos] = c;
-            ++len;
-            ++pos;
-            buf32[len] = '\0';
-            refreshLine(pi);
-          }
-        } else {
-          beep();  // buffer is full, beep on new characters
-        }
-        break;
-    }
-  }
-  return len;
-}
-
-static string preloadedBufferContents;  // used with linenoisePreloadBuffer
-static string preloadErrorMessage;
-
-/**
- * linenoisePreloadBuffer provides text to be inserted into the command buffer
- *
- * the provided text will be processed to be usable and will be used to preload
- * the input buffer on the next call to linenoise()
- *
- * @param preloadText text to begin with on the next call to linenoise()
- */
-void linenoisePreloadBuffer(const char* preloadText) {
-  if (!preloadText) {
-    return;
-  }
-  int bufferSize = static_cast<int>(strlen(preloadText) + 1);
-  unique_ptr<char[]> tempBuffer(new char[bufferSize]);
-  strncpy(&tempBuffer[0], preloadText, bufferSize);
-
-  // remove characters that won't display correctly
-  char* pIn = &tempBuffer[0];
-  char* pOut = pIn;
-  bool controlsStripped = false;
-  bool whitespaceSeen = false;
-  while (*pIn) {
-    unsigned char c =
-        *pIn++;       // we need unsigned so chars 0x80 and above are allowed
-    if ('\r' == c) {  // silently skip CR
-      continue;
-    }
-    if ('\n' == c || '\t' == c) {  // note newline or tab
-      whitespaceSeen = true;
-      continue;
-    }
-    if (isControlChar(
-            c)) {  // remove other control characters, flag for message
-      controlsStripped = true;
-      *pOut++ = ' ';
-      continue;
-    }
-    if (whitespaceSeen) {  // convert whitespace to a single space
-      *pOut++ = ' ';
-      whitespaceSeen = false;
-    }
-    *pOut++ = c;
-  }
-  *pOut = 0;
-  int processedLength = static_cast<int>(pOut - tempBuffer.get());
-  bool lineTruncated = false;
-  if (processedLength > (LINENOISE_MAX_LINE - 1)) {
-    lineTruncated = true;
-    tempBuffer[LINENOISE_MAX_LINE - 1] = 0;
-  }
-  preloadedBufferContents = tempBuffer.get();
-  if (controlsStripped) {
-    preloadErrorMessage +=
-        " [Edited line: control characters were converted to spaces]\n";
-  }
-  if (lineTruncated) {
-    preloadErrorMessage += " [Edited line: the line length was reduced from ";
-    char buf[128];
-    snprintf(buf, sizeof(buf), "%d to %d]\n", processedLength,
-             (LINENOISE_MAX_LINE - 1));
-    preloadErrorMessage += buf;
-  }
-}
-
-/**
- * linenoise is a readline replacement.
- *
- * call it with a prompt to display and it will return a line of input from the
- * user
- *
- * @param prompt text of prompt to display to the user
- * @return       the returned string belongs to the caller on return and must be
- * freed to prevent
- *               memory leaks
- */
-char* linenoise(const char* prompt) {
-#ifndef _WIN32
-  gotResize = false;
-#endif
-  if (isatty(STDIN_FILENO)) {  // input is from a terminal
-    char32_t buf32[LINENOISE_MAX_LINE];
-    char charWidths[LINENOISE_MAX_LINE];
-    if (!preloadErrorMessage.empty()) {
-      printf("%s", preloadErrorMessage.c_str());
-      fflush(stdout);
-      preloadErrorMessage.clear();
-    }
-    PromptInfo pi(prompt, getScreenColumns());
-    if (isUnsupportedTerm()) {
-      if (!pi.write()) return 0;
-      fflush(stdout);
-      if (preloadedBufferContents.empty()) {
-        unique_ptr<char[]> buf8(new char[LINENOISE_MAX_LINE]);
-        if (fgets(buf8.get(), LINENOISE_MAX_LINE, stdin) == NULL) {
-          return NULL;
-        }
-        size_t len = strlen(buf8.get());
-        while (len && (buf8[len - 1] == '\n' || buf8[len - 1] == '\r')) {
-          --len;
-          buf8[len] = '\0';
-        }
-        return strdup(buf8.get());  // caller must free buffer
-      } else {
-        char* buf8 = strdup(preloadedBufferContents.c_str());
-        preloadedBufferContents.clear();
-        return buf8;  // caller must free buffer
-      }
-    } else {
-      if (enableRawMode() == -1) {
-        return NULL;
-      }
-      InputBuffer ib(buf32, charWidths, LINENOISE_MAX_LINE);
-      if (!preloadedBufferContents.empty()) {
-        ib.preloadBuffer(preloadedBufferContents.c_str());
-        preloadedBufferContents.clear();
-      }
-      int count = ib.getInputLine(pi);
-      disableRawMode();
-      printf("\n");
-      if (count == -1) {
-        return NULL;
-      }
-      size_t bufferSize = sizeof(char32_t) * ib.length() + 1;
-      unique_ptr<char[]> buf8(new char[bufferSize]);
-      copyString32to8(buf8.get(), bufferSize, buf32);
-      return strdup(buf8.get());  // caller must free buffer
-    }
-  } else {  // input not from a terminal, we should work with piped input, i.e.
-            // redirected stdin
-    unique_ptr<char[]> buf8(new char[LINENOISE_MAX_LINE]);
-    if (fgets(buf8.get(), LINENOISE_MAX_LINE, stdin) == NULL) {
-      return NULL;
-    }
-
-    // if fgets() gave us the newline, remove it
-    int count = static_cast<int>(strlen(buf8.get()));
-    if (count > 0 && buf8[count - 1] == '\n') {
-      --count;
-      buf8[count] = '\0';
-    }
-    return strdup(buf8.get());  // caller must free buffer
-  }
-}
-
-/* Register a callback function to be called for tab-completion. */
-void linenoiseSetCompletionCallback(linenoiseCompletionCallback* fn) {
-  completionCallback = fn;
-}
-
-void linenoiseAddCompletion(linenoiseCompletions* lc, const char* str) {
-  lc->completionStrings.push_back(Utf32String(str));
-}
-
-int linenoiseHistoryAdd(const char* line) {
-  if (historyMaxLen == 0) {
-    return 0;
-  }
-  if (history == NULL) {
-    history =
-        reinterpret_cast<char8_t**>(malloc(sizeof(char8_t*) * historyMaxLen));
-    if (history == NULL) {
-      return 0;
-    }
-    memset(history, 0, (sizeof(char*) * historyMaxLen));
-  }
-  char8_t* linecopy = strdup8(line);
-  if (!linecopy) {
-    return 0;
-  }
-
-  // convert newlines in multi-line code to spaces before storing
-  char8_t* p = linecopy;
-  while (*p) {
-    if (*p == '\n') {
-      *p = ' ';
-    }
-    ++p;
-  }
-
-  // prevent duplicate history entries
-  if (historyLen > 0 && history[historyLen - 1] != nullptr &&
-      strcmp(reinterpret_cast<char const*>(history[historyLen - 1]),
-             reinterpret_cast<char const*>(linecopy)) == 0) {
-    free(linecopy);
-    return 0;
-  }
-
-  if (historyLen == historyMaxLen) {
-    free(history[0]);
-    memmove(history, history + 1, sizeof(char*) * (historyMaxLen - 1));
-    --historyLen;
-    if (--historyPreviousIndex < -1) {
-      historyPreviousIndex = -2;
-    }
-  }
-
-  history[historyLen] = linecopy;
-  ++historyLen;
-  return 1;
-}
-
-int linenoiseHistorySetMaxLen(int len) {
-  if (len < 1) {
-    return 0;
-  }
-  if (history) {
-    int tocopy = historyLen;
-    char8_t** newHistory =
-        reinterpret_cast<char8_t**>(malloc(sizeof(char8_t*) * len));
-    if (newHistory == NULL) {
-      return 0;
-    }
-    if (len < tocopy) {
-      tocopy = len;
-    }
-    memcpy(newHistory, history + historyMaxLen - tocopy,
-           sizeof(char8_t*) * tocopy);
-    free(history);
-    history = newHistory;
-  }
-  historyMaxLen = len;
-  if (historyLen > historyMaxLen) {
-    historyLen = historyMaxLen;
-  }
-  return 1;
-}
-
-/* Fetch a line of the history by (zero-based) index.  If the requested
- * line does not exist, NULL is returned.  The return value is a heap-allocated
- * copy of the line, and the caller is responsible for de-allocating it. */
-char* linenoiseHistoryLine(int index) {
-  if (index < 0 || index >= historyLen) return NULL;
-
-  return strdup(reinterpret_cast<char const*>(history[index]));
-}
-
-/* Save the history in the specified file. On success 0 is returned
- * otherwise -1 is returned. */
-int linenoiseHistorySave(const char* filename) {
-#if _WIN32
-  FILE* fp = fopen(filename, "wt");
-#else
-  int fd = open(filename, O_CREAT | O_TRUNC | O_WRONLY, S_IRUSR | S_IWUSR);
-
-  if (fd < 0) {
-    return -1;
-  }
-
-  FILE* fp = fdopen(fd, "wt");
-#endif
-
-  if (fp == NULL) {
-    return -1;
-  }
-
-  for (int j = 0; j < historyLen; ++j) {
-    if (history[j][0] != '\0') {
-      fprintf(fp, "%s\n", history[j]);
-    }
-  }
-
-  fclose(fp);
-
-  return 0;
-}
-
-/* Load the history from the specified file. If the file does not exist
- * zero is returned and no operation is performed.
- *
- * If the file exists and the operation succeeded 0 is returned, otherwise
- * on error -1 is returned. */
-int linenoiseHistoryLoad(const char* filename) {
-  FILE* fp = fopen(filename, "rt");
-  if (fp == NULL) {
-    return -1;
-  }
-
-  char buf[LINENOISE_MAX_LINE];
-  while (fgets(buf, LINENOISE_MAX_LINE, fp) != NULL) {
-    char* p = strchr(buf, '\r');
-    if (!p) {
-      p = strchr(buf, '\n');
-    }
-    if (p) {
-      *p = '\0';
-    }
-    if (p != buf) {
-      linenoiseHistoryAdd(buf);
-    }
-  }
-  fclose(fp);
-  return 0;
-}
-
-/* Set if to use or not the multi line mode. */
-/* note that this is a stub only, as linenoise-ng always multi-line */
-void linenoiseSetMultiLine(int) {}
-
-/* This special mode is used by linenoise in order to print scan codes
- * on screen for debugging / development purposes. It is implemented
- * by the linenoise_example program using the --keycodes option. */
-void linenoisePrintKeyCodes(void) {
-  char quit[4];
-
-  printf(
-      "Linenoise key codes debugging mode.\n"
-      "Press keys to see scan codes. Type 'quit' at any time to exit.\n");
-  if (enableRawMode() == -1) return;
-  memset(quit, ' ', 4);
-  while (1) {
-    char c;
-    int nread;
-
-#if _WIN32
-    nread = _read(STDIN_FILENO, &c, 1);
-#else
-    nread = read(STDIN_FILENO, &c, 1);
-#endif
-    if (nread <= 0) continue;
-    memmove(quit, quit + 1, sizeof(quit) - 1); /* shift string to left. */
-    quit[sizeof(quit) - 1] = c; /* Insert current char on the right. */
-    if (memcmp(quit, "quit", sizeof(quit)) == 0) break;
-
-    printf("'%c' %02x (%d) (type quit to exit)\n", isprint(c) ? c : '?', (int)c,
-           (int)c);
-    printf("\r"); /* Go left edge manually, we are in raw mode. */
-    fflush(stdout);
-  }
-  disableRawMode();
-}
-
-#ifndef _WIN32
-static void WindowSizeChanged(int) {
-  // do nothing here but setting this flag
-  gotResize = true;
-}
-#endif
-
-int linenoiseInstallWindowChangeHandler(void) {
-#ifndef _WIN32
-  struct sigaction sa;
-  sigemptyset(&sa.sa_mask);
-  sa.sa_flags = 0;
-  sa.sa_handler = &WindowSizeChanged;
-
-  if (sigaction(SIGWINCH, &sa, nullptr) == -1) {
-    return errno;
-  }
-#endif
-  return 0;
-}
-
-int linenoiseKeyType(void) {
-  return keyType;
-}
diff --git a/src/linenoise/linenoise.h b/src/linenoise/linenoise.h
deleted file mode 100644
index 3a8eb9f7ee63..000000000000
--- a/src/linenoise/linenoise.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/* linenoise.h -- guerrilla line editing library against the idea that a
- * line editing lib needs to be 20,000 lines of C code.
- *
- * See linenoise.c for more information.
- *
- * Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __LINENOISE_H
-#define __LINENOISE_H
-
-#define LINENOISE_VERSION "1.0.0"
-#define LINENOISE_VERSION_MAJOR 1
-#define LINENOISE_VERSION_MINOR 1
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct linenoiseCompletions linenoiseCompletions;
-
-typedef void(linenoiseCompletionCallback)(const char*, linenoiseCompletions*);
-void linenoiseSetCompletionCallback(linenoiseCompletionCallback* fn);
-void linenoiseAddCompletion(linenoiseCompletions* lc, const char* str);
-
-char* linenoise(const char* prompt);
-void linenoisePreloadBuffer(const char* preloadText);
-int linenoiseHistoryAdd(const char* line);
-int linenoiseHistorySetMaxLen(int len);
-char* linenoiseHistoryLine(int index);
-int linenoiseHistorySave(const char* filename);
-int linenoiseHistoryLoad(const char* filename);
-void linenoiseHistoryFree(void);
-void linenoiseClearScreen(void);
-void linenoiseSetMultiLine(int ml);
-void linenoisePrintKeyCodes(void);
-/* the following are extensions to the original linenoise API */
-int linenoiseInstallWindowChangeHandler(void);
-/* returns type of key pressed: 1 = CTRL-C, 2 = CTRL-D, 0 = other */ 
-int linenoiseKeyType(void);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* __LINENOISE_H */
diff --git a/src/linenoise/wcwidth.cpp b/src/linenoise/wcwidth.cpp
deleted file mode 100644
index deec0ba6b57f..000000000000
--- a/src/linenoise/wcwidth.cpp
+++ /dev/null
@@ -1,315 +0,0 @@
-/*
- * This is an implementation of wcwidth() and wcswidth() (defined in
- * IEEE Std 1002.1-2001) for Unicode.
- *
- * http://www.opengroup.org/onlinepubs/007904975/functions/wcwidth.html
- * http://www.opengroup.org/onlinepubs/007904975/functions/wcswidth.html
- *
- * In fixed-width output devices, Latin characters all occupy a single
- * "cell" position of equal width, whereas ideographic CJK characters
- * occupy two such cells. Interoperability between terminal-line
- * applications and (teletype-style) character terminals using the
- * UTF-8 encoding requires agreement on which character should advance
- * the cursor by how many cell positions. No established formal
- * standards exist at present on which Unicode character shall occupy
- * how many cell positions on character terminals. These routines are
- * a first attempt of defining such behavior based on simple rules
- * applied to data provided by the Unicode Consortium.
- *
- * For some graphical characters, the Unicode standard explicitly
- * defines a character-cell width via the definition of the East Asian
- * FullWidth (F), Wide (W), Half-width (H), and Narrow (Na) classes.
- * In all these cases, there is no ambiguity about which width a
- * terminal shall use. For characters in the East Asian Ambiguous (A)
- * class, the width choice depends purely on a preference of backward
- * compatibility with either historic CJK or Western practice.
- * Choosing single-width for these characters is easy to justify as
- * the appropriate long-term solution, as the CJK practice of
- * displaying these characters as double-width comes from historic
- * implementation simplicity (8-bit encoded characters were displayed
- * single-width and 16-bit ones double-width, even for Greek,
- * Cyrillic, etc.) and not any typographic considerations.
- *
- * Much less clear is the choice of width for the Not East Asian
- * (Neutral) class. Existing practice does not dictate a width for any
- * of these characters. It would nevertheless make sense
- * typographically to allocate two character cells to characters such
- * as for instance EM SPACE or VOLUME INTEGRAL, which cannot be
- * represented adequately with a single-width glyph. The following
- * routines at present merely assign a single-cell width to all
- * neutral characters, in the interest of simplicity. This is not
- * entirely satisfactory and should be reconsidered before
- * establishing a formal standard in this area. At the moment, the
- * decision which Not East Asian (Neutral) characters should be
- * represented by double-width glyphs cannot yet be answered by
- * applying a simple rule from the Unicode database content. Setting
- * up a proper standard for the behavior of UTF-8 character terminals
- * will require a careful analysis not only of each Unicode character,
- * but also of each presentation form, something the author of these
- * routines has avoided to do so far.
- *
- * http://www.unicode.org/unicode/reports/tr11/
- *
- * Markus Kuhn -- 2007-05-26 (Unicode 5.0)
- *
- * Permission to use, copy, modify, and distribute this software
- * for any purpose and without fee is hereby granted. The author
- * disclaims all warranties with regard to this software.
- *
- * Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
- */
-
-#include <wchar.h>
-#include <string>
-#include <memory>
-
-namespace linenoise_ng {
-
-struct interval {
-  char32_t first;
-  char32_t last;
-};
-
-/* auxiliary function for binary search in interval table */
-static int bisearch(char32_t ucs, const struct interval *table, int max) {
-  int min = 0;
-  int mid;
-
-  if (ucs < table[0].first || ucs > table[max].last)
-    return 0;
-  while (max >= min) {
-    mid = (min + max) / 2;
-    if (ucs > table[mid].last)
-      min = mid + 1;
-    else if (ucs < table[mid].first)
-      max = mid - 1;
-    else
-      return 1;
-  }
-
-  return 0;
-}
-
-
-/* The following two functions define the column width of an ISO 10646
- * character as follows:
- *
- *    - The null character (U+0000) has a column width of 0.
- *
- *    - Other C0/C1 control characters and DEL will lead to a return
- *      value of -1.
- *
- *    - Non-spacing and enclosing combining characters (general
- *      category code Mn or Me in the Unicode database) have a
- *      column width of 0.
- *
- *    - SOFT HYPHEN (U+00AD) has a column width of 1.
- *
- *    - Other format characters (general category code Cf in the Unicode
- *      database) and ZERO WIDTH SPACE (U+200B) have a column width of 0.
- *
- *    - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF)
- *      have a column width of 0.
- *
- *    - Spacing characters in the East Asian Wide (W) or East Asian
- *      Full-width (F) category as defined in Unicode Technical
- *      Report #11 have a column width of 2.
- *
- *    - All remaining characters (including all printable
- *      ISO 8859-1 and WGL4 characters, Unicode control characters,
- *      etc.) have a column width of 1.
- *
- * This implementation assumes that wchar_t characters are encoded
- * in ISO 10646.
- */
-
-int mk_wcwidth(char32_t ucs)
-{
-  /* sorted list of non-overlapping intervals of non-spacing characters */
-  /* generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" */
-  static const struct interval combining[] = {
-    { 0x0300, 0x036F }, { 0x0483, 0x0486 }, { 0x0488, 0x0489 },
-    { 0x0591, 0x05BD }, { 0x05BF, 0x05BF }, { 0x05C1, 0x05C2 },
-    { 0x05C4, 0x05C5 }, { 0x05C7, 0x05C7 }, { 0x0600, 0x0603 },
-    { 0x0610, 0x0615 }, { 0x064B, 0x065E }, { 0x0670, 0x0670 },
-    { 0x06D6, 0x06E4 }, { 0x06E7, 0x06E8 }, { 0x06EA, 0x06ED },
-    { 0x070F, 0x070F }, { 0x0711, 0x0711 }, { 0x0730, 0x074A },
-    { 0x07A6, 0x07B0 }, { 0x07EB, 0x07F3 }, { 0x0901, 0x0902 },
-    { 0x093C, 0x093C }, { 0x0941, 0x0948 }, { 0x094D, 0x094D },
-    { 0x0951, 0x0954 }, { 0x0962, 0x0963 }, { 0x0981, 0x0981 },
-    { 0x09BC, 0x09BC }, { 0x09C1, 0x09C4 }, { 0x09CD, 0x09CD },
-    { 0x09E2, 0x09E3 }, { 0x0A01, 0x0A02 }, { 0x0A3C, 0x0A3C },
-    { 0x0A41, 0x0A42 }, { 0x0A47, 0x0A48 }, { 0x0A4B, 0x0A4D },
-    { 0x0A70, 0x0A71 }, { 0x0A81, 0x0A82 }, { 0x0ABC, 0x0ABC },
-    { 0x0AC1, 0x0AC5 }, { 0x0AC7, 0x0AC8 }, { 0x0ACD, 0x0ACD },
-    { 0x0AE2, 0x0AE3 }, { 0x0B01, 0x0B01 }, { 0x0B3C, 0x0B3C },
-    { 0x0B3F, 0x0B3F }, { 0x0B41, 0x0B43 }, { 0x0B4D, 0x0B4D },
-    { 0x0B56, 0x0B56 }, { 0x0B82, 0x0B82 }, { 0x0BC0, 0x0BC0 },
-    { 0x0BCD, 0x0BCD }, { 0x0C3E, 0x0C40 }, { 0x0C46, 0x0C48 },
-    { 0x0C4A, 0x0C4D }, { 0x0C55, 0x0C56 }, { 0x0CBC, 0x0CBC },
-    { 0x0CBF, 0x0CBF }, { 0x0CC6, 0x0CC6 }, { 0x0CCC, 0x0CCD },
-    { 0x0CE2, 0x0CE3 }, { 0x0D41, 0x0D43 }, { 0x0D4D, 0x0D4D },
-    { 0x0DCA, 0x0DCA }, { 0x0DD2, 0x0DD4 }, { 0x0DD6, 0x0DD6 },
-    { 0x0E31, 0x0E31 }, { 0x0E34, 0x0E3A }, { 0x0E47, 0x0E4E },
-    { 0x0EB1, 0x0EB1 }, { 0x0EB4, 0x0EB9 }, { 0x0EBB, 0x0EBC },
-    { 0x0EC8, 0x0ECD }, { 0x0F18, 0x0F19 }, { 0x0F35, 0x0F35 },
-    { 0x0F37, 0x0F37 }, { 0x0F39, 0x0F39 }, { 0x0F71, 0x0F7E },
-    { 0x0F80, 0x0F84 }, { 0x0F86, 0x0F87 }, { 0x0F90, 0x0F97 },
-    { 0x0F99, 0x0FBC }, { 0x0FC6, 0x0FC6 }, { 0x102D, 0x1030 },
-    { 0x1032, 0x1032 }, { 0x1036, 0x1037 }, { 0x1039, 0x1039 },
-    { 0x1058, 0x1059 }, { 0x1160, 0x11FF }, { 0x135F, 0x135F },
-    { 0x1712, 0x1714 }, { 0x1732, 0x1734 }, { 0x1752, 0x1753 },
-    { 0x1772, 0x1773 }, { 0x17B4, 0x17B5 }, { 0x17B7, 0x17BD },
-    { 0x17C6, 0x17C6 }, { 0x17C9, 0x17D3 }, { 0x17DD, 0x17DD },
-    { 0x180B, 0x180D }, { 0x18A9, 0x18A9 }, { 0x1920, 0x1922 },
-    { 0x1927, 0x1928 }, { 0x1932, 0x1932 }, { 0x1939, 0x193B },
-    { 0x1A17, 0x1A18 }, { 0x1B00, 0x1B03 }, { 0x1B34, 0x1B34 },
-    { 0x1B36, 0x1B3A }, { 0x1B3C, 0x1B3C }, { 0x1B42, 0x1B42 },
-    { 0x1B6B, 0x1B73 }, { 0x1DC0, 0x1DCA }, { 0x1DFE, 0x1DFF },
-    { 0x200B, 0x200F }, { 0x202A, 0x202E }, { 0x2060, 0x2063 },
-    { 0x206A, 0x206F }, { 0x20D0, 0x20EF }, { 0x302A, 0x302F },
-    { 0x3099, 0x309A }, { 0xA806, 0xA806 }, { 0xA80B, 0xA80B },
-    { 0xA825, 0xA826 }, { 0xFB1E, 0xFB1E }, { 0xFE00, 0xFE0F },
-    { 0xFE20, 0xFE23 }, { 0xFEFF, 0xFEFF }, { 0xFFF9, 0xFFFB },
-    { 0x10A01, 0x10A03 }, { 0x10A05, 0x10A06 }, { 0x10A0C, 0x10A0F },
-    { 0x10A38, 0x10A3A }, { 0x10A3F, 0x10A3F }, { 0x1D167, 0x1D169 },
-    { 0x1D173, 0x1D182 }, { 0x1D185, 0x1D18B }, { 0x1D1AA, 0x1D1AD },
-    { 0x1D242, 0x1D244 }, { 0xE0001, 0xE0001 }, { 0xE0020, 0xE007F },
-    { 0xE0100, 0xE01EF }
-  };
-
-  /* test for 8-bit control characters */
-  if (ucs == 0)
-    return 0;
-  if (ucs < 32 || (ucs >= 0x7f && ucs < 0xa0))
-    return -1;
-
-  /* binary search in table of non-spacing characters */
-  if (bisearch(ucs, combining,
-               sizeof(combining) / sizeof(struct interval) - 1))
-    return 0;
-
-  /* if we arrive here, ucs is not a combining or C0/C1 control character */
-
-  return 1 + 
-    (ucs >= 0x1100 &&
-     (ucs <= 0x115f ||                    /* Hangul Jamo init. consonants */
-      ucs == 0x2329 || ucs == 0x232a ||
-      (ucs >= 0x2e80 && ucs <= 0xa4cf &&
-       ucs != 0x303f) ||                  /* CJK ... Yi */
-      (ucs >= 0xac00 && ucs <= 0xd7a3) || /* Hangul Syllables */
-      (ucs >= 0xf900 && ucs <= 0xfaff) || /* CJK Compatibility Ideographs */
-      (ucs >= 0xfe10 && ucs <= 0xfe19) || /* Vertical forms */
-      (ucs >= 0xfe30 && ucs <= 0xfe6f) || /* CJK Compatibility Forms */
-      (ucs >= 0xff00 && ucs <= 0xff60) || /* Fullwidth Forms */
-      (ucs >= 0xffe0 && ucs <= 0xffe6) ||
-      (ucs >= 0x20000 && ucs <= 0x2fffd) ||
-      (ucs >= 0x30000 && ucs <= 0x3fffd)));
-}
-
-
-int mk_wcswidth(const char32_t* pwcs, size_t n)
-{
-  int w, width = 0;
-
-  for (;*pwcs && n-- > 0; pwcs++)
-    if ((w = mk_wcwidth(*pwcs)) < 0)
-      return -1;
-    else
-      width += w;
-
-  return width;
-}
-
-
-/*
- * The following functions are the same as mk_wcwidth() and
- * mk_wcswidth(), except that spacing characters in the East Asian
- * Ambiguous (A) category as defined in Unicode Technical Report #11
- * have a column width of 2. This variant might be useful for users of
- * CJK legacy encodings who want to migrate to UCS without changing
- * the traditional terminal character-width behaviour. It is not
- * otherwise recommended for general use.
- */
-int mk_wcwidth_cjk(wchar_t ucs)
-{
-  /* sorted list of non-overlapping intervals of East Asian Ambiguous
-   * characters, generated by "uniset +WIDTH-A -cat=Me -cat=Mn -cat=Cf c" */
-  static const struct interval ambiguous[] = {
-    { 0x00A1, 0x00A1 }, { 0x00A4, 0x00A4 }, { 0x00A7, 0x00A8 },
-    { 0x00AA, 0x00AA }, { 0x00AE, 0x00AE }, { 0x00B0, 0x00B4 },
-    { 0x00B6, 0x00BA }, { 0x00BC, 0x00BF }, { 0x00C6, 0x00C6 },
-    { 0x00D0, 0x00D0 }, { 0x00D7, 0x00D8 }, { 0x00DE, 0x00E1 },
-    { 0x00E6, 0x00E6 }, { 0x00E8, 0x00EA }, { 0x00EC, 0x00ED },
-    { 0x00F0, 0x00F0 }, { 0x00F2, 0x00F3 }, { 0x00F7, 0x00FA },
-    { 0x00FC, 0x00FC }, { 0x00FE, 0x00FE }, { 0x0101, 0x0101 },
-    { 0x0111, 0x0111 }, { 0x0113, 0x0113 }, { 0x011B, 0x011B },
-    { 0x0126, 0x0127 }, { 0x012B, 0x012B }, { 0x0131, 0x0133 },
-    { 0x0138, 0x0138 }, { 0x013F, 0x0142 }, { 0x0144, 0x0144 },
-    { 0x0148, 0x014B }, { 0x014D, 0x014D }, { 0x0152, 0x0153 },
-    { 0x0166, 0x0167 }, { 0x016B, 0x016B }, { 0x01CE, 0x01CE },
-    { 0x01D0, 0x01D0 }, { 0x01D2, 0x01D2 }, { 0x01D4, 0x01D4 },
-    { 0x01D6, 0x01D6 }, { 0x01D8, 0x01D8 }, { 0x01DA, 0x01DA },
-    { 0x01DC, 0x01DC }, { 0x0251, 0x0251 }, { 0x0261, 0x0261 },
-    { 0x02C4, 0x02C4 }, { 0x02C7, 0x02C7 }, { 0x02C9, 0x02CB },
-    { 0x02CD, 0x02CD }, { 0x02D0, 0x02D0 }, { 0x02D8, 0x02DB },
-    { 0x02DD, 0x02DD }, { 0x02DF, 0x02DF }, { 0x0391, 0x03A1 },
-    { 0x03A3, 0x03A9 }, { 0x03B1, 0x03C1 }, { 0x03C3, 0x03C9 },
-    { 0x0401, 0x0401 }, { 0x0410, 0x044F }, { 0x0451, 0x0451 },
-    { 0x2010, 0x2010 }, { 0x2013, 0x2016 }, { 0x2018, 0x2019 },
-    { 0x201C, 0x201D }, { 0x2020, 0x2022 }, { 0x2024, 0x2027 },
-    { 0x2030, 0x2030 }, { 0x2032, 0x2033 }, { 0x2035, 0x2035 },
-    { 0x203B, 0x203B }, { 0x203E, 0x203E }, { 0x2074, 0x2074 },
-    { 0x207F, 0x207F }, { 0x2081, 0x2084 }, { 0x20AC, 0x20AC },
-    { 0x2103, 0x2103 }, { 0x2105, 0x2105 }, { 0x2109, 0x2109 },
-    { 0x2113, 0x2113 }, { 0x2116, 0x2116 }, { 0x2121, 0x2122 },
-    { 0x2126, 0x2126 }, { 0x212B, 0x212B }, { 0x2153, 0x2154 },
-    { 0x215B, 0x215E }, { 0x2160, 0x216B }, { 0x2170, 0x2179 },
-    { 0x2190, 0x2199 }, { 0x21B8, 0x21B9 }, { 0x21D2, 0x21D2 },
-    { 0x21D4, 0x21D4 }, { 0x21E7, 0x21E7 }, { 0x2200, 0x2200 },
-    { 0x2202, 0x2203 }, { 0x2207, 0x2208 }, { 0x220B, 0x220B },
-    { 0x220F, 0x220F }, { 0x2211, 0x2211 }, { 0x2215, 0x2215 },
-    { 0x221A, 0x221A }, { 0x221D, 0x2220 }, { 0x2223, 0x2223 },
-    { 0x2225, 0x2225 }, { 0x2227, 0x222C }, { 0x222E, 0x222E },
-    { 0x2234, 0x2237 }, { 0x223C, 0x223D }, { 0x2248, 0x2248 },
-    { 0x224C, 0x224C }, { 0x2252, 0x2252 }, { 0x2260, 0x2261 },
-    { 0x2264, 0x2267 }, { 0x226A, 0x226B }, { 0x226E, 0x226F },
-    { 0x2282, 0x2283 }, { 0x2286, 0x2287 }, { 0x2295, 0x2295 },
-    { 0x2299, 0x2299 }, { 0x22A5, 0x22A5 }, { 0x22BF, 0x22BF },
-    { 0x2312, 0x2312 }, { 0x2460, 0x24E9 }, { 0x24EB, 0x254B },
-    { 0x2550, 0x2573 }, { 0x2580, 0x258F }, { 0x2592, 0x2595 },
-    { 0x25A0, 0x25A1 }, { 0x25A3, 0x25A9 }, { 0x25B2, 0x25B3 },
-    { 0x25B6, 0x25B7 }, { 0x25BC, 0x25BD }, { 0x25C0, 0x25C1 },
-    { 0x25C6, 0x25C8 }, { 0x25CB, 0x25CB }, { 0x25CE, 0x25D1 },
-    { 0x25E2, 0x25E5 }, { 0x25EF, 0x25EF }, { 0x2605, 0x2606 },
-    { 0x2609, 0x2609 }, { 0x260E, 0x260F }, { 0x2614, 0x2615 },
-    { 0x261C, 0x261C }, { 0x261E, 0x261E }, { 0x2640, 0x2640 },
-    { 0x2642, 0x2642 }, { 0x2660, 0x2661 }, { 0x2663, 0x2665 },
-    { 0x2667, 0x266A }, { 0x266C, 0x266D }, { 0x266F, 0x266F },
-    { 0x273D, 0x273D }, { 0x2776, 0x277F }, { 0xE000, 0xF8FF },
-    { 0xFFFD, 0xFFFD }, { 0xF0000, 0xFFFFD }, { 0x100000, 0x10FFFD }
-  };
-
-  /* binary search in table of non-spacing characters */
-  if (bisearch(ucs, ambiguous,
-               sizeof(ambiguous) / sizeof(struct interval) - 1))
-    return 2;
-
-  return mk_wcwidth(ucs);
-}
-
-
-int mk_wcswidth_cjk(const wchar_t *pwcs, size_t n)
-{
-  int w, width = 0;
-
-  for (;*pwcs && n-- > 0; pwcs++)
-    if ((w = mk_wcwidth_cjk(*pwcs)) < 0)
-      return -1;
-    else
-      width += w;
-
-  return width;
-}
-
-}
diff --git a/src/nix-build/local.mk b/src/nix-build/local.mk
deleted file mode 100644
index a2d1c91dfd9d..000000000000
--- a/src/nix-build/local.mk
+++ /dev/null
@@ -1,9 +0,0 @@
-programs += nix-build
-
-nix-build_DIR := $(d)
-
-nix-build_SOURCES := $(d)/nix-build.cc
-
-nix-build_LIBS = libmain libexpr libstore libutil libformat
-
-$(eval $(call install-symlink, nix-build, $(bindir)/nix-shell))
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index a63b3e07ae77..c6a4d416648f 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -16,6 +16,7 @@
 #include "get-drvs.hh"
 #include "common-eval-args.hh"
 #include "attr-path.hh"
+#include "legacy.hh"
 
 using namespace nix;
 using namespace std::string_literals;
@@ -66,11 +67,8 @@ std::vector<string> shellwords(const string & s)
     return res;
 }
 
-void mainWrapped(int argc, char * * argv)
+static void _main(int argc, char * * argv)
 {
-    initNix();
-    initGC();
-
     auto dryRun = false;
     auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$"));
     auto pure = false;
@@ -85,7 +83,6 @@ void mainWrapped(int argc, char * * argv)
     BuildMode buildMode = bmNormal;
     bool readStdin = false;
 
-    auto shell = getEnv("SHELL", "/bin/sh");
     std::string envCommand; // interactive shell
     Strings envExclude;
 
@@ -99,6 +96,9 @@ void mainWrapped(int argc, char * * argv)
 
     std::string outLink = "./result";
 
+    // List of environment variables kept for --pure
+    std::set<string> keepVars{"HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL"};
+
     Strings args;
     for (int i = 1; i < argc; ++i)
         args.push_back(argv[i]);
@@ -218,6 +218,9 @@ void mainWrapped(int argc, char * * argv)
             }
         }
 
+        else if (*arg == "--keep")
+            keepVars.insert(getArg(*arg, arg, end));
+
         else if (*arg == "-")
             readStdin = true;
 
@@ -239,10 +242,10 @@ void mainWrapped(int argc, char * * argv)
 
     auto store = openStore();
 
-    EvalState state(myArgs.searchPath, store);
-    state.repair = repair;
+    auto state = std::make_unique<EvalState>(myArgs.searchPath, store);
+    state->repair = repair;
 
-    Bindings & autoArgs = *myArgs.getAutoArgs(state);
+    Bindings & autoArgs = *myArgs.getAutoArgs(*state);
 
     if (packages) {
         std::ostringstream joined;
@@ -268,22 +271,24 @@ void mainWrapped(int argc, char * * argv)
     std::vector<Expr *> exprs;
 
     if (readStdin)
-        exprs = {state.parseStdin()};
+        exprs = {state->parseStdin()};
     else
         for (auto i : left) {
-            auto absolute = i;
-            try {
-                absolute = canonPath(absPath(i), true);
-            } catch (Error e) {};
             if (fromArgs)
-                exprs.push_back(state.parseExprFromString(i, absPath(".")));
-            else if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?")))
-                drvs.push_back(DrvInfo(state, store, absolute));
+                exprs.push_back(state->parseExprFromString(i, absPath(".")));
+            else {
+                auto absolute = i;
+                try {
+                    absolute = canonPath(absPath(i), true);
+                } catch (Error e) {};
+                if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?")))
+                drvs.push_back(DrvInfo(*state, store, absolute));
             else
                 /* If we're in a #! script, interpret filenames
                    relative to the script. */
-                exprs.push_back(state.parseExprFromFile(resolveExprPath(state.checkSourcePath(lookupFileArg(state,
+                exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state,
                     inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i)))));
+            }
         }
 
     /* Evaluate them into derivations. */
@@ -291,15 +296,17 @@ void mainWrapped(int argc, char * * argv)
 
     for (auto e : exprs) {
         Value vRoot;
-        state.eval(e, vRoot);
+        state->eval(e, vRoot);
 
         for (auto & i : attrPaths) {
-            Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot));
-            state.forceValue(v);
-            getDerivations(state, v, "", autoArgs, drvs, false);
+            Value & v(*findAlongAttrPath(*state, i, autoArgs, vRoot));
+            state->forceValue(v);
+            getDerivations(*state, v, "", autoArgs, drvs, false);
         }
     }
 
+    state->printStats();
+
     auto buildPaths = [&](const PathSet & paths) {
         /* Note: we do this even when !printMissing to efficiently
            fetch binary cache data. */
@@ -332,12 +339,12 @@ void mainWrapped(int argc, char * * argv)
         if (shell == "") {
 
             try {
-                auto expr = state.parseExprFromString("(import <nixpkgs> {}).bashInteractive", absPath("."));
+                auto expr = state->parseExprFromString("(import <nixpkgs> {}).bashInteractive", absPath("."));
 
                 Value v;
-                state.eval(expr, v);
+                state->eval(expr, v);
 
-                auto drv = getDerivation(state, v, false);
+                auto drv = getDerivation(*state, v, false);
                 if (!drv)
                     throw Error("the 'bashInteractive' attribute in <nixpkgs> did not evaluate to a derivation");
 
@@ -354,7 +361,7 @@ void mainWrapped(int argc, char * * argv)
         // Build or fetch all dependencies of the derivation.
         for (const auto & input : drv.inputDrvs)
             if (std::all_of(envExclude.cbegin(), envExclude.cend(), [&](const string & exclude) { return !std::regex_search(input.first, std::regex(exclude)); }))
-                pathsToBuild.insert(input.first);
+                pathsToBuild.insert(makeDrvPathWithOutputs(input.first, input.second));
         for (const auto & src : drv.inputSrcs)
             pathsToBuild.insert(src);
 
@@ -368,7 +375,6 @@ void mainWrapped(int argc, char * * argv)
         auto tmp = getEnv("TMPDIR", getEnv("XDG_RUNTIME_DIR", "/tmp"));
 
         if (pure) {
-            std::set<string> keepVars{"HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL"};
             decltype(env) newEnv;
             for (auto & i : env)
                 if (keepVars.count(i.first))
@@ -411,17 +417,20 @@ void mainWrapped(int argc, char * * argv)
                 "dontAddDisableDepTrack=1; "
                 "[ -e $stdenv/setup ] && source $stdenv/setup; "
                 "%3%"
+                "PATH=\"%4%:$PATH\"; "
+                "SHELL=%5%; "
                 "set +e; "
                 R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s"
                 "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; "
                 "unset NIX_ENFORCE_PURITY; "
-                "unset NIX_INDENT_MAKE; "
                 "shopt -u nullglob; "
-                "unset TZ; %4%"
-                "%5%",
+                "unset TZ; %6%"
+                "%7%",
                 (Path) tmpDir,
                 (pure ? "" : "p=$PATH; "),
                 (pure ? "" : "PATH=$PATH:$p; unset p; "),
+                dirOf(shell),
+                shell,
                 (getenv("TZ") ? (string("export TZ='") + getenv("TZ") + "'; ") : ""),
                 envCommand));
 
@@ -495,9 +504,5 @@ void mainWrapped(int argc, char * * argv)
     }
 }
 
-int main(int argc, char * * argv)
-{
-    return handleExceptions(argv[0], [&]() {
-        return mainWrapped(argc, argv);
-    });
-}
+static RegisterLegacyCommand s1("nix-build", _main);
+static RegisterLegacyCommand s2("nix-shell", _main);
diff --git a/src/nix-channel/local.mk b/src/nix-channel/local.mk
deleted file mode 100644
index 49fc105c6f79..000000000000
--- a/src/nix-channel/local.mk
+++ /dev/null
@@ -1,7 +0,0 @@
-programs += nix-channel
-
-nix-channel_DIR := $(d)
-
-nix-channel_LIBS = libmain libutil libformat libstore
-
-nix-channel_SOURCES := $(d)/nix-channel.cc
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index d1b47ede803f..8b66cc7e314e 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -1,9 +1,11 @@
 #include "shared.hh"
 #include "globals.hh"
 #include "download.hh"
+#include "store-api.hh"
+#include "legacy.hh"
+
 #include <fcntl.h>
 #include <regex>
-#include "store-api.hh"
 #include <pwd.h>
 
 using namespace nix;
@@ -86,7 +88,7 @@ static void update(const StringSet & channelNames)
         // definition from a consistent location if the redirect changes mid-download.
         std::string effectiveUrl;
         auto dl = getDownloader();
-        auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl);
+        auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0);
         url = chomp(std::move(effectiveUrl));
 
         // If the URL contains a version number, append it to the name
@@ -157,11 +159,9 @@ static void update(const StringSet & channelNames)
     replaceSymlink(profile, channelLink);
 }
 
-int main(int argc, char ** argv)
+static int _main(int argc, char ** argv)
 {
-    return handleExceptions(argv[0], [&]() {
-        initNix();
-
+    {
         // Figure out the name of the `.nix-channels' file to use
         auto home = getHome();
         channelsList = home + "/.nix-channels";
@@ -169,7 +169,7 @@ int main(int argc, char ** argv)
 
         // Figure out the name of the channels profile.
         ;
-        auto pw = getpwuid(getuid());
+        auto pw = getpwuid(geteuid());
         std::string name = pw ? pw->pw_name : getEnv("USER", "");
         if (name.empty())
             throw Error("cannot figure out user name");
@@ -255,5 +255,9 @@ int main(int argc, char ** argv)
                 runProgram(settings.nixBinDir + "/nix-env", false, envArgs);
                 break;
         }
-    });
+
+        return 0;
+    }
 }
+
+static RegisterLegacyCommand s1("nix-channel", _main);
diff --git a/src/nix-collect-garbage/local.mk b/src/nix-collect-garbage/local.mk
deleted file mode 100644
index 02d14cf62199..000000000000
--- a/src/nix-collect-garbage/local.mk
+++ /dev/null
@@ -1,7 +0,0 @@
-programs += nix-collect-garbage
-
-nix-collect-garbage_DIR := $(d)
-
-nix-collect-garbage_SOURCES := $(d)/nix-collect-garbage.cc
-
-nix-collect-garbage_LIBS = libmain libstore libutil libformat
diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc
index 37fe22f48134..d4060ac937fc 100644
--- a/src/nix-collect-garbage/nix-collect-garbage.cc
+++ b/src/nix-collect-garbage/nix-collect-garbage.cc
@@ -2,6 +2,7 @@
 #include "profiles.hh"
 #include "shared.hh"
 #include "globals.hh"
+#include "legacy.hh"
 
 #include <iostream>
 #include <cerrno>
@@ -48,12 +49,10 @@ void removeOldGenerations(std::string dir)
     }
 }
 
-int main(int argc, char * * argv)
+static int _main(int argc, char * * argv)
 {
-    bool removeOld = false;
-
-    return handleExceptions(argv[0], [&]() {
-        initNix();
+    {
+        bool removeOld = false;
 
         GCOptions options;
 
@@ -90,5 +89,9 @@ int main(int argc, char * * argv)
             PrintFreed freed(true, results);
             store->collectGarbage(options, results);
         }
-    });
+
+        return 0;
+    }
 }
+
+static RegisterLegacyCommand s1("nix-collect-garbage", _main);
diff --git a/src/nix-copy-closure/local.mk b/src/nix-copy-closure/local.mk
deleted file mode 100644
index 42bb34dd8201..000000000000
--- a/src/nix-copy-closure/local.mk
+++ /dev/null
@@ -1,7 +0,0 @@
-programs += nix-copy-closure
-
-nix-copy-closure_DIR := $(d)
-
-nix-copy-closure_LIBS = libmain libutil libformat libstore
-
-nix-copy-closure_SOURCES := $(d)/nix-copy-closure.cc
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
index dfb1b8fc5dc4..fdcde8b076b5 100755
--- a/src/nix-copy-closure/nix-copy-closure.cc
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -1,13 +1,12 @@
 #include "shared.hh"
 #include "store-api.hh"
+#include "legacy.hh"
 
 using namespace nix;
 
-int main(int argc, char ** argv)
+static int _main(int argc, char ** argv)
 {
-    return handleExceptions(argv[0], [&]() {
-        initNix();
-
+    {
         auto gzip = false;
         auto toMode = true;
         auto includeOutputs = false;
@@ -61,5 +60,9 @@ int main(int argc, char ** argv)
         from->computeFSClosure(storePaths2, closure, false, includeOutputs);
 
         copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes);
-    });
+
+        return 0;
+    }
 }
+
+static RegisterLegacyCommand s1("nix-copy-closure", _main);
diff --git a/src/nix-daemon/local.mk b/src/nix-daemon/local.mk
deleted file mode 100644
index 5a4474465b3c..000000000000
--- a/src/nix-daemon/local.mk
+++ /dev/null
@@ -1,13 +0,0 @@
-programs += nix-daemon
-
-nix-daemon_DIR := $(d)
-
-nix-daemon_SOURCES := $(d)/nix-daemon.cc
-
-nix-daemon_LIBS = libmain libstore libutil libformat
-
-nix-daemon_LDFLAGS = -pthread
-
-ifeq ($(OS), SunOS)
-        nix-daemon_LDFLAGS += -lsocket
-endif
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 35603af7082a..8d63b8f362ec 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -9,6 +9,7 @@
 #include "monitor-fd.hh"
 #include "derivations.hh"
 #include "finally.hh"
+#include "legacy.hh"
 
 #include <algorithm>
 
@@ -120,8 +121,6 @@ struct TunnelLogger : public Logger
       want to send out stderr to the client. */
     void startWork()
     {
-        std::vector<std::string> pendingMsgs;
-
         auto state(state_.lock());
         state->canSendStderr = true;
 
@@ -197,7 +196,8 @@ struct TunnelSource : BufferedSource
 {
     Source & from;
     TunnelSource(Source & from) : from(from) { }
-    size_t readUnbuffered(unsigned char * data, size_t len)
+protected:
+    size_t readUnbuffered(unsigned char * data, size_t len) override
     {
         to << STDERR_READ << len;
         to.flush();
@@ -234,7 +234,7 @@ struct RetrieveRegularNARSink : ParseSink
 };
 
 
-static void performOp(TunnelLogger * logger, ref<LocalStore> store,
+static void performOp(TunnelLogger * logger, ref<Store> store,
     bool trusted, unsigned int clientVersion,
     Source & from, Sink & to, unsigned int op)
 {
@@ -363,7 +363,11 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store,
 
         logger->startWork();
         if (!savedRegular.regular) throw Error("regular file expected");
-        Path path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
+
+        auto store2 = store.dynamic_pointer_cast<LocalStore>();
+        if (!store2) throw Error("operation is only supported by LocalStore");
+
+        Path path = store2->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
         logger->stopWork();
 
         to << path;
@@ -471,11 +475,19 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store,
 
     case wopFindRoots: {
         logger->startWork();
-        Roots roots = store->findRoots();
+        Roots roots = store->findRoots(!trusted);
         logger->stopWork();
-        to << roots.size();
+
+        size_t size = 0;
         for (auto & i : roots)
-            to << i.first << i.second;
+            size += i.second.size();
+
+        to << size;
+
+        for (auto & [target, links] : roots)
+            for (auto & link : links)
+                to << link << target;
+
         break;
     }
 
@@ -554,7 +566,8 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store,
                     ;
                 else if (trusted
                     || name == settings.buildTimeout.name
-                    || name == settings.connectTimeout.name)
+                    || name == "connect-timeout"
+                    || (name == "builders" && value == ""))
                     settings.set(name, value);
                 else if (setSubstituters(settings.substituters))
                     ;
@@ -691,12 +704,23 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store,
         if (!trusted)
             info.ultimate = false;
 
-        TeeSink tee(from);
-        parseDump(tee, tee.source);
+        std::string saved;
+        std::unique_ptr<Source> source;
+        if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
+            source = std::make_unique<TunnelSource>(from);
+        else {
+            TeeSink tee(from);
+            parseDump(tee, tee.source);
+            saved = std::move(*tee.source.data);
+            source = std::make_unique<StringSource>(saved);
+        }
 
         logger->startWork();
-        store.cast<Store>()->addToStore(info, tee.source.data, (RepairFlag) repair,
+
+        // FIXME: race if addToStore doesn't read source?
+        store->addToStore(info, *source, (RepairFlag) repair,
             dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
+
         logger->stopWork();
         break;
     }
@@ -767,7 +791,7 @@ static void processConnection(bool trusted)
         Store::Params params; // FIXME: get params from somewhere
         // Disable caching since the client already does that.
         params["path-info-cache-size"] = "0";
-        auto store = make_ref<LocalStore>(params);
+        auto store = openStore(settings.storeUri, params);
 
         tunnelLogger->stopWork();
         to.flush();
@@ -1043,11 +1067,9 @@ static void daemonLoop(char * * argv)
 }
 
 
-int main(int argc, char * * argv)
+static int _main(int argc, char * * argv)
 {
-    return handleExceptions(argv[0], [&]() {
-        initNix();
-
+    {
         auto stdio = false;
 
         parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
@@ -1107,7 +1129,7 @@ int main(int argc, char * * argv)
                         if (res == -1)
                             throw SysError("splicing data from stdin to daemon socket");
                         else if (res == 0)
-                            return;
+                            return 0;
                     }
                 }
             } else {
@@ -1116,5 +1138,9 @@ int main(int argc, char * * argv)
         } else {
             daemonLoop(argv);
         }
-    });
+
+        return 0;
+    }
 }
+
+static RegisterLegacyCommand s1("nix-daemon", _main);
diff --git a/src/nix-env/local.mk b/src/nix-env/local.mk
deleted file mode 100644
index e80719cd76f7..000000000000
--- a/src/nix-env/local.mk
+++ /dev/null
@@ -1,7 +0,0 @@
-programs += nix-env
-
-nix-env_DIR := $(d)
-
-nix-env_SOURCES := $(wildcard $(d)/*.cc)
-
-nix-env_LIBS = libexpr libmain libstore libutil libformat
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 97e66cbd937e..56ed75daee44 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -13,6 +13,7 @@
 #include "json.hh"
 #include "value-to-json.hh"
 #include "xml-writer.hh"
+#include "legacy.hh"
 
 #include <cerrno>
 #include <ctime>
@@ -150,10 +151,8 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v)
     if (stat(path.c_str(), &st) == -1)
         throw SysError(format("getting information about '%1%'") % path);
 
-    if (isNixExpr(path, st)) {
+    if (isNixExpr(path, st))
         state.evalFile(path, v);
-        return;
-    }
 
     /* The path is a directory.  Put the Nix expressions in the
        directory in a set, with the file name of each expression as
@@ -161,13 +160,15 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v)
        set flat, not nested, to make it easier for a user to have a
        ~/.nix-defexpr directory that includes some system-wide
        directory). */
-    if (S_ISDIR(st.st_mode)) {
+    else if (S_ISDIR(st.st_mode)) {
         state.mkAttrs(v, 1024);
         state.mkList(*state.allocAttr(v, state.symbols.create("_combineChannels")), 0);
         StringSet attrs;
         getAllExprs(state, path, attrs, v);
         v.attrs->sort();
     }
+
+    else throw Error("path '%s' is not a directory or a Nix expression", path);
 }
 
 
@@ -198,13 +199,13 @@ static Path getDefNixExprPath()
 }
 
 
-static int getPriority(EvalState & state, DrvInfo & drv)
+static long getPriority(EvalState & state, DrvInfo & drv)
 {
     return drv.queryMetaInt("priority", 0);
 }
 
 
-static int comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2)
+static long comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2)
 {
     return getPriority(state, drv2) - getPriority(state, drv1);
 }
@@ -270,7 +271,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
 
             for (auto & j : matches) {
                 DrvName drvName(j.first.queryName());
-                int d = 1;
+                long d = 1;
 
                 Newest::iterator k = newest.find(drvName.name);
 
@@ -578,7 +579,7 @@ static void upgradeDerivations(Globals & globals,
                             (upgradeType == utEq && d == 0) ||
                             upgradeType == utAlways)
                         {
-                            int d2 = -1;
+                            long d2 = -1;
                             if (bestElem != availElems.end()) {
                                 d2 = comparePriorities(*globals.state, *bestElem, *j);
                                 if (d2 == 0) d2 = compareVersions(bestVersion, newName.version);
@@ -784,22 +785,22 @@ typedef list<Strings> Table;
 
 void printTable(Table & table)
 {
-    unsigned int nrColumns = table.size() > 0 ? table.front().size() : 0;
+    auto nrColumns = table.size() > 0 ? table.front().size() : 0;
 
-    vector<unsigned int> widths;
+    vector<size_t> widths;
     widths.resize(nrColumns);
 
     for (auto & i : table) {
         assert(i.size() == nrColumns);
         Strings::iterator j;
-        unsigned int column;
+        size_t column;
         for (j = i.begin(), column = 0; j != i.end(); ++j, ++column)
             if (j->size() > widths[column]) widths[column] = j->size();
     }
 
     for (auto & i : table) {
         Strings::iterator j;
-        unsigned int column;
+        size_t column;
         for (j = i.begin(), column = 0; j != i.end(); ++j, ++column) {
             string s = *j;
             replace(s.begin(), s.end(), '\n', ' ');
@@ -1284,6 +1285,14 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr
         deleteOldGenerations(globals.profile, globals.dryRun);
     } else if (opArgs.size() == 1 && opArgs.front().find('d') != string::npos) {
         deleteGenerationsOlderThan(globals.profile, opArgs.front(), globals.dryRun);
+    } else if (opArgs.size() == 1 && opArgs.front().find('+') != string::npos) {
+        if(opArgs.front().size() < 2)
+            throw Error(format("invalid number of generations โ€˜%1%โ€™") % opArgs.front());
+        string str_max = string(opArgs.front(), 1, opArgs.front().size());
+        int max;
+        if (!string2Int(str_max, max) || max == 0)
+            throw Error(format("invalid number of generations to keep โ€˜%1%โ€™") % opArgs.front());
+        deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun);
     } else {
         std::set<unsigned int> gens;
         for (auto & i : opArgs) {
@@ -1303,12 +1312,9 @@ static void opVersion(Globals & globals, Strings opFlags, Strings opArgs)
 }
 
 
-int main(int argc, char * * argv)
+static int _main(int argc, char * * argv)
 {
-    return handleExceptions(argv[0], [&]() {
-        initNix();
-        initGC();
-
+    {
         Strings opFlags, opArgs;
         Operation op = 0;
         RepairFlag repair = NoRepair;
@@ -1420,5 +1426,9 @@ int main(int argc, char * * argv)
         op(globals, opFlags, opArgs);
 
         globals.state->printStats();
-    });
+
+        return 0;
+    }
 }
+
+static RegisterLegacyCommand s1("nix-env", _main);
diff --git a/src/nix-instantiate/local.mk b/src/nix-instantiate/local.mk
deleted file mode 100644
index 7d1bc5ec9dfb..000000000000
--- a/src/nix-instantiate/local.mk
+++ /dev/null
@@ -1,7 +0,0 @@
-programs += nix-instantiate
-
-nix-instantiate_DIR := $(d)
-
-nix-instantiate_SOURCES := $(d)/nix-instantiate.cc
-
-nix-instantiate_LIBS = libexpr libmain libstore libutil libformat
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index 5049460c7544..a736caa8f056 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -9,6 +9,7 @@
 #include "util.hh"
 #include "store-api.hh"
 #include "common-eval-args.hh"
+#include "legacy.hh"
 
 #include <map>
 #include <iostream>
@@ -83,12 +84,9 @@ void processExpr(EvalState & state, const Strings & attrPaths,
 }
 
 
-int main(int argc, char * * argv)
+static int _main(int argc, char * * argv)
 {
-    return handleExceptions(argv[0], [&]() {
-        initNix();
-        initGC();
-
+    {
         Strings files;
         bool readStdin = false;
         bool fromArgs = false;
@@ -158,37 +156,41 @@ int main(int argc, char * * argv)
 
         auto store = openStore();
 
-        EvalState state(myArgs.searchPath, store);
-        state.repair = repair;
+        auto state = std::make_unique<EvalState>(myArgs.searchPath, store);
+        state->repair = repair;
 
-        Bindings & autoArgs = *myArgs.getAutoArgs(state);
+        Bindings & autoArgs = *myArgs.getAutoArgs(*state);
 
         if (attrPaths.empty()) attrPaths = {""};
 
         if (findFile) {
             for (auto & i : files) {
-                Path p = state.findFile(i);
+                Path p = state->findFile(i);
                 if (p == "") throw Error(format("unable to find '%1%'") % i);
                 std::cout << p << std::endl;
             }
-            return;
+            return 0;
         }
 
         if (readStdin) {
-            Expr * e = state.parseStdin();
-            processExpr(state, attrPaths, parseOnly, strict, autoArgs,
+            Expr * e = state->parseStdin();
+            processExpr(*state, attrPaths, parseOnly, strict, autoArgs,
                 evalOnly, outputKind, xmlOutputSourceLocation, e);
         } else if (files.empty() && !fromArgs)
             files.push_back("./default.nix");
 
         for (auto & i : files) {
             Expr * e = fromArgs
-                ? state.parseExprFromString(i, absPath("."))
-                : state.parseExprFromFile(resolveExprPath(state.checkSourcePath(lookupFileArg(state, i))));
-            processExpr(state, attrPaths, parseOnly, strict, autoArgs,
+                ? state->parseExprFromString(i, absPath("."))
+                : state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, i))));
+            processExpr(*state, attrPaths, parseOnly, strict, autoArgs,
                 evalOnly, outputKind, xmlOutputSourceLocation, e);
         }
 
-        state.printStats();
-    });
+        state->printStats();
+
+        return 0;
+    }
 }
+
+static RegisterLegacyCommand s1("nix-instantiate", _main);
diff --git a/src/nix-prefetch-url/local.mk b/src/nix-prefetch-url/local.mk
deleted file mode 100644
index 3e7735406af0..000000000000
--- a/src/nix-prefetch-url/local.mk
+++ /dev/null
@@ -1,7 +0,0 @@
-programs += nix-prefetch-url
-
-nix-prefetch-url_DIR := $(d)
-
-nix-prefetch-url_SOURCES := $(d)/nix-prefetch-url.cc
-
-nix-prefetch-url_LIBS = libmain libexpr libstore libutil libformat
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index fa7ee254500c..f54706a8a011 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -6,9 +6,16 @@
 #include "eval-inline.hh"
 #include "common-eval-args.hh"
 #include "attr-path.hh"
+#include "legacy.hh"
+#include "finally.hh"
+#include "progress-bar.hh"
 
 #include <iostream>
 
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
 using namespace nix;
 
 
@@ -40,12 +47,9 @@ string resolveMirrorUri(EvalState & state, string uri)
 }
 
 
-int main(int argc, char * * argv)
+static int _main(int argc, char * * argv)
 {
-    return handleExceptions(argv[0], [&]() {
-        initNix();
-        initGC();
-
+    {
         HashType ht = htSHA256;
         std::vector<string> args;
         bool printPath = getEnv("PRINT_PATH") != "";
@@ -94,10 +98,15 @@ int main(int argc, char * * argv)
         if (args.size() > 2)
             throw UsageError("too many arguments");
 
+        Finally f([]() { stopProgressBar(); });
+
+        if (isatty(STDERR_FILENO))
+          startProgressBar();
+
         auto store = openStore();
-        EvalState state(myArgs.searchPath, store);
+        auto state = std::make_unique<EvalState>(myArgs.searchPath, store);
 
-        Bindings & autoArgs = *myArgs.getAutoArgs(state);
+        Bindings & autoArgs = *myArgs.getAutoArgs(*state);
 
         /* If -A is given, get the URI from the specified Nix
            expression. */
@@ -107,33 +116,33 @@ int main(int argc, char * * argv)
                 throw UsageError("you must specify a URI");
             uri = args[0];
         } else {
-            Path path = resolveExprPath(lookupFileArg(state, args.empty() ? "." : args[0]));
+            Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0]));
             Value vRoot;
-            state.evalFile(path, vRoot);
-            Value & v(*findAlongAttrPath(state, attrPath, autoArgs, vRoot));
-            state.forceAttrs(v);
+            state->evalFile(path, vRoot);
+            Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot));
+            state->forceAttrs(v);
 
             /* Extract the URI. */
-            auto attr = v.attrs->find(state.symbols.create("urls"));
+            auto attr = v.attrs->find(state->symbols.create("urls"));
             if (attr == v.attrs->end())
                 throw Error("attribute set does not contain a 'urls' attribute");
-            state.forceList(*attr->value);
+            state->forceList(*attr->value);
             if (attr->value->listSize() < 1)
                 throw Error("'urls' list is empty");
-            uri = state.forceString(*attr->value->listElems()[0]);
+            uri = state->forceString(*attr->value->listElems()[0]);
 
             /* Extract the hash mode. */
-            attr = v.attrs->find(state.symbols.create("outputHashMode"));
+            attr = v.attrs->find(state->symbols.create("outputHashMode"));
             if (attr == v.attrs->end())
                 printInfo("warning: this does not look like a fetchurl call");
             else
-                unpack = state.forceString(*attr->value) == "recursive";
+                unpack = state->forceString(*attr->value) == "recursive";
 
             /* Extract the name. */
             if (name.empty()) {
-                attr = v.attrs->find(state.symbols.create("name"));
+                attr = v.attrs->find(state->symbols.create("name"));
                 if (attr != v.attrs->end())
-                    name = state.forceString(*attr->value);
+                    name = state->forceString(*attr->value);
             }
         }
 
@@ -158,16 +167,22 @@ int main(int argc, char * * argv)
 
         if (storePath.empty()) {
 
-            auto actualUri = resolveMirrorUri(state, uri);
-
-            /* Download the file. */
-            DownloadRequest req(actualUri);
-            req.decompress = false;
-            auto result = getDownloader()->download(req);
+            auto actualUri = resolveMirrorUri(*state, uri);
 
             AutoDelete tmpDir(createTempDir(), true);
             Path tmpFile = (Path) tmpDir + "/tmp";
-            writeFile(tmpFile, *result.data);
+
+            /* Download the file. */
+            {
+                AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600);
+                if (!fd) throw SysError("creating temporary file '%s'", tmpFile);
+
+                FdSink sink(fd.get());
+
+                DownloadRequest req(actualUri);
+                req.decompress = false;
+                getDownloader()->download(std::move(req), sink);
+            }
 
             /* Optionally unpack the file. */
             if (unpack) {
@@ -191,7 +206,7 @@ int main(int argc, char * * argv)
 
             /* FIXME: inefficient; addToStore() will also hash
                this. */
-            hash = unpack ? hashPath(ht, tmpFile).first : hashString(ht, *result.data);
+            hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile);
 
             if (expectedHash != Hash(ht) && expectedHash != hash)
                 throw Error(format("hash mismatch for '%1%'") % uri);
@@ -205,11 +220,17 @@ int main(int argc, char * * argv)
             assert(storePath == store->makeFixedOutputPath(unpack, hash, name));
         }
 
+        stopProgressBar();
+
         if (!printPath)
             printInfo(format("path is '%1%'") % storePath);
 
         std::cout << printHash16or32(hash) << std::endl;
         if (printPath)
             std::cout << storePath << std::endl;
-    });
+
+        return 0;
+    }
 }
+
+static RegisterLegacyCommand s1("nix-prefetch-url", _main);
diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc
index 51dedcf0a092..abdfa5e58f93 100644
--- a/src/nix-store/dotgraph.cc
+++ b/src/nix-store/dotgraph.cc
@@ -47,8 +47,7 @@ static string makeNode(const string & id, const string & label,
 static string symbolicName(const string & path)
 {
     string p = baseNameOf(path);
-    int dash = p.find('-');
-    return string(p, dash + 1);
+    return string(p, p.find('-') + 1);
 }
 
 
diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc
new file mode 100644
index 000000000000..670fbe227a4c
--- /dev/null
+++ b/src/nix-store/graphml.cc
@@ -0,0 +1,90 @@
+#include "graphml.hh"
+#include "util.hh"
+#include "store-api.hh"
+#include "derivations.hh"
+
+#include <iostream>
+
+
+using std::cout;
+
+namespace nix {
+
+
+static inline const string & xmlQuote(const string & s)
+{
+    // Luckily, store paths shouldn't contain any character that needs to be
+    // quoted.
+    return s;
+}
+
+
+static string symbolicName(const string & path)
+{
+    string p = baseNameOf(path);
+    return string(p, p.find('-') + 1);
+}
+
+
+static string makeEdge(const string & src, const string & dst)
+{
+    return fmt("  <edge source=\"%1%\" target=\"%2%\"/>\n",
+        xmlQuote(src), xmlQuote(dst));
+}
+
+
+static string makeNode(const ValidPathInfo & info)
+{
+    return fmt(
+        "  <node id=\"%1%\">\n"
+        "    <data key=\"narSize\">%2%</data>\n"
+        "    <data key=\"name\">%3%</data>\n"
+        "    <data key=\"type\">%4%</data>\n"
+        "  </node>\n",
+        info.path,
+        info.narSize,
+        symbolicName(info.path),
+        (isDerivation(info.path) ? "derivation" : "output-path"));
+}
+
+
+void printGraphML(ref<Store> store, const PathSet & roots)
+{
+    PathSet workList(roots);
+    PathSet doneSet;
+    std::pair<PathSet::iterator,bool> ret;
+
+    cout << "<?xml version='1.0' encoding='utf-8'?>\n"
+         << "<graphml xmlns='http://graphml.graphdrawing.org/xmlns'\n"
+         << "    xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'\n"
+         << "    xsi:schemaLocation='http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd'>\n"
+         << "<key id='narSize' for='node' attr.name='narSize' attr.type='int'/>"
+         << "<key id='name' for='node' attr.name='name' attr.type='string'/>"
+         << "<key id='type' for='node' attr.name='type' attr.type='string'/>"
+         << "<graph id='G' edgedefault='directed'>\n";
+
+    while (!workList.empty()) {
+        Path path = *(workList.begin());
+        workList.erase(path);
+
+        ret = doneSet.insert(path);
+        if (ret.second == false) continue;
+
+        ValidPathInfo info = *(store->queryPathInfo(path));
+        cout << makeNode(info);
+
+        for (auto & p : store->queryPathInfo(path)->references) {
+            if (p != path) {
+                workList.insert(p);
+                cout << makeEdge(path, p);
+            }
+        }
+
+    }
+
+    cout << "</graph>\n";
+    cout << "</graphml>\n";
+}
+
+
+}
diff --git a/src/nix-store/xmlgraph.hh b/src/nix-store/graphml.hh
index a6e7d4e2805a..b78df1e49a67 100644
--- a/src/nix-store/xmlgraph.hh
+++ b/src/nix-store/graphml.hh
@@ -6,6 +6,6 @@ namespace nix {
 
 class Store;
 
-void printXmlGraph(ref<Store> store, const PathSet & roots);
+void printGraphML(ref<Store> store, const PathSet & roots);
 
 }
diff --git a/src/nix-store/local.mk b/src/nix-store/local.mk
deleted file mode 100644
index ade0b233adf3..000000000000
--- a/src/nix-store/local.mk
+++ /dev/null
@@ -1,9 +0,0 @@
-programs += nix-store
-
-nix-store_DIR := $(d)
-
-nix-store_SOURCES := $(wildcard $(d)/*.cc)
-
-nix-store_LIBS = libmain libstore libutil libformat
-
-nix-store_LDFLAGS = -lbz2 -pthread $(SODIUM_LIBS)
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index e1e27ceef94d..f324056bb3a1 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -8,7 +8,8 @@
 #include "shared.hh"
 #include "util.hh"
 #include "worker-protocol.hh"
-#include "xmlgraph.hh"
+#include "graphml.hh"
+#include "legacy.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -273,7 +274,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
     enum QueryType
         { qDefault, qOutputs, qRequisites, qReferences, qReferrers
         , qReferrersClosure, qDeriver, qBinding, qHash, qSize
-        , qTree, qGraph, qXml, qResolve, qRoots };
+        , qTree, qGraph, qGraphML, qResolve, qRoots };
     QueryType query = qDefault;
     bool useOutput = false;
     bool includeOutputs = false;
@@ -299,7 +300,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
         else if (i == "--size") query = qSize;
         else if (i == "--tree") query = qTree;
         else if (i == "--graph") query = qGraph;
-        else if (i == "--xml") query = qXml;
+        else if (i == "--graphml") query = qGraphML;
         else if (i == "--resolve") query = qResolve;
         else if (i == "--roots") query = qRoots;
         else if (i == "--use-output" || i == "-u") useOutput = true;
@@ -403,13 +404,13 @@ static void opQuery(Strings opFlags, Strings opArgs)
             break;
         }
 
-        case qXml: {
+        case qGraphML: {
             PathSet roots;
             for (auto & i : opArgs) {
                 PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise);
                 roots.insert(paths.begin(), paths.end());
             }
-            printXmlGraph(ref<Store>(store), roots);
+            printGraphML(ref<Store>(store), roots);
             break;
         }
 
@@ -426,10 +427,11 @@ static void opQuery(Strings opFlags, Strings opArgs)
                     maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise),
                     referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations);
             }
-            Roots roots = store->findRoots();
-            for (auto & i : roots)
-                if (referrers.find(i.second) != referrers.end())
-                    cout << format("%1%\n") % i.first;
+            Roots roots = store->findRoots(false);
+            for (auto & [target, links] : roots)
+                if (referrers.find(target) != referrers.end())
+                    for (auto & link : links)
+                        cout << format("%1% -> %2%\n") % link % target;
             break;
         }
 
@@ -484,11 +486,16 @@ static void opReadLog(Strings opFlags, Strings opArgs)
 static void opDumpDB(Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
-    if (!opArgs.empty())
-        throw UsageError("no arguments expected");
-    PathSet validPaths = store->queryAllValidPaths();
-    for (auto & i : validPaths)
-        cout << store->makeValidityRegistration({i}, true, true);
+    if (!opArgs.empty()) {
+        for (auto & i : opArgs)
+            i = store->followLinksToStorePath(i);
+        for (auto & i : opArgs)
+            cout << store->makeValidityRegistration({i}, true, true);
+    } else {
+        PathSet validPaths = store->queryAllValidPaths();
+        for (auto & i : validPaths)
+            cout << store->makeValidityRegistration({i}, true, true);
+    }
 }
 
 
@@ -584,9 +591,14 @@ static void opGC(Strings opFlags, Strings opArgs)
     if (!opArgs.empty()) throw UsageError("no arguments expected");
 
     if (printRoots) {
-        Roots roots = store->findRoots();
-        for (auto & i : roots)
-            cout << i.first << " -> " << i.second << std::endl;
+        Roots roots = store->findRoots(false);
+        std::set<std::pair<Path, Path>> roots2;
+        // Transpose and sort the roots.
+        for (auto & [target, links] : roots)
+            for (auto & link : links)
+                roots2.emplace(link, target);
+        for (auto & [link, target] : roots2)
+            std::cout << link << " -> " << target << "\n";
     }
 
     else {
@@ -860,7 +872,7 @@ static void opServe(Strings opFlags, Strings opArgs)
             }
 
             case cmdDumpStorePath:
-                dumpPath(readStorePath(*store, in), out);
+                store->narFromPath(readStorePath(*store, in), out);
                 break;
 
             case cmdImportPaths: {
@@ -924,6 +936,28 @@ static void opServe(Strings opFlags, Strings opArgs)
                 break;
             }
 
+            case cmdAddToStoreNar: {
+                if (!writeAllowed) throw Error("importing paths is not allowed");
+
+                ValidPathInfo info;
+                info.path = readStorePath(*store, in);
+                in >> info.deriver;
+                if (!info.deriver.empty())
+                    store->assertStorePath(info.deriver);
+                info.narHash = Hash(readString(in), htSHA256);
+                info.references = readStorePaths<PathSet>(*store, in);
+                in >> info.registrationTime >> info.narSize >> info.ultimate;
+                info.sigs = readStrings<StringSet>(in);
+                in >> info.ca;
+
+                // FIXME: race if addToStore doesn't read source?
+                store->addToStore(info, in, NoRepair, NoCheckSigs);
+
+                out << 1; // indicate success
+
+                break;
+            }
+
             default:
                 throw Error(format("unknown serve command %1%") % cmd);
         }
@@ -971,11 +1005,9 @@ static void opVersion(Strings opFlags, Strings opArgs)
 /* Scan the arguments; find the operation, set global flags, put all
    other flags in a list, and put all other arguments in another
    list. */
-int main(int argc, char * * argv)
+static int _main(int argc, char * * argv)
 {
-    return handleExceptions(argv[0], [&]() {
-        initNix();
-
+    {
         Strings opFlags, opArgs;
         Operation op = 0;
 
@@ -1062,5 +1094,9 @@ int main(int argc, char * * argv)
             store = openStore();
 
         op(opFlags, opArgs);
-    });
+
+        return 0;
+    }
 }
+
+static RegisterLegacyCommand s1("nix-store", _main);
diff --git a/src/nix-store/xmlgraph.cc b/src/nix-store/xmlgraph.cc
deleted file mode 100644
index 0f7be7f7a02d..000000000000
--- a/src/nix-store/xmlgraph.cc
+++ /dev/null
@@ -1,66 +0,0 @@
-#include "xmlgraph.hh"
-#include "util.hh"
-#include "store-api.hh"
-
-#include <iostream>
-
-
-using std::cout;
-
-namespace nix {
-
-
-static inline const string & xmlQuote(const string & s)
-{
-    // Luckily, store paths shouldn't contain any character that needs to be
-    // quoted.
-    return s;
-}
-
-
-static string makeEdge(const string & src, const string & dst)
-{
-    format f = format("  <edge src=\"%1%\" dst=\"%2%\"/>\n")
-      % xmlQuote(src) % xmlQuote(dst);
-    return f.str();
-}
-
-
-static string makeNode(const string & id)
-{
-    format f = format("  <node name=\"%1%\"/>\n") % xmlQuote(id);
-    return f.str();
-}
-
-
-void printXmlGraph(ref<Store> store, const PathSet & roots)
-{
-    PathSet workList(roots);
-    PathSet doneSet;
-
-    cout << "<?xml version='1.0' encoding='utf-8'?>\n"
-         << "<nix>\n";
-
-    while (!workList.empty()) {
-        Path path = *(workList.begin());
-        workList.erase(path);
-
-        if (doneSet.find(path) != doneSet.end()) continue;
-        doneSet.insert(path);
-
-        cout << makeNode(path);
-
-        for (auto & p : store->queryPathInfo(path)->references) {
-            if (p != path) {
-                workList.insert(p);
-                cout << makeEdge(p, path);
-            }
-        }
-
-    }
-
-    cout << "</nix>\n";
-}
-
-
-}
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index d0003790d3b9..e86b96e3f3f2 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -8,7 +8,7 @@ using namespace nix;
 struct CmdAddToStore : MixDryRun, StoreCommand
 {
     Path path;
-    std::experimental::optional<std::string> namePart;
+    std::optional<std::string> namePart;
 
     CmdAddToStore()
     {
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index e4e6c3e303ed..96bd453d87b4 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -69,8 +69,12 @@ struct CmdCopy : StorePathsCommand
             },
 #ifdef ENABLE_S3
             Example{
-                "To populate the current folder build output to a S3 binary cache:",
-                "nix copy --to s3://my-bucket?region=eu-west-1"
+                "To copy Hello to an S3 binary cache:",
+                "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs.hello"
+            },
+            Example{
+                "To copy Hello to an S3-compatible binary cache:",
+                "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs.hello"
             },
 #endif
         };
diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
new file mode 100644
index 000000000000..7b5444619470
--- /dev/null
+++ b/src/nix/doctor.cc
@@ -0,0 +1,124 @@
+#include "command.hh"
+#include "serve-protocol.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "worker-protocol.hh"
+
+using namespace nix;
+
+std::string formatProtocol(unsigned int proto)
+{
+    if (proto) {
+        auto major = GET_PROTOCOL_MAJOR(proto) >> 8;
+        auto minor = GET_PROTOCOL_MINOR(proto);
+        return (format("%1%.%2%") % major % minor).str();
+    }
+    return "unknown";
+}
+
+struct CmdDoctor : StoreCommand
+{
+    bool success = true;
+
+    std::string name() override
+    {
+        return "doctor";
+    }
+
+    std::string description() override
+    {
+        return "check your system for potential problems";
+    }
+
+    void run(ref<Store> store) override
+    {
+        std::cout << "Store uri: " << store->getUri() << std::endl;
+        std::cout << std::endl;
+
+        auto type = getStoreType();
+
+        if (type < tOther) {
+            success &= checkNixInPath();
+            success &= checkProfileRoots(store);
+        }
+        success &= checkStoreProtocol(store->getProtocol());
+
+        if (!success)
+            throw Exit(2);
+    }
+
+    bool checkNixInPath()
+    {
+        PathSet dirs;
+
+        for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":"))
+            if (pathExists(dir + "/nix-env"))
+                dirs.insert(dirOf(canonPath(dir + "/nix-env", true)));
+
+        if (dirs.size() != 1) {
+            std::cout << "Warning: multiple versions of nix found in PATH." << std::endl;
+            std::cout << std::endl;
+            for (auto & dir : dirs)
+                std::cout << "  " << dir << std::endl;
+            std::cout << std::endl;
+            return false;
+        }
+
+        return true;
+    }
+
+    bool checkProfileRoots(ref<Store> store)
+    {
+        PathSet dirs;
+
+        for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":")) {
+            Path profileDir = dirOf(dir);
+            try {
+                Path userEnv = canonPath(profileDir, true);
+
+                if (store->isStorePath(userEnv) && hasSuffix(userEnv, "user-environment")) {
+                    while (profileDir.find("/profiles/") == std::string::npos && isLink(profileDir))
+                        profileDir = absPath(readLink(profileDir), dirOf(profileDir));
+
+                    if (profileDir.find("/profiles/") == std::string::npos)
+                        dirs.insert(dir);
+                }
+            } catch (SysError &) {}
+        }
+
+        if (!dirs.empty()) {
+            std::cout << "Warning: found profiles outside of " << settings.nixStateDir << "/profiles." << std::endl;
+            std::cout << "The generation this profile points to might not have a gcroot and could be" << std::endl;
+            std::cout << "garbage collected, resulting in broken symlinks." << std::endl;
+            std::cout << std::endl;
+            for (auto & dir : dirs)
+                std::cout << "  " << dir << std::endl;
+            std::cout << std::endl;
+            return false;
+        }
+
+        return true;
+    }
+
+    bool checkStoreProtocol(unsigned int storeProto)
+    {
+        unsigned int clientProto = GET_PROTOCOL_MAJOR(SERVE_PROTOCOL_VERSION) == GET_PROTOCOL_MAJOR(storeProto)
+            ? SERVE_PROTOCOL_VERSION
+            : PROTOCOL_VERSION;
+
+        if (clientProto != storeProto) {
+            std::cout << "Warning: protocol version of this client does not match the store." << std::endl;
+            std::cout << "While this is not necessarily a problem it's recommended to keep the client in" << std::endl;
+            std::cout << "sync with the daemon." << std::endl;
+            std::cout << std::endl;
+            std::cout << "Client protocol: " << formatProtocol(clientProto) << std::endl;
+            std::cout << "Store protocol: " << formatProtocol(storeProto) << std::endl;
+            std::cout << std::endl;
+            return false;
+        }
+
+        return true;
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdDoctor>());
diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index 7eaa86e2f914..c9671f76d0fa 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -61,7 +61,7 @@ struct CmdEdit : InstallableCommand
 
         auto editor = getEnv("EDITOR", "cat");
 
-        Strings args{editor};
+        auto args = tokenizeString<Strings>(editor);
 
         if (editor.find("emacs") != std::string::npos ||
             editor.find("nano") != std::string::npos ||
@@ -72,7 +72,7 @@ struct CmdEdit : InstallableCommand
 
         stopProgressBar();
 
-        execvp(editor.c_str(), stringsToCharPtrs(args).data());
+        execvp(args.front().c_str(), stringsToCharPtrs(args).data());
 
         throw SysError("cannot run editor '%s'", editor);
     }
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 64062fb97955..af4105e28904 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -9,13 +9,14 @@ struct CmdHash : Command
 {
     enum Mode { mFile, mPath };
     Mode mode;
-    Base base = Base16;
+    Base base = SRI;
     bool truncate = false;
-    HashType ht = htSHA512;
+    HashType ht = htSHA256;
     std::vector<std::string> paths;
 
     CmdHash(Mode mode) : mode(mode)
     {
+        mkFlag(0, "sri", "print hash in SRI format", &base, SRI);
         mkFlag(0, "base64", "print hash in base-64", &base, Base64);
         mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32);
         mkFlag(0, "base16", "print hash in base-16", &base, Base16);
@@ -43,7 +44,7 @@ struct CmdHash : Command
             Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first;
             if (truncate && h.hashSize > 20) h = compressHash(h, 20);
             std::cout << format("%1%\n") %
-                h.to_string(base, false);
+                h.to_string(base, base == SRI);
         }
     }
 };
@@ -54,7 +55,7 @@ static RegisterCommand r2(make_ref<CmdHash>(CmdHash::mPath));
 struct CmdToBase : Command
 {
     Base base;
-    HashType ht = htSHA512;
+    HashType ht = htUnknown;
     std::vector<std::string> args;
 
     CmdToBase(Base base) : base(base)
@@ -70,26 +71,30 @@ struct CmdToBase : Command
         return
             base == Base16 ? "to-base16" :
             base == Base32 ? "to-base32" :
-            "to-base64";
+            base == Base64 ? "to-base64" :
+            "to-sri";
     }
 
     std::string description() override
     {
-        return fmt("convert a hash to base-%d representation",
-            base == Base16 ? 16 :
-            base == Base32 ? 32 : 64);
+        return fmt("convert a hash to %s representation",
+            base == Base16 ? "base-16" :
+            base == Base32 ? "base-32" :
+            base == Base64 ? "base-64" :
+            "SRI");
     }
 
     void run() override
     {
         for (auto s : args)
-            std::cout << fmt("%s\n", Hash(s, ht).to_string(base, false));
+            std::cout << fmt("%s\n", Hash(s, ht).to_string(base, base == SRI));
     }
 };
 
 static RegisterCommand r3(make_ref<CmdToBase>(Base16));
 static RegisterCommand r4(make_ref<CmdToBase>(Base32));
 static RegisterCommand r5(make_ref<CmdToBase>(Base64));
+static RegisterCommand r6(make_ref<CmdToBase>(SRI));
 
 /* Legacy nix-hash command. */
 static int compatNixHash(int argc, char * * argv)
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index a3fdd8a2808d..0c1ad3ab3db0 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -86,22 +86,6 @@ Buildable Installable::toBuildable()
     return std::move(buildables[0]);
 }
 
-struct InstallableStoreDrv : Installable
-{
-    Path drvPath;
-
-    InstallableStoreDrv(const Path & drvPath) : drvPath(drvPath) { }
-
-    std::string what() override { return drvPath; }
-
-    Buildables toBuildables() override
-    {
-        Buildable b = {drvPath};
-        // FIXME: add outputs?
-        return {b};
-    }
-};
-
 struct InstallableStorePath : Installable
 {
     Path storePath;
@@ -112,7 +96,7 @@ struct InstallableStorePath : Installable
 
     Buildables toBuildables() override
     {
-        return {{"", {{"out", storePath}}}};
+        return {{isDerivation(storePath) ? storePath : "", {{"out", storePath}}}};
     }
 };
 
@@ -226,12 +210,8 @@ static std::vector<std::shared_ptr<Installable>> parseInstallables(
 
             auto path = store->toStorePath(store->followLinksToStore(s));
 
-            if (store->isStorePath(path)) {
-                if (isDerivation(path))
-                    result.push_back(std::make_shared<InstallableStoreDrv>(path));
-                else
-                    result.push_back(std::make_shared<InstallableStorePath>(path));
-            }
+            if (store->isStorePath(path))
+                result.push_back(std::make_shared<InstallableStorePath>(path));
         }
 
         else if (s == "" || std::regex_match(s, attrPathRegex))
diff --git a/src/nix/local.mk b/src/nix/local.mk
index f76da194467c..ca4604d566c3 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -2,10 +2,24 @@ programs += nix
 
 nix_DIR := $(d)
 
-nix_SOURCES := $(wildcard $(d)/*.cc) $(wildcard src/linenoise/*.cpp)
+nix_SOURCES := \
+  $(wildcard $(d)/*.cc) \
+  $(wildcard src/build-remote/*.cc) \
+  $(wildcard src/nix-build/*.cc) \
+  $(wildcard src/nix-channel/*.cc) \
+  $(wildcard src/nix-collect-garbage/*.cc) \
+  $(wildcard src/nix-copy-closure/*.cc) \
+  $(wildcard src/nix-daemon/*.cc) \
+  $(wildcard src/nix-env/*.cc) \
+  $(wildcard src/nix-instantiate/*.cc) \
+  $(wildcard src/nix-prefetch-url/*.cc) \
+  $(wildcard src/nix-store/*.cc) \
 
-nix_LIBS = libexpr libmain libstore libutil libformat
+nix_LIBS = libexpr libmain libstore libutil
 
-nix_LDFLAGS = -pthread
+nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS)
 
-$(eval $(call install-symlink, nix, $(bindir)/nix-hash))
+$(foreach name, \
+  nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \
+  $(eval $(call install-symlink, nix, $(bindir)/$(name))))
+$(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote))
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index e99622faf472..d089be42fb20 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -148,7 +148,7 @@ struct CmdLsNar : Command, MixLs
 
     void run() override
     {
-        list(makeNarAccessor(make_ref<std::string>(readFile(narPath))));
+        list(makeNarAccessor(make_ref<std::string>(readFile(narPath, true))));
     }
 };
 
diff --git a/src/nix/main.cc b/src/nix/main.cc
index bb107ec7d3f6..4f87ad72b65c 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -20,11 +20,12 @@ std::string programPath;
 
 struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 {
+    bool printBuildLogs = false;
+
     NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix")
     {
         mkFlag()
             .longName("help")
-            .shortName('h')
             .description("show usage information")
             .handler([&]() { showHelpAndExit(); });
 
@@ -34,14 +35,20 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
             .handler([&]() {
                 std::cout << "The following configuration options are available:\n\n";
                 Table2 tbl;
-                for (const auto & s : settings._getSettings())
-                    if (!s.second.isAlias)
-                        tbl.emplace_back(s.first, s.second.setting->description);
+                std::map<std::string, Config::SettingInfo> settings;
+                globalConfig.getSettings(settings);
+                for (const auto & s : settings)
+                    tbl.emplace_back(s.first, s.second.description);
                 printTable(std::cout, tbl);
                 throw Exit();
             });
 
         mkFlag()
+            .longName("print-build-logs")
+            .description("print full build logs on stderr")
+            .set(&printBuildLogs, true);
+
+        mkFlag()
             .longName("version")
             .description("show version information")
             .handler([&]() { printVersion(programName); });
@@ -67,9 +74,6 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 
 void mainWrapped(int argc, char * * argv)
 {
-    verbosity = lvlError;
-    settings.verboseBuild = false;
-
     /* The chroot helper needs to be run before any threads have been
        started. */
     if (argc > 0 && argv[0] == chrootHelperName) {
@@ -88,6 +92,9 @@ void mainWrapped(int argc, char * * argv)
         if (legacy) return legacy(argc, argv);
     }
 
+    verbosity = lvlError;
+    settings.verboseBuild = false;
+
     NixArgs args;
 
     args.parseCmdline(argvToStrings(argc, argv));
@@ -98,8 +105,7 @@ void mainWrapped(int argc, char * * argv)
 
     Finally f([]() { stopProgressBar(); });
 
-    if (isatty(STDERR_FILENO))
-        startProgressBar();
+    startProgressBar(args.printBuildLogs);
 
     args.command->prepare();
     args.command->run();
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index 47caa401d3c9..dea5f0557b81 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -4,8 +4,8 @@
 #include "json.hh"
 #include "common-args.hh"
 
-#include <iomanip>
 #include <algorithm>
+#include <array>
 
 using namespace nix;
 
@@ -13,12 +13,14 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
 {
     bool showSize = false;
     bool showClosureSize = false;
+    bool humanReadable = false;
     bool showSigs = false;
 
     CmdPathInfo()
     {
         mkFlag('s', "size", "print size of the NAR dump of each path", &showSize);
         mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize);
+        mkFlag('h', "human-readable", "with -s and -S, print sizes like 1K 234M 5.67G etc.", &humanReadable);
         mkFlag(0, "sigs", "show signatures", &showSigs);
     }
 
@@ -40,6 +42,10 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
                 "nix path-info -rS /run/current-system | sort -nk2"
             },
             Example{
+                "To show a package's closure size and all its dependencies with human readable sizes:",
+                "nix path-info -rsSh nixpkgs.rust"
+            },
+            Example{
                 "To check the existence of a path in a binary cache:",
                 "nix path-info -r /nix/store/7qvk5c91...-geeqie-1.1 --store https://cache.nixos.org/"
             },
@@ -58,6 +64,25 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
         };
     }
 
+    void printSize(unsigned long long value)
+    {
+        if (!humanReadable) {
+            std::cout << fmt("\t%11d", value);
+            return;
+        }
+
+        static const std::array<char, 9> idents{{
+            ' ', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'
+        }};
+        size_t power = 0;
+        double res = value;
+        while (res > 1024 && power < idents.size()) {
+            ++power;
+            res /= 1024;
+        }
+        std::cout << fmt("\t%6.1f%c", res, idents.at(power));
+    }
+
     void run(ref<Store> store, Paths storePaths) override
     {
         size_t pathLen = 0;
@@ -78,13 +103,16 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
                 auto info = store->queryPathInfo(storePath);
                 storePath = info->path; // FIXME: screws up padding
 
-                std::cout << storePath << std::string(std::max(0, (int) pathLen - (int) storePath.size()), ' ');
+                std::cout << storePath;
+
+                if (showSize || showClosureSize || showSigs)
+                    std::cout << std::string(std::max(0, (int) pathLen - (int) storePath.size()), ' ');
 
                 if (showSize)
-                    std::cout << '\t' << std::setw(11) << info->narSize;
+                    printSize(info->narSize);
 
                 if (showClosureSize)
-                    std::cout << '\t' << std::setw(11) << store->getClosureSize(storePath).first;
+                    printSize(store->getClosureSize(storePath).first);
 
                 if (showSigs) {
                     std::cout << '\t';
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 40b905ba3243..e7104540816b 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -2,6 +2,7 @@
 #include "util.hh"
 #include "sync.hh"
 #include "store-api.hh"
+#include "names.hh"
 
 #include <atomic>
 #include <map>
@@ -38,6 +39,7 @@ private:
         std::map<ActivityType, uint64_t> expectedByType;
         bool visible = true;
         ActivityId parent;
+        std::optional<std::string> name;
     };
 
     struct ActivitiesByType
@@ -68,10 +70,16 @@ private:
 
     std::condition_variable quitCV, updateCV;
 
+    bool printBuildLogs;
+    bool isTTY;
+
 public:
 
-    ProgressBar()
+    ProgressBar(bool printBuildLogs, bool isTTY)
+        : printBuildLogs(printBuildLogs)
+        , isTTY(isTTY)
     {
+        state_.lock()->active = isTTY;
         updateThread = std::thread([&]() {
             auto state(state_.lock());
             while (state->active) {
@@ -109,8 +117,14 @@ public:
 
     void log(State & state, Verbosity lvl, const std::string & s)
     {
-        writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n");
-        draw(state);
+        if (state.active) {
+            writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n");
+            draw(state);
+        } else {
+            auto s2 = s + ANSI_NORMAL "\n";
+            if (!isTTY) s2 = filterANSIEscapes(s2, true);
+            writeToStderr(s2);
+        }
     }
 
     void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
@@ -141,6 +155,7 @@ public:
             auto nrRounds = getI(fields, 3);
             if (nrRounds != 1)
                 i->s += fmt(" (round %d/%d)", curRound, nrRounds);
+            i->name = DrvName(name).name;
         }
 
         if (type == actSubstitute) {
@@ -217,11 +232,15 @@ public:
                 auto i = state->its.find(act);
                 assert(i != state->its.end());
                 ActInfo info = *i->second;
-                state->activities.erase(i->second);
-                info.lastLine = lastLine;
-                state->activities.emplace_back(info);
-                i->second = std::prev(state->activities.end());
-                update();
+                if (printBuildLogs) {
+                    log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + "> " + ANSI_NORMAL + lastLine);
+                } else {
+                    state->activities.erase(i->second);
+                    info.lastLine = lastLine;
+                    state->activities.emplace_back(info);
+                    i->second = std::prev(state->activities.end());
+                    update();
+                }
             }
         }
 
@@ -333,11 +352,18 @@ public:
 
             if (running || done || expected || failed) {
                 if (running)
-                    s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt,
-                        running / unit, done / unit, expected / unit);
+                    if (expected != 0)
+                        s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt,
+                            running / unit, done / unit, expected / unit);
+                    else
+                        s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL,
+                            running / unit, done / unit);
                 else if (expected != done)
-                    s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt,
-                        done / unit, expected / unit);
+                    if (expected != 0)
+                        s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt,
+                            done / unit, expected / unit);
+                    else
+                        s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL, done / unit);
                 else
                     s = fmt(done ? ANSI_GREEN + numberFmt + ANSI_NORMAL : numberFmt, done / unit);
                 s = fmt(itemFmt, s);
@@ -395,9 +421,9 @@ public:
     }
 };
 
-void startProgressBar()
+void startProgressBar(bool printBuildLogs)
 {
-    logger = new ProgressBar();
+    logger = new ProgressBar(printBuildLogs, isatty(STDERR_FILENO));
 }
 
 void stopProgressBar()
diff --git a/src/nix/progress-bar.hh b/src/nix/progress-bar.hh
index af8eda5a84fd..4d61175c24e4 100644
--- a/src/nix/progress-bar.hh
+++ b/src/nix/progress-bar.hh
@@ -4,7 +4,7 @@
 
 namespace nix {
 
-void startProgressBar();
+void startProgressBar(bool printBuildLogs = false);
 
 void stopProgressBar();
 
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index f84774a53367..d8f812149069 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -1,8 +1,17 @@
 #include <iostream>
 #include <cstdlib>
+#include <cstring>
+#include <climits>
 
 #include <setjmp.h>
 
+#ifdef READLINE
+#include <readline/history.h>
+#include <readline/readline.h>
+#else
+#include <editline.h>
+#endif
+
 #include "shared.hh"
 #include "eval.hh"
 #include "eval-inline.hh"
@@ -15,8 +24,6 @@
 #include "command.hh"
 #include "finally.hh"
 
-#include "src/linenoise/linenoise.h"
-
 namespace nix {
 
 #define ESC_RED "\033[31m"
@@ -31,6 +38,7 @@ struct NixRepl
 {
     string curDir;
     EvalState state;
+    Bindings * autoArgs;
 
     Strings loadedFiles;
 
@@ -117,19 +125,81 @@ NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store)
 
 NixRepl::~NixRepl()
 {
-    linenoiseHistorySave(historyFile.c_str());
+    write_history(historyFile.c_str());
 }
 
-
 static NixRepl * curRepl; // ugly
 
-static void completionCallback(const char * s, linenoiseCompletions *lc)
-{
-    /* Otherwise, return all symbols that start with the prefix. */
-    for (auto & c : curRepl->completePrefix(s))
-        linenoiseAddCompletion(lc, c.c_str());
+static char * completionCallback(char * s, int *match) {
+  auto possible = curRepl->completePrefix(s);
+  if (possible.size() == 1) {
+    *match = 1;
+    auto *res = strdup(possible.begin()->c_str() + strlen(s));
+    if (!res) throw Error("allocation failure");
+    return res;
+  } else if (possible.size() > 1) {
+    auto checkAllHaveSameAt = [&](size_t pos) {
+      auto &first = *possible.begin();
+      for (auto &p : possible) {
+        if (p.size() <= pos || p[pos] != first[pos])
+          return false;
+      }
+      return true;
+    };
+    size_t start = strlen(s);
+    size_t len = 0;
+    while (checkAllHaveSameAt(start + len)) ++len;
+    if (len > 0) {
+      *match = 1;
+      auto *res = strdup(std::string(*possible.begin(), start, len).c_str());
+      if (!res) throw Error("allocation failure");
+      return res;
+    }
+  }
+
+  *match = 0;
+  return nullptr;
+}
+
+static int listPossibleCallback(char *s, char ***avp) {
+  auto possible = curRepl->completePrefix(s);
+
+  if (possible.size() > (INT_MAX / sizeof(char*)))
+    throw Error("too many completions");
+
+  int ac = 0;
+  char **vp = nullptr;
+
+  auto check = [&](auto *p) {
+    if (!p) {
+      if (vp) {
+        while (--ac >= 0)
+          free(vp[ac]);
+        free(vp);
+      }
+      throw Error("allocation failure");
+    }
+    return p;
+  };
+
+  vp = check((char **)malloc(possible.size() * sizeof(char*)));
+
+  for (auto & p : possible)
+    vp[ac++] = check(strdup(p.c_str()));
+
+  *avp = vp;
+
+  return ac;
 }
 
+namespace {
+    // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline.
+    volatile sig_atomic_t g_signal_received = 0;
+
+    void sigintHandler(int signo) {
+        g_signal_received = signo;
+    }
+}
 
 void NixRepl::mainLoop(const std::vector<std::string> & files)
 {
@@ -142,12 +212,18 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
     reloadFiles();
     if (!loadedFiles.empty()) std::cout << std::endl;
 
+    // Allow nix-repl specific settings in .inputrc
+    rl_readline_name = "nix-repl";
     createDirs(dirOf(historyFile));
-    linenoiseHistorySetMaxLen(1000);
-    linenoiseHistoryLoad(historyFile.c_str());
-
+#ifndef READLINE
+    el_hist_size = 1000;
+#endif
+    read_history(historyFile.c_str());
     curRepl = this;
-    linenoiseSetCompletionCallback(completionCallback);
+#ifndef READLINE
+    rl_set_complete_func(completionCallback);
+    rl_set_list_possib_func(listPossibleCallback);
+#endif
 
     std::string input;
 
@@ -175,7 +251,6 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
 
         // We handled the current input fully, so we should clear it
         // and read brand new input.
-        linenoiseHistoryAdd(input.c_str());
         input.clear();
         std::cout << std::endl;
     }
@@ -184,19 +259,42 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
 
 bool NixRepl::getLine(string & input, const std::string &prompt)
 {
-    char * s = linenoise(prompt.c_str());
+    struct sigaction act, old;
+    sigset_t savedSignalMask, set;
+
+    auto setupSignals = [&]() {
+        act.sa_handler = sigintHandler;
+        sigfillset(&act.sa_mask);
+        act.sa_flags = 0;
+        if (sigaction(SIGINT, &act, &old))
+            throw SysError("installing handler for SIGINT");
+
+        sigemptyset(&set);
+        sigaddset(&set, SIGINT);
+        if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask))
+            throw SysError("unblocking SIGINT");
+    };
+    auto restoreSignals = [&]() {
+        if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
+            throw SysError("restoring signals");
+
+        if (sigaction(SIGINT, &old, 0))
+            throw SysError("restoring handler for SIGINT");
+    };
+
+    setupSignals();
+    char * s = readline(prompt.c_str());
     Finally doFree([&]() { free(s); });
-    if (!s) {
-      switch (auto type = linenoiseKeyType()) {
-        case 1: // ctrl-C
-          input = "";
-          return true;
-        case 2: // ctrl-D
-          return false;
-        default:
-          throw Error(format("Unexpected linenoise keytype: %1%") % type);
-      }
+    restoreSignals();
+
+    if (g_signal_received) {
+        g_signal_received = 0;
+        input.clear();
+        return true;
     }
+
+    if (!s)
+      return false;
     input += s;
     input += '\n';
     return true;
@@ -385,7 +483,7 @@ bool NixRepl::processLine(string line)
             /* We could do the build in this process using buildPaths(),
                but doing it in a child makes it easier to recover from
                problems / SIGINT. */
-            if (runProgram(settings.nixBinDir + "/nix-store", Strings{"-r", drvPath}) == 0) {
+            if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) {
                 Derivation drv = readDerivation(drvPath);
                 std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
                 for (auto & i : drv.outputs)
@@ -441,8 +539,7 @@ void NixRepl::loadFile(const Path & path)
     loadedFiles.push_back(path);
     Value v, v2;
     state.evalFile(lookupFileArg(state, path), v);
-    Bindings & bindings(*state.allocBindings(0));
-    state.autoCallFunction(bindings, v, v2);
+    state.autoCallFunction(*autoArgs, v, v2);
     addAttrsToScope(v2);
 }
 
@@ -584,30 +681,13 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
             for (auto & i : *v.attrs)
                 sorted[i.name] = i.value;
 
-            /* If this is a derivation, then don't show the
-               self-references ("all", "out", etc.). */
-            StringSet hidden;
-            if (isDrv) {
-                hidden.insert("all");
-                Bindings::iterator i = v.attrs->find(state.sOutputs);
-                if (i == v.attrs->end())
-                    hidden.insert("out");
-                else {
-                    state.forceList(*i->value);
-                    for (unsigned int j = 0; j < i->value->listSize(); ++j)
-                        hidden.insert(state.forceStringNoCtx(*i->value->listElems()[j]));
-                }
-            }
-
             for (auto & i : sorted) {
                 if (isVarName(i.first))
                     str << i.first;
                 else
                     printStringValue(str, i.first.c_str());
                 str << " = ";
-                if (hidden.find(i.first) != hidden.end())
-                    str << "ยซ...ยป";
-                else if (seen.find(i.second) != seen.end())
+                if (seen.find(i.second) != seen.end())
                     str << "ยซrepeatedยป";
                 else
                     try {
@@ -693,8 +773,9 @@ struct CmdRepl : StoreCommand, MixEvalArgs
 
     void run(ref<Store> store) override
     {
-        NixRepl repl(searchPath, openStore());
-        repl.mainLoop(files);
+        auto repl = std::make_unique<NixRepl>(searchPath, openStore());
+        repl->autoArgs = getAutoArgs(repl->state);
+        repl->mainLoop(files);
     }
 };
 
diff --git a/src/nix/run.cc b/src/nix/run.cc
index d04e106e037b..35b763345872 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -7,11 +7,14 @@
 #include "finally.hh"
 #include "fs-accessor.hh"
 #include "progress-bar.hh"
+#include "affinity.hh"
 
 #if __linux__
 #include <sys/mount.h>
 #endif
 
+#include <queue>
+
 using namespace nix;
 
 std::string chrootHelperName = "__run_in_chroot";
@@ -121,10 +124,27 @@ struct CmdRun : InstallablesCommand
                 unsetenv(var.c_str());
         }
 
+        std::unordered_set<Path> done;
+        std::queue<Path> todo;
+        for (auto & path : outPaths) todo.push(path);
+
         auto unixPath = tokenizeString<Strings>(getEnv("PATH"), ":");
-        for (auto & path : outPaths)
-            if (accessor->stat(path + "/bin").type != FSAccessor::tMissing)
+
+        while (!todo.empty()) {
+            Path path = todo.front();
+            todo.pop();
+            if (!done.insert(path).second) continue;
+
+            if (true)
                 unixPath.push_front(path + "/bin");
+
+            auto propPath = path + "/nix-support/propagated-user-env-packages";
+            if (accessor->stat(propPath).type == FSAccessor::tRegular) {
+                for (auto & p : tokenizeString<Paths>(readFile(propPath)))
+                    todo.push(p);
+            }
+        }
+
         setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1);
 
         std::string cmd = *command.begin();
@@ -135,6 +155,8 @@ struct CmdRun : InstallablesCommand
 
         restoreSignals();
 
+        restoreAffinity();
+
         /* If this is a diverted store (i.e. its "logical" location
            (typically /nix/store) differs from its "physical" location
            (e.g. /home/eelco/nix/store), then run the command in a
diff --git a/src/nix/search.cc b/src/nix/search.cc
index 5ccf1b7cf529..e086de2260a6 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -7,32 +7,38 @@
 #include "common-args.hh"
 #include "json.hh"
 #include "json-to-value.hh"
+#include "shared.hh"
 
 #include <regex>
 #include <fstream>
 
 using namespace nix;
 
-std::string hilite(const std::string & s, const std::smatch & m)
+std::string wrap(std::string prefix, std::string s)
+{
+    return prefix + s + ANSI_NORMAL;
+}
+
+std::string hilite(const std::string & s, const std::smatch & m, std::string postfix)
 {
     return
         m.empty()
         ? s
         : std::string(m.prefix())
-          + ANSI_RED + std::string(m.str()) + ANSI_NORMAL
+          + ANSI_RED + std::string(m.str()) + postfix
           + std::string(m.suffix());
 }
 
 struct CmdSearch : SourceExprCommand, MixJSON
 {
-    std::string re;
+    std::vector<std::string> res;
 
     bool writeCache = true;
     bool useCache = true;
 
     CmdSearch()
     {
-        expectArg("regex", &re, true);
+        expectArgs("regex", &res);
 
         mkFlag()
             .longName("update-cache")
@@ -68,9 +74,17 @@ struct CmdSearch : SourceExprCommand, MixJSON
                 "nix search blender"
             },
             Example{
-                "To search for Firefox and Chromium:",
+                "To search for Firefox or Chromium:",
                 "nix search 'firefox|chromium'"
             },
+            Example{
+                "To search for git and frontend or gui:",
+                "nix search git 'frontend|gui'"
+            },
+            Example{
+                "To display the description of the found packages:",
+                "nix search git --verbose"
+            }
         };
     }
 
@@ -81,9 +95,16 @@ struct CmdSearch : SourceExprCommand, MixJSON
         // Empty search string should match all packages
         // Use "^" here instead of ".*" due to differences in resulting highlighting
         // (see #1893 -- libc++ claims empty search string is not in POSIX grammar)
-        if (re.empty()) re = "^";
+        if (res.empty()) {
+            res.push_back("^");
+        }
 
-        std::regex regex(re, std::regex::extended | std::regex::icase);
+        std::vector<std::regex> regexes;
+        regexes.reserve(res.size());
+
+        for (auto &re : res) {
+            regexes.push_back(std::regex(re, std::regex::extended | std::regex::icase));
+        }
 
         auto state = getEvalState();
 
@@ -102,6 +123,7 @@ struct CmdSearch : SourceExprCommand, MixJSON
             debug("at attribute '%s'", attrPath);
 
             try {
+                uint found = 0;
 
                 state->forceValue(*v);
 
@@ -115,25 +137,33 @@ struct CmdSearch : SourceExprCommand, MixJSON
                 if (state->isDerivation(*v)) {
 
                     DrvInfo drv(*state, attrPath, v->attrs);
+                    std::string description;
+                    std::smatch attrPathMatch;
+                    std::smatch descriptionMatch;
+                    std::smatch nameMatch;
+                    std::string name;
 
                     DrvName parsed(drv.queryName());
 
-                    std::smatch attrPathMatch;
-                    std::regex_search(attrPath, attrPathMatch, regex);
+                    for (auto &regex : regexes) {
+                        std::regex_search(attrPath, attrPathMatch, regex);
 
-                    auto name = parsed.name;
-                    std::smatch nameMatch;
-                    std::regex_search(name, nameMatch, regex);
+                        name = parsed.name;
+                        std::regex_search(name, nameMatch, regex);
 
-                    std::string description = drv.queryMetaString("description");
-                    std::replace(description.begin(), description.end(), '\n', ' ');
-                    std::smatch descriptionMatch;
-                    std::regex_search(description, descriptionMatch, regex);
+                        description = drv.queryMetaString("description");
+                        std::replace(description.begin(), description.end(), '\n', ' ');
+                        std::regex_search(description, descriptionMatch, regex);
 
-                    if (!attrPathMatch.empty()
-                        || !nameMatch.empty()
-                        || !descriptionMatch.empty())
-                    {
+                        if (!attrPathMatch.empty()
+                            || !nameMatch.empty()
+                            || !descriptionMatch.empty())
+                        {
+                            found++;
+                        }
+                    }
+
+                    if (found == res.size()) {
                         if (json) {
 
                             auto jsonElem = jsonOut->object(attrPath);
@@ -143,15 +173,13 @@ struct CmdSearch : SourceExprCommand, MixJSON
                             jsonElem.attr("description", description);
 
                         } else {
+                            auto name = hilite(parsed.name, nameMatch, "\e[0;2m")
+                                + std::string(parsed.fullName, parsed.name.length());
                             results[attrPath] = fmt(
-                                "Attribute name: %s\n"
-                                "Package name: %s\n"
-                                "Version: %s\n"
-                                "Description: %s\n",
-                                hilite(attrPath, attrPathMatch),
-                                hilite(name, nameMatch),
-                                parsed.version,
-                                hilite(description, descriptionMatch));
+                                "* %s (%s)\n  %s\n",
+                                wrap("\e[0;1m", hilite(attrPath, attrPathMatch, "\e[0;1m")),
+                                wrap("\e[0;2m", hilite(name, nameMatch, "\e[0;2m")),
+                                hilite(description, descriptionMatch, ANSI_NORMAL));
                         }
                     }
 
@@ -243,6 +271,10 @@ struct CmdSearch : SourceExprCommand, MixJSON
                 throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName);
         }
 
+        if (results.size() == 0)
+            throw Error("no results for the given search term(s)!");
+
+        RunPager pager;
         for (auto el : results) std::cout << el.second << "\n";
 
     }
diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc
index c64b12c8dd62..86638b50d2c6 100644
--- a/src/nix/show-config.cc
+++ b/src/nix/show-config.cc
@@ -27,10 +27,12 @@ struct CmdShowConfig : Command, MixJSON
         if (json) {
             // FIXME: use appropriate JSON types (bool, ints, etc).
             JSONObject jsonObj(std::cout);
-            settings.toJSON(jsonObj);
+            globalConfig.toJSON(jsonObj);
         } else {
-            for (auto & s : settings.getSettings())
-                std::cout << s.first + " = " + s.second + "\n";
+            std::map<std::string, Config::SettingInfo> settings;
+            globalConfig.getSettings(settings);
+            for (auto & s : settings)
+                std::cout << s.first + " = " + s.second.value + "\n";
         }
     }
 };
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 758bbbc688bc..35c44a70cf52 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -1,14 +1,18 @@
 #include "command.hh"
+#include "common-args.hh"
 #include "store-api.hh"
 #include "download.hh"
 #include "eval.hh"
 #include "attr-path.hh"
+#include "names.hh"
+#include "progress-bar.hh"
 
 using namespace nix;
 
-struct CmdUpgradeNix : StoreCommand
+struct CmdUpgradeNix : MixDryRun, StoreCommand
 {
     Path profileDir;
+    std::string storePathsUrl = "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix";
 
     CmdUpgradeNix()
     {
@@ -18,6 +22,12 @@ struct CmdUpgradeNix : StoreCommand
             .labels({"profile-dir"})
             .description("the Nix profile to upgrade")
             .dest(&profileDir);
+
+        mkFlag()
+            .longName("nix-store-paths-url")
+            .labels({"url"})
+            .description("URL of the file that contains the store paths of the latest Nix release")
+            .dest(&storePathsUrl);
     }
 
     std::string name() override
@@ -46,7 +56,7 @@ struct CmdUpgradeNix : StoreCommand
 
     void run(ref<Store> store) override
     {
-        settings.pureEval = true;
+        evalSettings.pureEval = true;
 
         if (profileDir == "")
             profileDir = getProfileDir(store);
@@ -59,6 +69,14 @@ struct CmdUpgradeNix : StoreCommand
             storePath = getLatestNix(store);
         }
 
+        auto version = DrvName(storePathToName(storePath)).version;
+
+        if (dryRun) {
+            stopProgressBar();
+            printError("would upgrade to version %s", version);
+            return;
+        }
+
         {
             Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", storePath));
             store->ensurePath(storePath);
@@ -72,11 +90,15 @@ struct CmdUpgradeNix : StoreCommand
                 throw Error("could not verify that '%s' works", program);
         }
 
+        stopProgressBar();
+
         {
             Activity act(*logger, lvlInfo, actUnknown, fmt("installing '%s' into profile '%s'...", storePath, profileDir));
             runProgram(settings.nixBinDir + "/nix-env", false,
                 {"--profile", profileDir, "-i", storePath, "--no-sandbox"});
         }
+
+        printError(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version);
     }
 
     /* Return the profile in which Nix is installed. */
@@ -98,11 +120,18 @@ struct CmdUpgradeNix : StoreCommand
         if (hasPrefix(where, "/run/current-system"))
             throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'");
 
-        Path profileDir;
-        Path userEnv;
+        Path profileDir = dirOf(where);
+
+        // Resolve profile to /nix/var/nix/profiles/<name> link.
+        while (canonPath(profileDir).find("/profiles/") == std::string::npos && isLink(profileDir))
+            profileDir = readLink(profileDir);
+
+        printInfo("found profile '%s'", profileDir);
+
+        Path userEnv = canonPath(profileDir, true);
 
         if (baseNameOf(where) != "bin" ||
-            !hasSuffix(userEnv = canonPath(profileDir = dirOf(where), true), "user-environment"))
+            !hasSuffix(userEnv, "user-environment"))
             throw Error("directory '%s' does not appear to be part of a Nix profile", where);
 
         if (!store->isValidPath(userEnv))
@@ -115,16 +144,16 @@ struct CmdUpgradeNix : StoreCommand
     Path getLatestNix(ref<Store> store)
     {
         // FIXME: use nixos.org?
-        auto req = DownloadRequest("https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix");
+        auto req = DownloadRequest(storePathsUrl);
         auto res = getDownloader()->download(req);
 
-        EvalState state(Strings(), store);
-        auto v = state.allocValue();
-        state.eval(state.parseExprFromString(*res.data, "/no-such-path"), *v);
-        Bindings & bindings(*state.allocBindings(0));
-        auto v2 = findAlongAttrPath(state, settings.thisSystem, bindings, *v);
+        auto state = std::make_unique<EvalState>(Strings(), store);
+        auto v = state->allocValue();
+        state->eval(state->parseExprFromString(*res.data, "/no-such-path"), *v);
+        Bindings & bindings(*state->allocBindings(0));
+        auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v);
 
-        return state.forceString(*v2);
+        return state->forceString(*v2);
     }
 };
 
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 6540208a8a2c..7ef571561a0e 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -120,7 +120,7 @@ struct CmdVerify : StorePathsCommand
                             for (auto sig : sigs) {
                                 if (sigsSeen.count(sig)) continue;
                                 sigsSeen.insert(sig);
-                                if (info->checkSignature(publicKeys, sig))
+                                if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(publicKeys, sig))
                                     validSigs++;
                             }
                         };
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index 17e0595ae887..325a2be0a793 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -2,6 +2,7 @@
 #include "store-api.hh"
 #include "progress-bar.hh"
 #include "fs-accessor.hh"
+#include "shared.hh"
 
 #include <queue>
 
@@ -237,6 +238,7 @@ struct CmdWhyDepends : SourceExprCommand
 
             visitPath(node.path);
 
+            RunPager pager;
             for (auto & ref : refs) {
                 auto hash = storePathToHash(ref.second->path);
 
diff --git a/src/nlohmann/json.hpp b/src/nlohmann/json.hpp
index 5b0b0ea5b301..c9af0bed36d6 100644
--- a/src/nlohmann/json.hpp
+++ b/src/nlohmann/json.hpp
@@ -1,11 +1,12 @@
 /*
     __ _____ _____ _____
  __|  |   __|     |   | |  JSON for Modern C++
-|  |  |__   |  |  | | | |  version 3.0.1
+|  |  |__   |  |  | | | |  version 3.5.0
 |_____|_____|_____|_|___|  https://github.com/nlohmann/json
 
 Licensed under the MIT License <http://opensource.org/licenses/MIT>.
-Copyright (c) 2013-2017 Niels Lohmann <http://nlohmann.me>.
+SPDX-License-Identifier: MIT
+Copyright (c) 2013-2018 Niels Lohmann <http://nlohmann.me>.
 
 Permission is hereby  granted, free of charge, to any  person obtaining a copy
 of this software and associated  documentation files (the "Software"), to deal
@@ -29,42 +30,104 @@ SOFTWARE.
 #ifndef NLOHMANN_JSON_HPP
 #define NLOHMANN_JSON_HPP
 
-#include <algorithm> // all_of, copy, fill, find, for_each, generate_n, none_of, remove, reverse, transform
-#include <array> // array
+#define NLOHMANN_JSON_VERSION_MAJOR 3
+#define NLOHMANN_JSON_VERSION_MINOR 5
+#define NLOHMANN_JSON_VERSION_PATCH 0
+
+#include <algorithm> // all_of, find, for_each
 #include <cassert> // assert
 #include <ciso646> // and, not, or
-#include <clocale> // lconv, localeconv
-#include <cmath> // isfinite, labs, ldexp, signbit
 #include <cstddef> // nullptr_t, ptrdiff_t, size_t
-#include <cstdint> // int64_t, uint64_t
-#include <cstdlib> // abort, strtod, strtof, strtold, strtoul, strtoll, strtoull
-#include <cstring> // memcpy, strlen
-#include <forward_list> // forward_list
-#include <functional> // function, hash, less
+#include <functional> // hash, less
 #include <initializer_list> // initializer_list
-#include <iomanip> // hex
-#include <iosfwd>   // istream, ostream
-#include <iterator> // advance, begin, back_inserter, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator
-#include <limits> // numeric_limits
-#include <locale> // locale
-#include <map> // map
-#include <memory> // addressof, allocator, allocator_traits, unique_ptr
+#include <iosfwd> // istream, ostream
+#include <iterator> // random_access_iterator_tag
 #include <numeric> // accumulate
-#include <sstream> // stringstream
-#include <string> // getline, stoi, string, to_string
-#include <type_traits> // add_pointer, conditional, decay, enable_if, false_type, integral_constant, is_arithmetic, is_base_of, is_const, is_constructible, is_convertible, is_default_constructible, is_enum, is_floating_point, is_integral, is_nothrow_move_assignable, is_nothrow_move_constructible, is_pointer, is_reference, is_same, is_scalar, is_signed, remove_const, remove_cv, remove_pointer, remove_reference, true_type, underlying_type
-#include <utility> // declval, forward, make_pair, move, pair, swap
-#include <valarray> // valarray
+#include <string> // string, stoi, to_string
+#include <utility> // declval, forward, move, pair, swap
+
+// #include <nlohmann/json_fwd.hpp>
+#ifndef NLOHMANN_JSON_FWD_HPP
+#define NLOHMANN_JSON_FWD_HPP
+
+#include <cstdint> // int64_t, uint64_t
+#include <map> // map
+#include <memory> // allocator
+#include <string> // string
 #include <vector> // vector
 
+/*!
+@brief namespace for Niels Lohmann
+@see https://github.com/nlohmann
+@since version 1.0.0
+*/
+namespace nlohmann
+{
+/*!
+@brief default JSONSerializer template argument
+
+This serializer ignores the template arguments and uses ADL
+([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl))
+for serialization.
+*/
+template<typename T = void, typename SFINAE = void>
+struct adl_serializer;
+
+template<template<typename U, typename V, typename... Args> class ObjectType =
+         std::map,
+         template<typename U, typename... Args> class ArrayType = std::vector,
+         class StringType = std::string, class BooleanType = bool,
+         class NumberIntegerType = std::int64_t,
+         class NumberUnsignedType = std::uint64_t,
+         class NumberFloatType = double,
+         template<typename U> class AllocatorType = std::allocator,
+         template<typename T, typename SFINAE = void> class JSONSerializer =
+         adl_serializer>
+class basic_json;
+
+/*!
+@brief JSON Pointer
+
+A JSON pointer defines a string syntax for identifying a specific value
+within a JSON document. It can be used with functions `at` and
+`operator[]`. Furthermore, JSON pointers are the base for JSON patches.
+
+@sa [RFC 6901](https://tools.ietf.org/html/rfc6901)
+
+@since version 2.0.0
+*/
+template<typename BasicJsonType>
+class json_pointer;
+
+/*!
+@brief default JSON class
+
+This type is the default specialization of the @ref basic_json class which
+uses the standard template types.
+
+@since version 1.0.0
+*/
+using json = basic_json<>;
+}  // namespace nlohmann
+
+#endif
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+
+// This file contains all internal macro definitions
+// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them
+
 // exclude unsupported compilers
-#if defined(__clang__)
-    #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400
-        #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers"
-    #endif
-#elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER))
-    #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40900
-        #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers"
+#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK)
+    #if defined(__clang__)
+        #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400
+            #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers"
+        #endif
+    #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER))
+        #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800
+            #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers"
+        #endif
     #endif
 #endif
 
@@ -90,14 +153,36 @@ SOFTWARE.
 #endif
 
 // allow to disable exceptions
-#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && not defined(JSON_NOEXCEPTION)
+#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION)
     #define JSON_THROW(exception) throw exception
     #define JSON_TRY try
     #define JSON_CATCH(exception) catch(exception)
+    #define JSON_INTERNAL_CATCH(exception) catch(exception)
 #else
     #define JSON_THROW(exception) std::abort()
     #define JSON_TRY if(true)
     #define JSON_CATCH(exception) if(false)
+    #define JSON_INTERNAL_CATCH(exception) if(false)
+#endif
+
+// override exception macros
+#if defined(JSON_THROW_USER)
+    #undef JSON_THROW
+    #define JSON_THROW JSON_THROW_USER
+#endif
+#if defined(JSON_TRY_USER)
+    #undef JSON_TRY
+    #define JSON_TRY JSON_TRY_USER
+#endif
+#if defined(JSON_CATCH_USER)
+    #undef JSON_CATCH
+    #define JSON_CATCH JSON_CATCH_USER
+    #undef JSON_INTERNAL_CATCH
+    #define JSON_INTERNAL_CATCH JSON_CATCH_USER
+#endif
+#if defined(JSON_INTERNAL_CATCH_USER)
+    #undef JSON_INTERNAL_CATCH
+    #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER
 #endif
 
 // manual branch prediction
@@ -118,25 +203,35 @@ SOFTWARE.
 #endif
 
 /*!
-@brief namespace for Niels Lohmann
-@see https://github.com/nlohmann
-@since version 1.0.0
+@brief macro to briefly define a mapping between an enum and JSON
+@def NLOHMANN_JSON_SERIALIZE_ENUM
+@since version 3.4.0
 */
-namespace nlohmann
-{
-template<typename = void, typename = void>
-struct adl_serializer;
-
-// forward declaration of basic_json (required to split the class)
-template<template<typename, typename, typename...> class ObjectType = std::map,
-         template<typename, typename...> class ArrayType = std::vector,
-         class StringType = std::string, class BooleanType = bool,
-         class NumberIntegerType = std::int64_t,
-         class NumberUnsignedType = std::uint64_t,
-         class NumberFloatType = double,
-         template<typename> class AllocatorType = std::allocator,
-         template<typename, typename = void> class JSONSerializer = adl_serializer>
-class basic_json;
+#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...)                                           \
+    template<typename BasicJsonType>                                                           \
+    inline void to_json(BasicJsonType& j, const ENUM_TYPE& e)                                  \
+    {                                                                                          \
+        static_assert(std::is_enum<ENUM_TYPE>::value, #ENUM_TYPE " must be an enum!");         \
+        static const std::pair<ENUM_TYPE, BasicJsonType> m[] = __VA_ARGS__;                    \
+        auto it = std::find_if(std::begin(m), std::end(m),                                     \
+                               [e](const std::pair<ENUM_TYPE, BasicJsonType>& ej_pair) -> bool \
+        {                                                                                      \
+            return ej_pair.first == e;                                                         \
+        });                                                                                    \
+        j = ((it != std::end(m)) ? it : std::begin(m))->second;                                \
+    }                                                                                          \
+    template<typename BasicJsonType>                                                           \
+    inline void from_json(const BasicJsonType& j, ENUM_TYPE& e)                                \
+    {                                                                                          \
+        static_assert(std::is_enum<ENUM_TYPE>::value, #ENUM_TYPE " must be an enum!");         \
+        static const std::pair<ENUM_TYPE, BasicJsonType> m[] = __VA_ARGS__;                    \
+        auto it = std::find_if(std::begin(m), std::end(m),                                     \
+                               [j](const std::pair<ENUM_TYPE, BasicJsonType>& ej_pair) -> bool \
+        {                                                                                      \
+            return ej_pair.second == j;                                                        \
+        });                                                                                    \
+        e = ((it != std::end(m)) ? it : std::begin(m))->first;                                 \
+    }
 
 // Ugly macros to avoid uglier copy-paste when specializing basic_json. They
 // may be removed in the future once the class is split.
@@ -154,17 +249,590 @@ class basic_json;
     NumberIntegerType, NumberUnsignedType, NumberFloatType,                \
     AllocatorType, JSONSerializer>
 
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+
+#include <ciso646> // not
+#include <cstddef> // size_t
+#include <type_traits> // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type
+
+namespace nlohmann
+{
+namespace detail
+{
+// alias templates to reduce boilerplate
+template<bool B, typename T = void>
+using enable_if_t = typename std::enable_if<B, T>::type;
+
+template<typename T>
+using uncvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type;
+
+// implementation of C++14 index_sequence and affiliates
+// source: https://stackoverflow.com/a/32223343
+template<std::size_t... Ints>
+struct index_sequence
+{
+    using type = index_sequence;
+    using value_type = std::size_t;
+    static constexpr std::size_t size() noexcept
+    {
+        return sizeof...(Ints);
+    }
+};
+
+template<class Sequence1, class Sequence2>
+struct merge_and_renumber;
+
+template<std::size_t... I1, std::size_t... I2>
+struct merge_and_renumber<index_sequence<I1...>, index_sequence<I2...>>
+        : index_sequence < I1..., (sizeof...(I1) + I2)... > {};
+
+template<std::size_t N>
+struct make_index_sequence
+    : merge_and_renumber < typename make_index_sequence < N / 2 >::type,
+      typename make_index_sequence < N - N / 2 >::type > {};
+
+template<> struct make_index_sequence<0> : index_sequence<> {};
+template<> struct make_index_sequence<1> : index_sequence<0> {};
+
+template<typename... Ts>
+using index_sequence_for = make_index_sequence<sizeof...(Ts)>;
+
+// dispatch utility (taken from ranges-v3)
+template<unsigned N> struct priority_tag : priority_tag < N - 1 > {};
+template<> struct priority_tag<0> {};
+
+// taken from ranges-v3
+template<typename T>
+struct static_const
+{
+    static constexpr T value{};
+};
+
+template<typename T>
+constexpr T static_const<T>::value;
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+
+#include <ciso646> // not
+#include <limits> // numeric_limits
+#include <type_traits> // false_type, is_constructible, is_integral, is_same, true_type
+#include <utility> // declval
+
+// #include <nlohmann/json_fwd.hpp>
+
+// #include <nlohmann/detail/iterators/iterator_traits.hpp>
+
+
+#include <iterator> // random_access_iterator_tag
+
+// #include <nlohmann/detail/meta/void_t.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+template <typename ...Ts> struct make_void
+{
+    using type = void;
+};
+template <typename ...Ts> using void_t = typename make_void<Ts...>::type;
+} // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
 
+namespace nlohmann
+{
+namespace detail
+{
+template <typename It, typename = void>
+struct iterator_types {};
+
+template <typename It>
+struct iterator_types <
+    It,
+    void_t<typename It::difference_type, typename It::value_type, typename It::pointer,
+    typename It::reference, typename It::iterator_category >>
+{
+    using difference_type = typename It::difference_type;
+    using value_type = typename It::value_type;
+    using pointer = typename It::pointer;
+    using reference = typename It::reference;
+    using iterator_category = typename It::iterator_category;
+};
+
+// This is required as some compilers implement std::iterator_traits in a way that
+// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341.
+template <typename T, typename = void>
+struct iterator_traits
+{
+};
+
+template <typename T>
+struct iterator_traits < T, enable_if_t < !std::is_pointer<T>::value >>
+            : iterator_types<T>
+{
+};
+
+template <typename T>
+struct iterator_traits<T*, enable_if_t<std::is_object<T>::value>>
+{
+    using iterator_category = std::random_access_iterator_tag;
+    using value_type = T;
+    using difference_type = ptrdiff_t;
+    using pointer = T*;
+    using reference = T&;
+};
+}
+}
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+// #include <nlohmann/detail/meta/detected.hpp>
+
+
+#include <type_traits>
+
+// #include <nlohmann/detail/meta/void_t.hpp>
+
+
+// http://en.cppreference.com/w/cpp/experimental/is_detected
+namespace nlohmann
+{
+namespace detail
+{
+struct nonesuch
+{
+    nonesuch() = delete;
+    ~nonesuch() = delete;
+    nonesuch(nonesuch const&) = delete;
+    void operator=(nonesuch const&) = delete;
+};
+
+template <class Default,
+          class AlwaysVoid,
+          template <class...> class Op,
+          class... Args>
+struct detector
+{
+    using value_t = std::false_type;
+    using type = Default;
+};
+
+template <class Default, template <class...> class Op, class... Args>
+struct detector<Default, void_t<Op<Args...>>, Op, Args...>
+{
+    using value_t = std::true_type;
+    using type = Op<Args...>;
+};
+
+template <template <class...> class Op, class... Args>
+using is_detected = typename detector<nonesuch, void, Op, Args...>::value_t;
+
+template <template <class...> class Op, class... Args>
+using detected_t = typename detector<nonesuch, void, Op, Args...>::type;
+
+template <class Default, template <class...> class Op, class... Args>
+using detected_or = detector<Default, void, Op, Args...>;
+
+template <class Default, template <class...> class Op, class... Args>
+using detected_or_t = typename detected_or<Default, Op, Args...>::type;
+
+template <class Expected, template <class...> class Op, class... Args>
+using is_detected_exact = std::is_same<Expected, detected_t<Op, Args...>>;
+
+template <class To, template <class...> class Op, class... Args>
+using is_detected_convertible =
+    std::is_convertible<detected_t<Op, Args...>, To>;
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+
+namespace nlohmann
+{
 /*!
-@brief unnamed namespace with internal helper functions
+@brief detail namespace with internal helper functions
 
-This namespace collects some functions that could not be defined inside the
-@ref basic_json class.
+This namespace collects functions that should not be exposed,
+implementations of some @ref basic_json methods, and meta-programming helpers.
 
 @since version 2.1.0
 */
 namespace detail
 {
+/////////////
+// helpers //
+/////////////
+
+// Note to maintainers:
+//
+// Every trait in this file expects a non CV-qualified type.
+// The only exceptions are in the 'aliases for detected' section
+// (i.e. those of the form: decltype(T::member_function(std::declval<T>())))
+//
+// In this case, T has to be properly CV-qualified to constraint the function arguments
+// (e.g. to_json(BasicJsonType&, const T&))
+
+template<typename> struct is_basic_json : std::false_type {};
+
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+struct is_basic_json<NLOHMANN_BASIC_JSON_TPL> : std::true_type {};
+
+//////////////////////////
+// aliases for detected //
+//////////////////////////
+
+template <typename T>
+using mapped_type_t = typename T::mapped_type;
+
+template <typename T>
+using key_type_t = typename T::key_type;
+
+template <typename T>
+using value_type_t = typename T::value_type;
+
+template <typename T>
+using difference_type_t = typename T::difference_type;
+
+template <typename T>
+using pointer_t = typename T::pointer;
+
+template <typename T>
+using reference_t = typename T::reference;
+
+template <typename T>
+using iterator_category_t = typename T::iterator_category;
+
+template <typename T>
+using iterator_t = typename T::iterator;
+
+template <typename T, typename... Args>
+using to_json_function = decltype(T::to_json(std::declval<Args>()...));
+
+template <typename T, typename... Args>
+using from_json_function = decltype(T::from_json(std::declval<Args>()...));
+
+template <typename T, typename U>
+using get_template_function = decltype(std::declval<T>().template get<U>());
+
+// trait checking if JSONSerializer<T>::from_json(json const&, udt&) exists
+template <typename BasicJsonType, typename T, typename = void>
+struct has_from_json : std::false_type {};
+
+template <typename BasicJsonType, typename T>
+struct has_from_json<BasicJsonType, T,
+           enable_if_t<not is_basic_json<T>::value>>
+{
+    using serializer = typename BasicJsonType::template json_serializer<T, void>;
+
+    static constexpr bool value =
+        is_detected_exact<void, from_json_function, serializer,
+        const BasicJsonType&, T&>::value;
+};
+
+// This trait checks if JSONSerializer<T>::from_json(json const&) exists
+// this overload is used for non-default-constructible user-defined-types
+template <typename BasicJsonType, typename T, typename = void>
+struct has_non_default_from_json : std::false_type {};
+
+template<typename BasicJsonType, typename T>
+struct has_non_default_from_json<BasicJsonType, T, enable_if_t<not is_basic_json<T>::value>>
+{
+    using serializer = typename BasicJsonType::template json_serializer<T, void>;
+
+    static constexpr bool value =
+        is_detected_exact<T, from_json_function, serializer,
+        const BasicJsonType&>::value;
+};
+
+// This trait checks if BasicJsonType::json_serializer<T>::to_json exists
+// Do not evaluate the trait when T is a basic_json type, to avoid template instantiation infinite recursion.
+template <typename BasicJsonType, typename T, typename = void>
+struct has_to_json : std::false_type {};
+
+template <typename BasicJsonType, typename T>
+struct has_to_json<BasicJsonType, T, enable_if_t<not is_basic_json<T>::value>>
+{
+    using serializer = typename BasicJsonType::template json_serializer<T, void>;
+
+    static constexpr bool value =
+        is_detected_exact<void, to_json_function, serializer, BasicJsonType&,
+        T>::value;
+};
+
+
+///////////////////
+// is_ functions //
+///////////////////
+
+template <typename T, typename = void>
+struct is_iterator_traits : std::false_type {};
+
+template <typename T>
+struct is_iterator_traits<iterator_traits<T>>
+{
+  private:
+    using traits = iterator_traits<T>;
+
+  public:
+    static constexpr auto value =
+        is_detected<value_type_t, traits>::value &&
+        is_detected<difference_type_t, traits>::value &&
+        is_detected<pointer_t, traits>::value &&
+        is_detected<iterator_category_t, traits>::value &&
+        is_detected<reference_t, traits>::value;
+};
+
+// source: https://stackoverflow.com/a/37193089/4116453
+
+template <typename T, typename = void>
+struct is_complete_type : std::false_type {};
+
+template <typename T>
+struct is_complete_type<T, decltype(void(sizeof(T)))> : std::true_type {};
+
+template <typename BasicJsonType, typename CompatibleObjectType,
+          typename = void>
+struct is_compatible_object_type_impl : std::false_type {};
+
+template <typename BasicJsonType, typename CompatibleObjectType>
+struct is_compatible_object_type_impl <
+    BasicJsonType, CompatibleObjectType,
+    enable_if_t<is_detected<mapped_type_t, CompatibleObjectType>::value and
+    is_detected<key_type_t, CompatibleObjectType>::value >>
+{
+
+    using object_t = typename BasicJsonType::object_t;
+
+    // macOS's is_constructible does not play well with nonesuch...
+    static constexpr bool value =
+        std::is_constructible<typename object_t::key_type,
+        typename CompatibleObjectType::key_type>::value and
+        std::is_constructible<typename object_t::mapped_type,
+        typename CompatibleObjectType::mapped_type>::value;
+};
+
+template <typename BasicJsonType, typename CompatibleObjectType>
+struct is_compatible_object_type
+    : is_compatible_object_type_impl<BasicJsonType, CompatibleObjectType> {};
+
+template <typename BasicJsonType, typename ConstructibleObjectType,
+          typename = void>
+struct is_constructible_object_type_impl : std::false_type {};
+
+template <typename BasicJsonType, typename ConstructibleObjectType>
+struct is_constructible_object_type_impl <
+    BasicJsonType, ConstructibleObjectType,
+    enable_if_t<is_detected<mapped_type_t, ConstructibleObjectType>::value and
+    is_detected<key_type_t, ConstructibleObjectType>::value >>
+{
+    using object_t = typename BasicJsonType::object_t;
+
+    static constexpr bool value =
+        (std::is_constructible<typename ConstructibleObjectType::key_type, typename object_t::key_type>::value and
+         std::is_same<typename object_t::mapped_type, typename ConstructibleObjectType::mapped_type>::value) or
+        (has_from_json<BasicJsonType, typename ConstructibleObjectType::mapped_type>::value or
+         has_non_default_from_json<BasicJsonType, typename ConstructibleObjectType::mapped_type >::value);
+};
+
+template <typename BasicJsonType, typename ConstructibleObjectType>
+struct is_constructible_object_type
+    : is_constructible_object_type_impl<BasicJsonType,
+      ConstructibleObjectType> {};
+
+template <typename BasicJsonType, typename CompatibleStringType,
+          typename = void>
+struct is_compatible_string_type_impl : std::false_type {};
+
+template <typename BasicJsonType, typename CompatibleStringType>
+struct is_compatible_string_type_impl <
+    BasicJsonType, CompatibleStringType,
+    enable_if_t<is_detected_exact<typename BasicJsonType::string_t::value_type,
+    value_type_t, CompatibleStringType>::value >>
+{
+    static constexpr auto value =
+        std::is_constructible<typename BasicJsonType::string_t, CompatibleStringType>::value;
+};
+
+template <typename BasicJsonType, typename ConstructibleStringType>
+struct is_compatible_string_type
+    : is_compatible_string_type_impl<BasicJsonType, ConstructibleStringType> {};
+
+template <typename BasicJsonType, typename ConstructibleStringType,
+          typename = void>
+struct is_constructible_string_type_impl : std::false_type {};
+
+template <typename BasicJsonType, typename ConstructibleStringType>
+struct is_constructible_string_type_impl <
+    BasicJsonType, ConstructibleStringType,
+    enable_if_t<is_detected_exact<typename BasicJsonType::string_t::value_type,
+    value_type_t, ConstructibleStringType>::value >>
+{
+    static constexpr auto value =
+        std::is_constructible<ConstructibleStringType,
+        typename BasicJsonType::string_t>::value;
+};
+
+template <typename BasicJsonType, typename ConstructibleStringType>
+struct is_constructible_string_type
+    : is_constructible_string_type_impl<BasicJsonType, ConstructibleStringType> {};
+
+template <typename BasicJsonType, typename CompatibleArrayType, typename = void>
+struct is_compatible_array_type_impl : std::false_type {};
+
+template <typename BasicJsonType, typename CompatibleArrayType>
+struct is_compatible_array_type_impl <
+    BasicJsonType, CompatibleArrayType,
+    enable_if_t<is_detected<value_type_t, CompatibleArrayType>::value and
+    is_detected<iterator_t, CompatibleArrayType>::value and
+// This is needed because json_reverse_iterator has a ::iterator type...
+// Therefore it is detected as a CompatibleArrayType.
+// The real fix would be to have an Iterable concept.
+    not is_iterator_traits<
+    iterator_traits<CompatibleArrayType>>::value >>
+{
+    static constexpr bool value =
+        std::is_constructible<BasicJsonType,
+        typename CompatibleArrayType::value_type>::value;
+};
+
+template <typename BasicJsonType, typename CompatibleArrayType>
+struct is_compatible_array_type
+    : is_compatible_array_type_impl<BasicJsonType, CompatibleArrayType> {};
+
+template <typename BasicJsonType, typename ConstructibleArrayType, typename = void>
+struct is_constructible_array_type_impl : std::false_type {};
+
+template <typename BasicJsonType, typename ConstructibleArrayType>
+struct is_constructible_array_type_impl <
+    BasicJsonType, ConstructibleArrayType,
+    enable_if_t<std::is_same<ConstructibleArrayType,
+    typename BasicJsonType::value_type>::value >>
+            : std::true_type {};
+
+template <typename BasicJsonType, typename ConstructibleArrayType>
+struct is_constructible_array_type_impl <
+    BasicJsonType, ConstructibleArrayType,
+    enable_if_t<not std::is_same<ConstructibleArrayType,
+    typename BasicJsonType::value_type>::value and
+    is_detected<value_type_t, ConstructibleArrayType>::value and
+    is_detected<iterator_t, ConstructibleArrayType>::value and
+    is_complete_type<
+    detected_t<value_type_t, ConstructibleArrayType>>::value >>
+{
+    static constexpr bool value =
+        // This is needed because json_reverse_iterator has a ::iterator type,
+        // furthermore, std::back_insert_iterator (and other iterators) have a base class `iterator`...
+        // Therefore it is detected as a ConstructibleArrayType.
+        // The real fix would be to have an Iterable concept.
+        not is_iterator_traits <
+        iterator_traits<ConstructibleArrayType >>::value and
+
+        (std::is_same<typename ConstructibleArrayType::value_type, typename BasicJsonType::array_t::value_type>::value or
+         has_from_json<BasicJsonType,
+         typename ConstructibleArrayType::value_type>::value or
+         has_non_default_from_json <
+         BasicJsonType, typename ConstructibleArrayType::value_type >::value);
+};
+
+template <typename BasicJsonType, typename ConstructibleArrayType>
+struct is_constructible_array_type
+    : is_constructible_array_type_impl<BasicJsonType, ConstructibleArrayType> {};
+
+template <typename RealIntegerType, typename CompatibleNumberIntegerType,
+          typename = void>
+struct is_compatible_integer_type_impl : std::false_type {};
+
+template <typename RealIntegerType, typename CompatibleNumberIntegerType>
+struct is_compatible_integer_type_impl <
+    RealIntegerType, CompatibleNumberIntegerType,
+    enable_if_t<std::is_integral<RealIntegerType>::value and
+    std::is_integral<CompatibleNumberIntegerType>::value and
+    not std::is_same<bool, CompatibleNumberIntegerType>::value >>
+{
+    // is there an assert somewhere on overflows?
+    using RealLimits = std::numeric_limits<RealIntegerType>;
+    using CompatibleLimits = std::numeric_limits<CompatibleNumberIntegerType>;
+
+    static constexpr auto value =
+        std::is_constructible<RealIntegerType,
+        CompatibleNumberIntegerType>::value and
+        CompatibleLimits::is_integer and
+        RealLimits::is_signed == CompatibleLimits::is_signed;
+};
+
+template <typename RealIntegerType, typename CompatibleNumberIntegerType>
+struct is_compatible_integer_type
+    : is_compatible_integer_type_impl<RealIntegerType,
+      CompatibleNumberIntegerType> {};
+
+template <typename BasicJsonType, typename CompatibleType, typename = void>
+struct is_compatible_type_impl: std::false_type {};
+
+template <typename BasicJsonType, typename CompatibleType>
+struct is_compatible_type_impl <
+    BasicJsonType, CompatibleType,
+    enable_if_t<is_complete_type<CompatibleType>::value >>
+{
+    static constexpr bool value =
+        has_to_json<BasicJsonType, CompatibleType>::value;
+};
+
+template <typename BasicJsonType, typename CompatibleType>
+struct is_compatible_type
+    : is_compatible_type_impl<BasicJsonType, CompatibleType> {};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+
+#include <exception> // exception
+#include <stdexcept> // runtime_error
+#include <string> // to_string
+
+// #include <nlohmann/detail/input/position_t.hpp>
+
+
+#include <cstddef> // size_t
+
+namespace nlohmann
+{
+namespace detail
+{
+/// struct to capture the start position of the current token
+struct position_t
+{
+    /// the total number of characters read
+    std::size_t chars_read_total = 0;
+    /// the number of characters read in the current line
+    std::size_t chars_read_current_line = 0;
+    /// the number of lines read
+    std::size_t lines_read = 0;
+
+    /// conversion to size_t to preserve SAX interface
+    constexpr operator size_t() const
+    {
+        return chars_read_total;
+    }
+};
+
+}
+}
+
+
+namespace nlohmann
+{
+namespace detail
+{
 ////////////////
 // exceptions //
 ////////////////
@@ -248,6 +916,7 @@ json.exception.parse_error.109 | parse error: array index 'one' is not a number
 json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read.
 json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read.
 json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read.
+json.exception.parse_error.114 | parse error: Unsupported BSON record type 0x0F | The parsing of the corresponding BSON record type is not implemented (yet).
 
 @note For an input with n bytes, 1 is the index of the first character and n+1
       is the index of the terminating null byte or the end of file. This also
@@ -271,15 +940,23 @@ class parse_error : public exception
     /*!
     @brief create a parse error exception
     @param[in] id_       the id of the exception
-    @param[in] byte_     the byte index where the error occurred (or 0 if the
-                         position cannot be determined)
+    @param[in] position  the position where the error occurred (or with
+                         chars_read_total=0 if the position cannot be
+                         determined)
     @param[in] what_arg  the explanatory string
     @return parse_error object
     */
+    static parse_error create(int id_, const position_t& pos, const std::string& what_arg)
+    {
+        std::string w = exception::name("parse_error", id_) + "parse error" +
+                        position_string(pos) + ": " + what_arg;
+        return parse_error(id_, pos.chars_read_total, w.c_str());
+    }
+
     static parse_error create(int id_, std::size_t byte_, const std::string& what_arg)
     {
         std::string w = exception::name("parse_error", id_) + "parse error" +
-                        (byte_ != 0 ? (" at " + std::to_string(byte_)) : "") +
+                        (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") +
                         ": " + what_arg;
         return parse_error(id_, byte_, w.c_str());
     }
@@ -298,6 +975,12 @@ class parse_error : public exception
   private:
     parse_error(int id_, std::size_t byte_, const char* what_arg)
         : exception(id_, what_arg), byte(byte_) {}
+
+    static std::string position_string(const position_t& pos)
+    {
+        return " at line " + std::to_string(pos.lines_read + 1) +
+               ", column " + std::to_string(pos.chars_read_current_line);
+    }
 };
 
 /*!
@@ -377,6 +1060,7 @@ json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten
 json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers.
 json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive.
 json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. |
+json.exception.type_error.317 | JSON value cannot be serialized to requested format | The dynamic type of the object cannot be represented in the requested serialization format (e.g. a raw `true` or `null` JSON object cannot be serialized to BSON) |
 
 @liveexample{The following code shows how a `type_error` exception can be
 caught.,type_error}
@@ -419,6 +1103,9 @@ json.exception.out_of_range.403 | key 'foo' not found | The provided key was not
 json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved.
 json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value.
 json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF.
+json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. |
+json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. |
+json.exception.out_of_range.409 | BSON key cannot contain code point U+0000 (at byte 2) | Key identifiers to be serialized to BSON cannot contain code point U+0000, since the key is stored as zero-terminated c-string |
 
 @liveexample{The following code shows how an `out_of_range` exception can be
 caught.,out_of_range}
@@ -481,9 +1168,21 @@ class other_error : public exception
   private:
     other_error(int id_, const char* what_arg) : exception(id_, what_arg) {}
 };
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/value_t.hpp>
 
 
+#include <array> // array
+#include <ciso646> // and
+#include <cstddef> // size_t
+#include <cstdint> // uint8_t
 
+namespace nlohmann
+{
+namespace detail
+{
 ///////////////////////////
 // JSON type enumeration //
 ///////////////////////////
@@ -512,7 +1211,7 @@ value with the default value for a given type
 
 @since version 1.0.0
 */
-enum class value_t : uint8_t
+enum class value_t : std::uint8_t
 {
     null,             ///< null value
     object,           ///< object (unordered set of name/value pairs)
@@ -537,7 +1236,7 @@ Returns an ordering that is similar to Python:
 */
 inline bool operator<(const value_t lhs, const value_t rhs) noexcept
 {
-    static constexpr std::array<uint8_t, 8> order = {{
+    static constexpr std::array<std::uint8_t, 8> order = {{
             0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */,
             1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */
         }
@@ -547,80 +1246,573 @@ inline bool operator<(const value_t lhs, const value_t rhs) noexcept
     const auto r_index = static_cast<std::size_t>(rhs);
     return l_index < order.size() and r_index < order.size() and order[l_index] < order[r_index];
 }
+}  // namespace detail
+}  // namespace nlohmann
 
+// #include <nlohmann/detail/conversions/from_json.hpp>
 
-/////////////
-// helpers //
-/////////////
 
-template<typename> struct is_basic_json : std::false_type {};
+#include <algorithm> // transform
+#include <array> // array
+#include <ciso646> // and, not
+#include <forward_list> // forward_list
+#include <iterator> // inserter, front_inserter, end
+#include <map> // map
+#include <string> // string
+#include <tuple> // tuple, make_tuple
+#include <type_traits> // is_arithmetic, is_same, is_enum, underlying_type, is_convertible
+#include <unordered_map> // unordered_map
+#include <utility> // pair, declval
+#include <valarray> // valarray
 
-NLOHMANN_BASIC_JSON_TPL_DECLARATION
-struct is_basic_json<NLOHMANN_BASIC_JSON_TPL> : std::true_type {};
+// #include <nlohmann/detail/exceptions.hpp>
 
-// alias templates to reduce boilerplate
-template<bool B, typename T = void>
-using enable_if_t = typename std::enable_if<B, T>::type;
+// #include <nlohmann/detail/macro_scope.hpp>
 
-template<typename T>
-using uncvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type;
+// #include <nlohmann/detail/meta/cpp_future.hpp>
 
-// implementation of C++14 index_sequence and affiliates
-// source: https://stackoverflow.com/a/32223343
-template<std::size_t... Ints>
-struct index_sequence
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+namespace nlohmann
 {
-    using type = index_sequence;
-    using value_type = std::size_t;
-    static constexpr std::size_t size() noexcept
+namespace detail
+{
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename std::nullptr_t& n)
+{
+    if (JSON_UNLIKELY(not j.is_null()))
     {
-        return sizeof...(Ints);
+        JSON_THROW(type_error::create(302, "type must be null, but is " + std::string(j.type_name())));
+    }
+    n = nullptr;
+}
+
+// overloads for basic_json template parameters
+template<typename BasicJsonType, typename ArithmeticType,
+         enable_if_t<std::is_arithmetic<ArithmeticType>::value and
+                     not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value,
+                     int> = 0>
+void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val)
+{
+    switch (static_cast<value_t>(j))
+    {
+        case value_t::number_unsigned:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>());
+            break;
+        }
+        case value_t::number_integer:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>());
+            break;
+        }
+        case value_t::number_float:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>());
+            break;
+        }
+
+        default:
+            JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name())));
+    }
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b)
+{
+    if (JSON_UNLIKELY(not j.is_boolean()))
+    {
+        JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(j.type_name())));
+    }
+    b = *j.template get_ptr<const typename BasicJsonType::boolean_t*>();
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s)
+{
+    if (JSON_UNLIKELY(not j.is_string()))
+    {
+        JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name())));
+    }
+    s = *j.template get_ptr<const typename BasicJsonType::string_t*>();
+}
+
+template <
+    typename BasicJsonType, typename ConstructibleStringType,
+    enable_if_t <
+        is_constructible_string_type<BasicJsonType, ConstructibleStringType>::value and
+        not std::is_same<typename BasicJsonType::string_t,
+                         ConstructibleStringType>::value,
+        int > = 0 >
+void from_json(const BasicJsonType& j, ConstructibleStringType& s)
+{
+    if (JSON_UNLIKELY(not j.is_string()))
+    {
+        JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name())));
+    }
+
+    s = *j.template get_ptr<const typename BasicJsonType::string_t*>();
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val)
+{
+    get_arithmetic_value(j, val);
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val)
+{
+    get_arithmetic_value(j, val);
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val)
+{
+    get_arithmetic_value(j, val);
+}
+
+template<typename BasicJsonType, typename EnumType,
+         enable_if_t<std::is_enum<EnumType>::value, int> = 0>
+void from_json(const BasicJsonType& j, EnumType& e)
+{
+    typename std::underlying_type<EnumType>::type val;
+    get_arithmetic_value(j, val);
+    e = static_cast<EnumType>(val);
+}
+
+// forward_list doesn't have an insert method
+template<typename BasicJsonType, typename T, typename Allocator,
+         enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0>
+void from_json(const BasicJsonType& j, std::forward_list<T, Allocator>& l)
+{
+    if (JSON_UNLIKELY(not j.is_array()))
+    {
+        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
+    }
+    std::transform(j.rbegin(), j.rend(),
+                   std::front_inserter(l), [](const BasicJsonType & i)
+    {
+        return i.template get<T>();
+    });
+}
+
+// valarray doesn't have an insert method
+template<typename BasicJsonType, typename T,
+         enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0>
+void from_json(const BasicJsonType& j, std::valarray<T>& l)
+{
+    if (JSON_UNLIKELY(not j.is_array()))
+    {
+        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
+    }
+    l.resize(j.size());
+    std::copy(j.m_value.array->begin(), j.m_value.array->end(), std::begin(l));
+}
+
+template<typename BasicJsonType>
+void from_json_array_impl(const BasicJsonType& j, typename BasicJsonType::array_t& arr, priority_tag<3> /*unused*/)
+{
+    arr = *j.template get_ptr<const typename BasicJsonType::array_t*>();
+}
+
+template <typename BasicJsonType, typename T, std::size_t N>
+auto from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr,
+                          priority_tag<2> /*unused*/)
+-> decltype(j.template get<T>(), void())
+{
+    for (std::size_t i = 0; i < N; ++i)
+    {
+        arr[i] = j.at(i).template get<T>();
+    }
+}
+
+template<typename BasicJsonType, typename ConstructibleArrayType>
+auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, priority_tag<1> /*unused*/)
+-> decltype(
+    arr.reserve(std::declval<typename ConstructibleArrayType::size_type>()),
+    j.template get<typename ConstructibleArrayType::value_type>(),
+    void())
+{
+    using std::end;
+
+    arr.reserve(j.size());
+    std::transform(j.begin(), j.end(),
+                   std::inserter(arr, end(arr)), [](const BasicJsonType & i)
+    {
+        // get<BasicJsonType>() returns *this, this won't call a from_json
+        // method when value_type is BasicJsonType
+        return i.template get<typename ConstructibleArrayType::value_type>();
+    });
+}
+
+template <typename BasicJsonType, typename ConstructibleArrayType>
+void from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr,
+                          priority_tag<0> /*unused*/)
+{
+    using std::end;
+
+    std::transform(
+        j.begin(), j.end(), std::inserter(arr, end(arr)),
+        [](const BasicJsonType & i)
+    {
+        // get<BasicJsonType>() returns *this, this won't call a from_json
+        // method when value_type is BasicJsonType
+        return i.template get<typename ConstructibleArrayType::value_type>();
+    });
+}
+
+template <typename BasicJsonType, typename ConstructibleArrayType,
+          enable_if_t <
+              is_constructible_array_type<BasicJsonType, ConstructibleArrayType>::value and
+              not is_constructible_object_type<BasicJsonType, ConstructibleArrayType>::value and
+              not is_constructible_string_type<BasicJsonType, ConstructibleArrayType>::value and
+              not is_basic_json<ConstructibleArrayType>::value,
+              int > = 0 >
+
+auto from_json(const BasicJsonType& j, ConstructibleArrayType& arr)
+-> decltype(from_json_array_impl(j, arr, priority_tag<3> {}),
+j.template get<typename ConstructibleArrayType::value_type>(),
+void())
+{
+    if (JSON_UNLIKELY(not j.is_array()))
+    {
+        JSON_THROW(type_error::create(302, "type must be array, but is " +
+                                      std::string(j.type_name())));
+    }
+
+    from_json_array_impl(j, arr, priority_tag<3> {});
+}
+
+template<typename BasicJsonType, typename ConstructibleObjectType,
+         enable_if_t<is_constructible_object_type<BasicJsonType, ConstructibleObjectType>::value, int> = 0>
+void from_json(const BasicJsonType& j, ConstructibleObjectType& obj)
+{
+    if (JSON_UNLIKELY(not j.is_object()))
+    {
+        JSON_THROW(type_error::create(302, "type must be object, but is " + std::string(j.type_name())));
+    }
+
+    auto inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();
+    using value_type = typename ConstructibleObjectType::value_type;
+    std::transform(
+        inner_object->begin(), inner_object->end(),
+        std::inserter(obj, obj.begin()),
+        [](typename BasicJsonType::object_t::value_type const & p)
+    {
+        return value_type(p.first, p.second.template get<typename ConstructibleObjectType::mapped_type>());
+    });
+}
+
+// overload for arithmetic types, not chosen for basic_json template arguments
+// (BooleanType, etc..); note: Is it really necessary to provide explicit
+// overloads for boolean_t etc. in case of a custom BooleanType which is not
+// an arithmetic type?
+template<typename BasicJsonType, typename ArithmeticType,
+         enable_if_t <
+             std::is_arithmetic<ArithmeticType>::value and
+             not std::is_same<ArithmeticType, typename BasicJsonType::number_unsigned_t>::value and
+             not std::is_same<ArithmeticType, typename BasicJsonType::number_integer_t>::value and
+             not std::is_same<ArithmeticType, typename BasicJsonType::number_float_t>::value and
+             not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value,
+             int> = 0>
+void from_json(const BasicJsonType& j, ArithmeticType& val)
+{
+    switch (static_cast<value_t>(j))
+    {
+        case value_t::number_unsigned:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>());
+            break;
+        }
+        case value_t::number_integer:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>());
+            break;
+        }
+        case value_t::number_float:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>());
+            break;
+        }
+        case value_t::boolean:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::boolean_t*>());
+            break;
+        }
+
+        default:
+            JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name())));
+    }
+}
+
+template<typename BasicJsonType, typename A1, typename A2>
+void from_json(const BasicJsonType& j, std::pair<A1, A2>& p)
+{
+    p = {j.at(0).template get<A1>(), j.at(1).template get<A2>()};
+}
+
+template<typename BasicJsonType, typename Tuple, std::size_t... Idx>
+void from_json_tuple_impl(const BasicJsonType& j, Tuple& t, index_sequence<Idx...> /*unused*/)
+{
+    t = std::make_tuple(j.at(Idx).template get<typename std::tuple_element<Idx, Tuple>::type>()...);
+}
+
+template<typename BasicJsonType, typename... Args>
+void from_json(const BasicJsonType& j, std::tuple<Args...>& t)
+{
+    from_json_tuple_impl(j, t, index_sequence_for<Args...> {});
+}
+
+template <typename BasicJsonType, typename Key, typename Value, typename Compare, typename Allocator,
+          typename = enable_if_t<not std::is_constructible<
+                                     typename BasicJsonType::string_t, Key>::value>>
+void from_json(const BasicJsonType& j, std::map<Key, Value, Compare, Allocator>& m)
+{
+    if (JSON_UNLIKELY(not j.is_array()))
+    {
+        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
+    }
+    for (const auto& p : j)
+    {
+        if (JSON_UNLIKELY(not p.is_array()))
+        {
+            JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name())));
+        }
+        m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>());
+    }
+}
+
+template <typename BasicJsonType, typename Key, typename Value, typename Hash, typename KeyEqual, typename Allocator,
+          typename = enable_if_t<not std::is_constructible<
+                                     typename BasicJsonType::string_t, Key>::value>>
+void from_json(const BasicJsonType& j, std::unordered_map<Key, Value, Hash, KeyEqual, Allocator>& m)
+{
+    if (JSON_UNLIKELY(not j.is_array()))
+    {
+        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
+    }
+    for (const auto& p : j)
+    {
+        if (JSON_UNLIKELY(not p.is_array()))
+        {
+            JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name())));
+        }
+        m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>());
+    }
+}
+
+struct from_json_fn
+{
+    template<typename BasicJsonType, typename T>
+    auto operator()(const BasicJsonType& j, T& val) const
+    noexcept(noexcept(from_json(j, val)))
+    -> decltype(from_json(j, val), void())
+    {
+        return from_json(j, val);
     }
 };
+}  // namespace detail
 
-template<class Sequence1, class Sequence2>
-struct merge_and_renumber;
+/// namespace to hold default `from_json` function
+/// to see why this is required:
+/// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4381.html
+namespace
+{
+constexpr const auto& from_json = detail::static_const<detail::from_json_fn>::value;
+} // namespace
+}  // namespace nlohmann
 
-template<std::size_t... I1, std::size_t... I2>
-struct merge_and_renumber<index_sequence<I1...>, index_sequence<I2...>>
-        : index_sequence < I1..., (sizeof...(I1) + I2)... > {};
+// #include <nlohmann/detail/conversions/to_json.hpp>
 
-template<std::size_t N>
-struct make_index_sequence
-    : merge_and_renumber < typename make_index_sequence < N / 2 >::type,
-      typename make_index_sequence < N - N / 2 >::type > {};
 
-template<> struct make_index_sequence<0> : index_sequence<> {};
-template<> struct make_index_sequence<1> : index_sequence<0> {};
+#include <ciso646> // or, and, not
+#include <iterator> // begin, end
+#include <tuple> // tuple, get
+#include <type_traits> // is_same, is_constructible, is_floating_point, is_enum, underlying_type
+#include <utility> // move, forward, declval, pair
+#include <valarray> // valarray
+#include <vector> // vector
 
-template<typename... Ts>
-using index_sequence_for = make_index_sequence<sizeof...(Ts)>;
+// #include <nlohmann/detail/meta/cpp_future.hpp>
 
-/*
-Implementation of two C++17 constructs: conjunction, negation. This is needed
-to avoid evaluating all the traits in a condition
+// #include <nlohmann/detail/meta/type_traits.hpp>
 
-For example: not std::is_same<void, T>::value and has_value_type<T>::value
-will not compile when T = void (on MSVC at least). Whereas
-conjunction<negation<std::is_same<void, T>>, has_value_type<T>>::value will
-stop evaluating if negation<...>::value == false
+// #include <nlohmann/detail/value_t.hpp>
 
-Please note that those constructs must be used with caution, since symbols can
-become very long quickly (which can slow down compilation and cause MSVC
-internal compiler errors). Only use it when you have to (see example ahead).
-*/
-template<class...> struct conjunction : std::true_type {};
-template<class B1> struct conjunction<B1> : B1 {};
-template<class B1, class... Bn>
-struct conjunction<B1, Bn...> : std::conditional<bool(B1::value), conjunction<Bn...>, B1>::type {};
+// #include <nlohmann/detail/iterators/iteration_proxy.hpp>
 
-template<class B> struct negation : std::integral_constant<bool, not B::value> {};
 
-// dispatch utility (taken from ranges-v3)
-template<unsigned N> struct priority_tag : priority_tag < N - 1 > {};
-template<> struct priority_tag<0> {};
+#include <cstddef> // size_t
+#include <string> // string, to_string
+#include <iterator> // input_iterator_tag
+#include <tuple> // tuple_size, get, tuple_element
+
+// #include <nlohmann/detail/value_t.hpp>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+template <typename IteratorType> class iteration_proxy_value
+{
+  public:
+    using difference_type = std::ptrdiff_t;
+    using value_type = iteration_proxy_value;
+    using pointer = value_type * ;
+    using reference = value_type & ;
+    using iterator_category = std::input_iterator_tag;
+
+  private:
+    /// the iterator
+    IteratorType anchor;
+    /// an index for arrays (used to create key names)
+    std::size_t array_index = 0;
+    /// last stringified array index
+    mutable std::size_t array_index_last = 0;
+    /// a string representation of the array index
+    mutable std::string array_index_str = "0";
+    /// an empty string (to return a reference for primitive values)
+    const std::string empty_str = "";
+
+  public:
+    explicit iteration_proxy_value(IteratorType it) noexcept : anchor(it) {}
+
+    /// dereference operator (needed for range-based for)
+    iteration_proxy_value& operator*()
+    {
+        return *this;
+    }
+
+    /// increment operator (needed for range-based for)
+    iteration_proxy_value& operator++()
+    {
+        ++anchor;
+        ++array_index;
+
+        return *this;
+    }
+
+    /// equality operator (needed for InputIterator)
+    bool operator==(const iteration_proxy_value& o) const noexcept
+    {
+        return anchor == o.anchor;
+    }
+
+    /// inequality operator (needed for range-based for)
+    bool operator!=(const iteration_proxy_value& o) const noexcept
+    {
+        return anchor != o.anchor;
+    }
+
+    /// return key of the iterator
+    const std::string& key() const
+    {
+        assert(anchor.m_object != nullptr);
+
+        switch (anchor.m_object->type())
+        {
+            // use integer array index as key
+            case value_t::array:
+            {
+                if (array_index != array_index_last)
+                {
+                    array_index_str = std::to_string(array_index);
+                    array_index_last = array_index;
+                }
+                return array_index_str;
+            }
+
+            // use key from the object
+            case value_t::object:
+                return anchor.key();
+
+            // use an empty key for all primitive types
+            default:
+                return empty_str;
+        }
+    }
+
+    /// return value of the iterator
+    typename IteratorType::reference value() const
+    {
+        return anchor.value();
+    }
+};
+
+/// proxy class for the items() function
+template<typename IteratorType> class iteration_proxy
+{
+  private:
+    /// the container to iterate
+    typename IteratorType::reference container;
+
+  public:
+    /// construct iteration proxy from a container
+    explicit iteration_proxy(typename IteratorType::reference cont) noexcept
+        : container(cont) {}
+
+    /// return iterator begin (needed for range-based for)
+    iteration_proxy_value<IteratorType> begin() noexcept
+    {
+        return iteration_proxy_value<IteratorType>(container.begin());
+    }
+
+    /// return iterator end (needed for range-based for)
+    iteration_proxy_value<IteratorType> end() noexcept
+    {
+        return iteration_proxy_value<IteratorType>(container.end());
+    }
+};
+// Structured Bindings Support
+// For further reference see https://blog.tartanllama.xyz/structured-bindings/
+// And see https://github.com/nlohmann/json/pull/1391
+template <std::size_t N, typename IteratorType, enable_if_t<N == 0, int> = 0>
+auto get(const nlohmann::detail::iteration_proxy_value<IteratorType>& i) -> decltype(i.key())
+{
+    return i.key();
+}
+// Structured Bindings Support
+// For further reference see https://blog.tartanllama.xyz/structured-bindings/
+// And see https://github.com/nlohmann/json/pull/1391
+template <std::size_t N, typename IteratorType, enable_if_t<N == 1, int> = 0>
+auto get(const nlohmann::detail::iteration_proxy_value<IteratorType>& i) -> decltype(i.value())
+{
+    return i.value();
+}
+}  // namespace detail
+}  // namespace nlohmann
 
+// The Addition to the STD Namespace is required to add
+// Structured Bindings Support to the iteration_proxy_value class
+// For further reference see https://blog.tartanllama.xyz/structured-bindings/
+// And see https://github.com/nlohmann/json/pull/1391
+namespace std
+{
+template <typename IteratorType>
+class tuple_size<::nlohmann::detail::iteration_proxy_value<IteratorType>>
+            : public std::integral_constant<std::size_t, 2> {};
 
+template <std::size_t N, typename IteratorType>
+class tuple_element<N, ::nlohmann::detail::iteration_proxy_value<IteratorType >>
+{
+  public:
+    using type = decltype(
+                     get<N>(std::declval <
+                            ::nlohmann::detail::iteration_proxy_value<IteratorType >> ()));
+};
+}
+
+namespace nlohmann
+{
+namespace detail
+{
 //////////////////
 // constructors //
 //////////////////
@@ -657,6 +1849,16 @@ struct external_constructor<value_t::string>
         j.m_value = std::move(s);
         j.assert_invariant();
     }
+
+    template<typename BasicJsonType, typename CompatibleStringType,
+             enable_if_t<not std::is_same<CompatibleStringType, typename BasicJsonType::string_t>::value,
+                         int> = 0>
+    static void construct(BasicJsonType& j, const CompatibleStringType& str)
+    {
+        j.m_type = value_t::string;
+        j.m_value.string = j.template create<typename BasicJsonType::string_t>(str);
+        j.assert_invariant();
+    }
 };
 
 template<>
@@ -783,159 +1985,6 @@ struct external_constructor<value_t::object>
     }
 };
 
-
-////////////////////////
-// has_/is_ functions //
-////////////////////////
-
-/*!
-@brief Helper to determine whether there's a key_type for T.
-
-This helper is used to tell associative containers apart from other containers
-such as sequence containers. For instance, `std::map` passes the test as it
-contains a `mapped_type`, whereas `std::vector` fails the test.
-
-@sa http://stackoverflow.com/a/7728728/266378
-@since version 1.0.0, overworked in version 2.0.6
-*/
-#define NLOHMANN_JSON_HAS_HELPER(type)                                        \
-    template<typename T> struct has_##type {                                  \
-    private:                                                                  \
-        template<typename U, typename = typename U::type>                     \
-        static int detect(U &&);                                              \
-        static void detect(...);                                              \
-    public:                                                                   \
-        static constexpr bool value =                                         \
-                std::is_integral<decltype(detect(std::declval<T>()))>::value; \
-    }
-
-NLOHMANN_JSON_HAS_HELPER(mapped_type);
-NLOHMANN_JSON_HAS_HELPER(key_type);
-NLOHMANN_JSON_HAS_HELPER(value_type);
-NLOHMANN_JSON_HAS_HELPER(iterator);
-
-#undef NLOHMANN_JSON_HAS_HELPER
-
-
-template<bool B, class RealType, class CompatibleObjectType>
-struct is_compatible_object_type_impl : std::false_type {};
-
-template<class RealType, class CompatibleObjectType>
-struct is_compatible_object_type_impl<true, RealType, CompatibleObjectType>
-{
-    static constexpr auto value =
-        std::is_constructible<typename RealType::key_type, typename CompatibleObjectType::key_type>::value and
-        std::is_constructible<typename RealType::mapped_type, typename CompatibleObjectType::mapped_type>::value;
-};
-
-template<class BasicJsonType, class CompatibleObjectType>
-struct is_compatible_object_type
-{
-    static auto constexpr value = is_compatible_object_type_impl <
-                                  conjunction<negation<std::is_same<void, CompatibleObjectType>>,
-                                  has_mapped_type<CompatibleObjectType>,
-                                  has_key_type<CompatibleObjectType>>::value,
-                                  typename BasicJsonType::object_t, CompatibleObjectType >::value;
-};
-
-template<typename BasicJsonType, typename T>
-struct is_basic_json_nested_type
-{
-    static auto constexpr value = std::is_same<T, typename BasicJsonType::iterator>::value or
-                                  std::is_same<T, typename BasicJsonType::const_iterator>::value or
-                                  std::is_same<T, typename BasicJsonType::reverse_iterator>::value or
-                                  std::is_same<T, typename BasicJsonType::const_reverse_iterator>::value;
-};
-
-template<class BasicJsonType, class CompatibleArrayType>
-struct is_compatible_array_type
-{
-    static auto constexpr value =
-        conjunction<negation<std::is_same<void, CompatibleArrayType>>,
-        negation<is_compatible_object_type<
-        BasicJsonType, CompatibleArrayType>>,
-        negation<std::is_constructible<typename BasicJsonType::string_t,
-        CompatibleArrayType>>,
-        negation<is_basic_json_nested_type<BasicJsonType, CompatibleArrayType>>,
-        has_value_type<CompatibleArrayType>,
-        has_iterator<CompatibleArrayType>>::value;
-};
-
-template<bool, typename, typename>
-struct is_compatible_integer_type_impl : std::false_type {};
-
-template<typename RealIntegerType, typename CompatibleNumberIntegerType>
-struct is_compatible_integer_type_impl<true, RealIntegerType, CompatibleNumberIntegerType>
-{
-    // is there an assert somewhere on overflows?
-    using RealLimits = std::numeric_limits<RealIntegerType>;
-    using CompatibleLimits = std::numeric_limits<CompatibleNumberIntegerType>;
-
-    static constexpr auto value =
-        std::is_constructible<RealIntegerType, CompatibleNumberIntegerType>::value and
-        CompatibleLimits::is_integer and
-        RealLimits::is_signed == CompatibleLimits::is_signed;
-};
-
-template<typename RealIntegerType, typename CompatibleNumberIntegerType>
-struct is_compatible_integer_type
-{
-    static constexpr auto value =
-        is_compatible_integer_type_impl <
-        std::is_integral<CompatibleNumberIntegerType>::value and
-        not std::is_same<bool, CompatibleNumberIntegerType>::value,
-        RealIntegerType, CompatibleNumberIntegerType >::value;
-};
-
-
-// trait checking if JSONSerializer<T>::from_json(json const&, udt&) exists
-template<typename BasicJsonType, typename T>
-struct has_from_json
-{
-  private:
-    // also check the return type of from_json
-    template<typename U, typename = enable_if_t<std::is_same<void, decltype(uncvref_t<U>::from_json(
-                 std::declval<BasicJsonType>(), std::declval<T&>()))>::value>>
-    static int detect(U&&);
-    static void detect(...);
-
-  public:
-    static constexpr bool value = std::is_integral<decltype(
-                                      detect(std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value;
-};
-
-// This trait checks if JSONSerializer<T>::from_json(json const&) exists
-// this overload is used for non-default-constructible user-defined-types
-template<typename BasicJsonType, typename T>
-struct has_non_default_from_json
-{
-  private:
-    template<typename U, typename =
-             enable_if_t<std::is_same<T, decltype(uncvref_t<U>::from_json(std::declval<BasicJsonType>()))>::value>>
-    static int detect(U&&);
-    static void detect(...);
-
-  public:
-    static constexpr bool value = std::is_integral<decltype(detect(
-                                      std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value;
-};
-
-// This trait checks if BasicJsonType::json_serializer<T>::to_json exists
-template<typename BasicJsonType, typename T>
-struct has_to_json
-{
-  private:
-    template<typename U, typename = decltype(uncvref_t<U>::to_json(
-                 std::declval<BasicJsonType&>(), std::declval<T>()))>
-    static int detect(U&&);
-    static void detect(...);
-
-  public:
-    static constexpr bool value = std::is_integral<decltype(detect(
-                                      std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value;
-};
-
-
 /////////////
 // to_json //
 /////////////
@@ -995,10 +2044,14 @@ void to_json(BasicJsonType& j, const std::vector<bool>& e)
     external_constructor<value_t::array>::construct(j, e);
 }
 
-template<typename BasicJsonType, typename CompatibleArrayType,
-         enable_if_t<is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value or
-                     std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value,
-                     int> = 0>
+template <typename BasicJsonType, typename CompatibleArrayType,
+          enable_if_t<is_compatible_array_type<BasicJsonType,
+                      CompatibleArrayType>::value and
+                      not is_compatible_object_type<
+                          BasicJsonType, CompatibleArrayType>::value and
+                      not is_compatible_string_type<BasicJsonType, CompatibleArrayType>::value and
+                      not is_basic_json<CompatibleArrayType>::value,
+                      int> = 0>
 void to_json(BasicJsonType& j, const CompatibleArrayType& arr)
 {
     external_constructor<value_t::array>::construct(j, arr);
@@ -1006,7 +2059,7 @@ void to_json(BasicJsonType& j, const CompatibleArrayType& arr)
 
 template<typename BasicJsonType, typename T,
          enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>
-void to_json(BasicJsonType& j, std::valarray<T> arr)
+void to_json(BasicJsonType& j, const std::valarray<T>& arr)
 {
     external_constructor<value_t::array>::construct(j, std::move(arr));
 }
@@ -1018,7 +2071,7 @@ void to_json(BasicJsonType& j, typename BasicJsonType::array_t&& arr)
 }
 
 template<typename BasicJsonType, typename CompatibleObjectType,
-         enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value, int> = 0>
+         enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value and not is_basic_json<CompatibleObjectType>::value, int> = 0>
 void to_json(BasicJsonType& j, const CompatibleObjectType& obj)
 {
     external_constructor<value_t::object>::construct(j, obj);
@@ -1030,9 +2083,12 @@ void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj)
     external_constructor<value_t::object>::construct(j, std::move(obj));
 }
 
-template<typename BasicJsonType, typename T, std::size_t N,
-         enable_if_t<not std::is_constructible<typename BasicJsonType::string_t, T (&)[N]>::value, int> = 0>
-void to_json(BasicJsonType& j, T (&arr)[N])
+template <
+    typename BasicJsonType, typename T, std::size_t N,
+    enable_if_t<not std::is_constructible<typename BasicJsonType::string_t,
+                const T(&)[N]>::value,
+                int> = 0 >
+void to_json(BasicJsonType& j, const T(&arr)[N])
 {
     external_constructor<value_t::array>::construct(j, arr);
 }
@@ -1040,351 +2096,71 @@ void to_json(BasicJsonType& j, T (&arr)[N])
 template<typename BasicJsonType, typename... Args>
 void to_json(BasicJsonType& j, const std::pair<Args...>& p)
 {
-    j = {p.first, p.second};
-}
-
-template<typename BasicJsonType, typename Tuple, std::size_t... Idx>
-void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<Idx...>)
-{
-    j = {std::get<Idx>(t)...};
-}
-
-template<typename BasicJsonType, typename... Args>
-void to_json(BasicJsonType& j, const std::tuple<Args...>& t)
-{
-    to_json_tuple_impl(j, t, index_sequence_for<Args...> {});
-}
-
-///////////////
-// from_json //
-///////////////
-
-// overloads for basic_json template parameters
-template<typename BasicJsonType, typename ArithmeticType,
-         enable_if_t<std::is_arithmetic<ArithmeticType>::value and
-                     not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value,
-                     int> = 0>
-void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val)
-{
-    switch (static_cast<value_t>(j))
-    {
-        case value_t::number_unsigned:
-        {
-            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>());
-            break;
-        }
-        case value_t::number_integer:
-        {
-            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>());
-            break;
-        }
-        case value_t::number_float:
-        {
-            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>());
-            break;
-        }
-
-        default:
-            JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name())));
-    }
-}
-
-template<typename BasicJsonType>
-void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b)
-{
-    if (JSON_UNLIKELY(not j.is_boolean()))
-    {
-        JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(j.type_name())));
-    }
-    b = *j.template get_ptr<const typename BasicJsonType::boolean_t*>();
-}
-
-template<typename BasicJsonType>
-void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s)
-{
-    if (JSON_UNLIKELY(not j.is_string()))
-    {
-        JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name())));
-    }
-    s = *j.template get_ptr<const typename BasicJsonType::string_t*>();
-}
-
-template<typename BasicJsonType>
-void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val)
-{
-    get_arithmetic_value(j, val);
-}
-
-template<typename BasicJsonType>
-void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val)
-{
-    get_arithmetic_value(j, val);
-}
-
-template<typename BasicJsonType>
-void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val)
-{
-    get_arithmetic_value(j, val);
-}
-
-template<typename BasicJsonType, typename EnumType,
-         enable_if_t<std::is_enum<EnumType>::value, int> = 0>
-void from_json(const BasicJsonType& j, EnumType& e)
-{
-    typename std::underlying_type<EnumType>::type val;
-    get_arithmetic_value(j, val);
-    e = static_cast<EnumType>(val);
-}
-
-template<typename BasicJsonType>
-void from_json(const BasicJsonType& j, typename BasicJsonType::array_t& arr)
-{
-    if (JSON_UNLIKELY(not j.is_array()))
-    {
-        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
-    }
-    arr = *j.template get_ptr<const typename BasicJsonType::array_t*>();
-}
-
-// forward_list doesn't have an insert method
-template<typename BasicJsonType, typename T, typename Allocator,
-         enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0>
-void from_json(const BasicJsonType& j, std::forward_list<T, Allocator>& l)
-{
-    if (JSON_UNLIKELY(not j.is_array()))
-    {
-        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
-    }
-    std::transform(j.rbegin(), j.rend(),
-                   std::front_inserter(l), [](const BasicJsonType & i)
-    {
-        return i.template get<T>();
-    });
-}
-
-// valarray doesn't have an insert method
-template<typename BasicJsonType, typename T,
-         enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0>
-void from_json(const BasicJsonType& j, std::valarray<T>& l)
-{
-    if (JSON_UNLIKELY(not j.is_array()))
-    {
-        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
-    }
-    l.resize(j.size());
-    std::copy(j.m_value.array->begin(), j.m_value.array->end(), std::begin(l));
-}
-
-template<typename BasicJsonType, typename CompatibleArrayType>
-void from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<0> /*unused*/)
-{
-    using std::end;
-
-    std::transform(j.begin(), j.end(),
-                   std::inserter(arr, end(arr)), [](const BasicJsonType & i)
-    {
-        // get<BasicJsonType>() returns *this, this won't call a from_json
-        // method when value_type is BasicJsonType
-        return i.template get<typename CompatibleArrayType::value_type>();
-    });
-}
-
-template<typename BasicJsonType, typename CompatibleArrayType>
-auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<1> /*unused*/)
--> decltype(
-    arr.reserve(std::declval<typename CompatibleArrayType::size_type>()),
-    void())
-{
-    using std::end;
-
-    arr.reserve(j.size());
-    std::transform(j.begin(), j.end(),
-                   std::inserter(arr, end(arr)), [](const BasicJsonType & i)
-    {
-        // get<BasicJsonType>() returns *this, this won't call a from_json
-        // method when value_type is BasicJsonType
-        return i.template get<typename CompatibleArrayType::value_type>();
-    });
-}
-
-template<typename BasicJsonType, typename T, std::size_t N>
-void from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr, priority_tag<2> /*unused*/)
-{
-    for (std::size_t i = 0; i < N; ++i)
-    {
-        arr[i] = j.at(i).template get<T>();
-    }
-}
-
-template<typename BasicJsonType, typename CompatibleArrayType,
-         enable_if_t<is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value and
-                     std::is_convertible<BasicJsonType, typename CompatibleArrayType::value_type>::value and
-                     not std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value, int> = 0>
-void from_json(const BasicJsonType& j, CompatibleArrayType& arr)
-{
-    if (JSON_UNLIKELY(not j.is_array()))
-    {
-        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
-    }
-
-    from_json_array_impl(j, arr, priority_tag<2> {});
-}
-
-template<typename BasicJsonType, typename CompatibleObjectType,
-         enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value, int> = 0>
-void from_json(const BasicJsonType& j, CompatibleObjectType& obj)
-{
-    if (JSON_UNLIKELY(not j.is_object()))
-    {
-        JSON_THROW(type_error::create(302, "type must be object, but is " + std::string(j.type_name())));
-    }
-
-    auto inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();
-    using value_type = typename CompatibleObjectType::value_type;
-    std::transform(
-        inner_object->begin(), inner_object->end(),
-        std::inserter(obj, obj.begin()),
-        [](typename BasicJsonType::object_t::value_type const & p)
-    {
-        return value_type(p.first, p.second.template get<typename CompatibleObjectType::mapped_type>());
-    });
-}
-
-// overload for arithmetic types, not chosen for basic_json template arguments
-// (BooleanType, etc..); note: Is it really necessary to provide explicit
-// overloads for boolean_t etc. in case of a custom BooleanType which is not
-// an arithmetic type?
-template<typename BasicJsonType, typename ArithmeticType,
-         enable_if_t <
-             std::is_arithmetic<ArithmeticType>::value and
-             not std::is_same<ArithmeticType, typename BasicJsonType::number_unsigned_t>::value and
-             not std::is_same<ArithmeticType, typename BasicJsonType::number_integer_t>::value and
-             not std::is_same<ArithmeticType, typename BasicJsonType::number_float_t>::value and
-             not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value,
-             int> = 0>
-void from_json(const BasicJsonType& j, ArithmeticType& val)
-{
-    switch (static_cast<value_t>(j))
-    {
-        case value_t::number_unsigned:
-        {
-            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>());
-            break;
-        }
-        case value_t::number_integer:
-        {
-            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>());
-            break;
-        }
-        case value_t::number_float:
-        {
-            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>());
-            break;
-        }
-        case value_t::boolean:
-        {
-            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::boolean_t*>());
-            break;
-        }
-
-        default:
-            JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name())));
-    }
+    j = { p.first, p.second };
 }
 
-template<typename BasicJsonType, typename A1, typename A2>
-void from_json(const BasicJsonType& j, std::pair<A1, A2>& p)
+// for https://github.com/nlohmann/json/pull/1134
+template < typename BasicJsonType, typename T,
+           enable_if_t<std::is_same<T, iteration_proxy_value<typename BasicJsonType::iterator>>::value, int> = 0>
+void to_json(BasicJsonType& j, const T& b)
 {
-    p = {j.at(0).template get<A1>(), j.at(1).template get<A2>()};
+    j = { {b.key(), b.value()} };
 }
 
 template<typename BasicJsonType, typename Tuple, std::size_t... Idx>
-void from_json_tuple_impl(const BasicJsonType& j, Tuple& t, index_sequence<Idx...>)
+void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<Idx...> /*unused*/)
 {
-    t = std::make_tuple(j.at(Idx).template get<typename std::tuple_element<Idx, Tuple>::type>()...);
+    j = { std::get<Idx>(t)... };
 }
 
 template<typename BasicJsonType, typename... Args>
-void from_json(const BasicJsonType& j, std::tuple<Args...>& t)
+void to_json(BasicJsonType& j, const std::tuple<Args...>& t)
 {
-    from_json_tuple_impl(j, t, index_sequence_for<Args...> {});
+    to_json_tuple_impl(j, t, index_sequence_for<Args...> {});
 }
 
 struct to_json_fn
 {
-  private:
     template<typename BasicJsonType, typename T>
-    auto call(BasicJsonType& j, T&& val, priority_tag<1> /*unused*/) const noexcept(noexcept(to_json(j, std::forward<T>(val))))
+    auto operator()(BasicJsonType& j, T&& val) const noexcept(noexcept(to_json(j, std::forward<T>(val))))
     -> decltype(to_json(j, std::forward<T>(val)), void())
     {
         return to_json(j, std::forward<T>(val));
     }
+};
+}  // namespace detail
 
-    template<typename BasicJsonType, typename T>
-    void call(BasicJsonType& /*unused*/, T&& /*unused*/, priority_tag<0> /*unused*/) const noexcept
-    {
-        static_assert(sizeof(BasicJsonType) == 0,
-                      "could not find to_json() method in T's namespace");
+/// namespace to hold default `to_json` function
+namespace
+{
+constexpr const auto& to_json = detail::static_const<detail::to_json_fn>::value;
+} // namespace
+}  // namespace nlohmann
 
-#ifdef _MSC_VER
-        // MSVC does not show a stacktrace for the above assert
-        using decayed = uncvref_t<T>;
-        static_assert(sizeof(typename decayed::force_msvc_stacktrace) == 0,
-                      "forcing MSVC stacktrace to show which T we're talking about.");
-#endif
-    }
+// #include <nlohmann/detail/input/input_adapters.hpp>
 
-  public:
-    template<typename BasicJsonType, typename T>
-    void operator()(BasicJsonType& j, T&& val) const
-    noexcept(noexcept(std::declval<to_json_fn>().call(j, std::forward<T>(val), priority_tag<1> {})))
-    {
-        return call(j, std::forward<T>(val), priority_tag<1> {});
-    }
-};
 
-struct from_json_fn
-{
-  private:
-    template<typename BasicJsonType, typename T>
-    auto call(const BasicJsonType& j, T& val, priority_tag<1> /*unused*/) const
-    noexcept(noexcept(from_json(j, val)))
-    -> decltype(from_json(j, val), void())
-    {
-        return from_json(j, val);
-    }
+#include <cassert> // assert
+#include <cstddef> // size_t
+#include <cstring> // strlen
+#include <istream> // istream
+#include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next
+#include <memory> // shared_ptr, make_shared, addressof
+#include <numeric> // accumulate
+#include <string> // string, char_traits
+#include <type_traits> // enable_if, is_base_of, is_pointer, is_integral, remove_pointer
+#include <utility> // pair, declval
+#include <cstdio> //FILE *
 
-    template<typename BasicJsonType, typename T>
-    void call(const BasicJsonType& /*unused*/, T& /*unused*/, priority_tag<0> /*unused*/) const noexcept
-    {
-        static_assert(sizeof(BasicJsonType) == 0,
-                      "could not find from_json() method in T's namespace");
-#ifdef _MSC_VER
-        // MSVC does not show a stacktrace for the above assert
-        using decayed = uncvref_t<T>;
-        static_assert(sizeof(typename decayed::force_msvc_stacktrace) == 0,
-                      "forcing MSVC stacktrace to show which T we're talking about.");
-#endif
-    }
+// #include <nlohmann/detail/macro_scope.hpp>
 
-  public:
-    template<typename BasicJsonType, typename T>
-    void operator()(const BasicJsonType& j, T& val) const
-    noexcept(noexcept(std::declval<from_json_fn>().call(j, val, priority_tag<1> {})))
-    {
-        return call(j, val, priority_tag<1> {});
-    }
-};
 
-// taken from ranges-v3
-template<typename T>
-struct static_const
+namespace nlohmann
 {
-    static constexpr T value{};
-};
-
-template<typename T>
-constexpr T static_const<T>::value;
+namespace detail
+{
+/// the supported input formats
+enum class input_format_t { json, cbor, msgpack, ubjson, bson };
 
 ////////////////////
 // input adapters //
@@ -1394,19 +2170,17 @@ constexpr T static_const<T>::value;
 @brief abstract input adapter interface
 
 Produces a stream of std::char_traits<char>::int_type characters from a
-std::istream, a buffer, or some other input type.  Accepts the return of exactly
-one non-EOF character for future input.  The int_type characters returned
-consist of all valid char values as positive values (typically unsigned char),
-plus an EOF value outside that range, specified by the value of the function
-std::char_traits<char>::eof().  This value is typically -1, but could be any
-arbitrary value which is not a valid char value.
+std::istream, a buffer, or some other input type. Accepts the return of
+exactly one non-EOF character for future input. The int_type characters
+returned consist of all valid char values as positive values (typically
+unsigned char), plus an EOF value outside that range, specified by the value
+of the function std::char_traits<char>::eof(). This value is typically -1, but
+could be any arbitrary value which is not a valid char value.
 */
 struct input_adapter_protocol
 {
     /// get a character [0,255] or std::char_traits<char>::eof().
     virtual std::char_traits<char>::int_type get_character() = 0;
-    /// restore the last non-eof() character to input
-    virtual void unget_character() = 0;
     virtual ~input_adapter_protocol() = default;
 };
 
@@ -1414,6 +2188,27 @@ struct input_adapter_protocol
 using input_adapter_t = std::shared_ptr<input_adapter_protocol>;
 
 /*!
+Input adapter for stdio file access. This adapter read only 1 byte and do not use any
+ buffer. This adapter is a very low level adapter.
+*/
+class file_input_adapter : public input_adapter_protocol
+{
+  public:
+    explicit file_input_adapter(std::FILE* f)  noexcept
+        : m_file(f)
+    {}
+
+    std::char_traits<char>::int_type get_character() noexcept override
+    {
+        return std::fgetc(m_file);
+    }
+  private:
+    /// the file pointer to read from
+    std::FILE* m_file;
+};
+
+
+/*!
 Input adapter for a (caching) istream. Ignores a UFT Byte Order Mark at
 beginning of input. Does not support changing the underlying std::streambuf
 in mid-input. Maintains underlying std::istream and std::streambuf to support
@@ -1428,56 +2223,32 @@ class input_stream_adapter : public input_adapter_protocol
     ~input_stream_adapter() override
     {
         // clear stream flags; we use underlying streambuf I/O, do not
-        // maintain ifstream flags
-        is.clear();
+        // maintain ifstream flags, except eof
+        is.clear(is.rdstate() & std::ios::eofbit);
     }
 
     explicit input_stream_adapter(std::istream& i)
         : is(i), sb(*i.rdbuf())
-    {
-        // skip byte order mark
-        std::char_traits<char>::int_type c;
-        if ((c = get_character()) == 0xEF)
-        {
-            if ((c = get_character()) == 0xBB)
-            {
-                if ((c = get_character()) == 0xBF)
-                {
-                    return; // Ignore BOM
-                }
-                else if (c != std::char_traits<char>::eof())
-                {
-                    is.unget();
-                }
-                is.putback('\xBB');
-            }
-            else if (c != std::char_traits<char>::eof())
-            {
-                is.unget();
-            }
-            is.putback('\xEF');
-        }
-        else if (c != std::char_traits<char>::eof())
-        {
-            is.unget(); // no byte order mark; process as usual
-        }
-    }
+    {}
 
     // delete because of pointer members
     input_stream_adapter(const input_stream_adapter&) = delete;
     input_stream_adapter& operator=(input_stream_adapter&) = delete;
+    input_stream_adapter(input_stream_adapter&&) = delete;
+    input_stream_adapter& operator=(input_stream_adapter&&) = delete;
 
     // std::istream/std::streambuf use std::char_traits<char>::to_int_type, to
     // ensure that std::char_traits<char>::eof() and the character 0xFF do not
     // end up as the same value, eg. 0xFFFFFFFF.
     std::char_traits<char>::int_type get_character() override
     {
-        return sb.sbumpc();
-    }
-
-    void unget_character() override
-    {
-        sb.sungetc();  // is.unget() avoided for performance
+        auto res = sb.sbumpc();
+        // set eof manually, as we don't use the istream interface.
+        if (res == EOF)
+        {
+            is.clear(is.rdstate() | std::ios::eofbit);
+        }
+        return res;
     }
 
   private:
@@ -1490,19 +2261,16 @@ class input_stream_adapter : public input_adapter_protocol
 class input_buffer_adapter : public input_adapter_protocol
 {
   public:
-    input_buffer_adapter(const char* b, const std::size_t l)
-        : cursor(b), limit(b + l), start(b)
-    {
-        // skip byte order mark
-        if (l >= 3 and b[0] == '\xEF' and b[1] == '\xBB' and b[2] == '\xBF')
-        {
-            cursor += 3;
-        }
-    }
+    input_buffer_adapter(const char* b, const std::size_t l) noexcept
+        : cursor(b), limit(b + l)
+    {}
 
     // delete because of pointer members
     input_buffer_adapter(const input_buffer_adapter&) = delete;
     input_buffer_adapter& operator=(input_buffer_adapter&) = delete;
+    input_buffer_adapter(input_buffer_adapter&&) = delete;
+    input_buffer_adapter& operator=(input_buffer_adapter&&) = delete;
+    ~input_buffer_adapter() override = default;
 
     std::char_traits<char>::int_type get_character() noexcept override
     {
@@ -1514,28 +2282,182 @@ class input_buffer_adapter : public input_adapter_protocol
         return std::char_traits<char>::eof();
     }
 
-    void unget_character() noexcept override
+  private:
+    /// pointer to the current character
+    const char* cursor;
+    /// pointer past the last character
+    const char* const limit;
+};
+
+template<typename WideStringType, size_t T>
+struct wide_string_input_helper
+{
+    // UTF-32
+    static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array<std::char_traits<char>::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled)
     {
-        if (JSON_LIKELY(cursor > start))
+        utf8_bytes_index = 0;
+
+        if (current_wchar == str.size())
         {
-            --cursor;
+            utf8_bytes[0] = std::char_traits<char>::eof();
+            utf8_bytes_filled = 1;
+        }
+        else
+        {
+            // get the current character
+            const auto wc = static_cast<int>(str[current_wchar++]);
+
+            // UTF-32 to UTF-8 encoding
+            if (wc < 0x80)
+            {
+                utf8_bytes[0] = wc;
+                utf8_bytes_filled = 1;
+            }
+            else if (wc <= 0x7FF)
+            {
+                utf8_bytes[0] = 0xC0 | ((wc >> 6) & 0x1F);
+                utf8_bytes[1] = 0x80 | (wc & 0x3F);
+                utf8_bytes_filled = 2;
+            }
+            else if (wc <= 0xFFFF)
+            {
+                utf8_bytes[0] = 0xE0 | ((wc >> 12) & 0x0F);
+                utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F);
+                utf8_bytes[2] = 0x80 | (wc & 0x3F);
+                utf8_bytes_filled = 3;
+            }
+            else if (wc <= 0x10FFFF)
+            {
+                utf8_bytes[0] = 0xF0 | ((wc >> 18) & 0x07);
+                utf8_bytes[1] = 0x80 | ((wc >> 12) & 0x3F);
+                utf8_bytes[2] = 0x80 | ((wc >> 6) & 0x3F);
+                utf8_bytes[3] = 0x80 | (wc & 0x3F);
+                utf8_bytes_filled = 4;
+            }
+            else
+            {
+                // unknown character
+                utf8_bytes[0] = wc;
+                utf8_bytes_filled = 1;
+            }
         }
     }
+};
+
+template<typename WideStringType>
+struct wide_string_input_helper<WideStringType, 2>
+{
+    // UTF-16
+    static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array<std::char_traits<char>::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled)
+    {
+        utf8_bytes_index = 0;
+
+        if (current_wchar == str.size())
+        {
+            utf8_bytes[0] = std::char_traits<char>::eof();
+            utf8_bytes_filled = 1;
+        }
+        else
+        {
+            // get the current character
+            const auto wc = static_cast<int>(str[current_wchar++]);
+
+            // UTF-16 to UTF-8 encoding
+            if (wc < 0x80)
+            {
+                utf8_bytes[0] = wc;
+                utf8_bytes_filled = 1;
+            }
+            else if (wc <= 0x7FF)
+            {
+                utf8_bytes[0] = 0xC0 | ((wc >> 6));
+                utf8_bytes[1] = 0x80 | (wc & 0x3F);
+                utf8_bytes_filled = 2;
+            }
+            else if (0xD800 > wc or wc >= 0xE000)
+            {
+                utf8_bytes[0] = 0xE0 | ((wc >> 12));
+                utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F);
+                utf8_bytes[2] = 0x80 | (wc & 0x3F);
+                utf8_bytes_filled = 3;
+            }
+            else
+            {
+                if (current_wchar < str.size())
+                {
+                    const auto wc2 = static_cast<int>(str[current_wchar++]);
+                    const int charcode = 0x10000 + (((wc & 0x3FF) << 10) | (wc2 & 0x3FF));
+                    utf8_bytes[0] = 0xf0 | (charcode >> 18);
+                    utf8_bytes[1] = 0x80 | ((charcode >> 12) & 0x3F);
+                    utf8_bytes[2] = 0x80 | ((charcode >> 6) & 0x3F);
+                    utf8_bytes[3] = 0x80 | (charcode & 0x3F);
+                    utf8_bytes_filled = 4;
+                }
+                else
+                {
+                    // unknown character
+                    ++current_wchar;
+                    utf8_bytes[0] = wc;
+                    utf8_bytes_filled = 1;
+                }
+            }
+        }
+    }
+};
+
+template<typename WideStringType>
+class wide_string_input_adapter : public input_adapter_protocol
+{
+  public:
+    explicit wide_string_input_adapter(const WideStringType& w)  noexcept
+        : str(w)
+    {}
+
+    std::char_traits<char>::int_type get_character() noexcept override
+    {
+        // check if buffer needs to be filled
+        if (utf8_bytes_index == utf8_bytes_filled)
+        {
+            fill_buffer<sizeof(typename WideStringType::value_type)>();
+
+            assert(utf8_bytes_filled > 0);
+            assert(utf8_bytes_index == 0);
+        }
+
+        // use buffer
+        assert(utf8_bytes_filled > 0);
+        assert(utf8_bytes_index < utf8_bytes_filled);
+        return utf8_bytes[utf8_bytes_index++];
+    }
 
   private:
-    /// pointer to the current character
-    const char* cursor;
-    /// pointer past the last character
-    const char* limit;
-    /// pointer to the first character
-    const char* start;
+    template<size_t T>
+    void fill_buffer()
+    {
+        wide_string_input_helper<WideStringType, T>::fill_buffer(str, current_wchar, utf8_bytes, utf8_bytes_index, utf8_bytes_filled);
+    }
+
+    /// the wstring to process
+    const WideStringType& str;
+
+    /// index of the current wchar in str
+    std::size_t current_wchar = 0;
+
+    /// a buffer for UTF-8 bytes
+    std::array<std::char_traits<char>::int_type, 4> utf8_bytes = {{0, 0, 0, 0}};
+
+    /// index to the utf8_codes array for the next valid byte
+    std::size_t utf8_bytes_index = 0;
+    /// number of valid bytes in the utf8_codes array
+    std::size_t utf8_bytes_filled = 0;
 };
 
 class input_adapter
 {
   public:
     // native support
-
+    input_adapter(std::FILE* file)
+        : ia(std::make_shared<file_input_adapter>(file)) {}
     /// input adapter for input stream
     input_adapter(std::istream& i)
         : ia(std::make_shared<input_stream_adapter>(i)) {}
@@ -1544,6 +2466,15 @@ class input_adapter
     input_adapter(std::istream&& i)
         : ia(std::make_shared<input_stream_adapter>(i)) {}
 
+    input_adapter(const std::wstring& ws)
+        : ia(std::make_shared<wide_string_input_adapter<std::wstring>>(ws)) {}
+
+    input_adapter(const std::u16string& ws)
+        : ia(std::make_shared<wide_string_input_adapter<std::u16string>>(ws)) {}
+
+    input_adapter(const std::u32string& ws)
+        : ia(std::make_shared<wide_string_input_adapter<std::u32string>>(ws)) {}
+
     /// input adapter for buffer
     template<typename CharT,
              typename std::enable_if<
@@ -1570,23 +2501,26 @@ class input_adapter
     /// input adapter for iterator range with contiguous storage
     template<class IteratorType,
              typename std::enable_if<
-                 std::is_same<typename std::iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value,
+                 std::is_same<typename iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value,
                  int>::type = 0>
     input_adapter(IteratorType first, IteratorType last)
     {
+#ifndef NDEBUG
         // assertion to check that the iterator range is indeed contiguous,
         // see http://stackoverflow.com/a/35008842/266378 for more discussion
-        assert(std::accumulate(
-                   first, last, std::pair<bool, int>(true, 0),
-                   [&first](std::pair<bool, int> res, decltype(*first) val)
+        const auto is_contiguous = std::accumulate(
+                                       first, last, std::pair<bool, int>(true, 0),
+                                       [&first](std::pair<bool, int> res, decltype(*first) val)
         {
             res.first &= (val == *(std::next(std::addressof(*first), res.second++)));
             return res;
-        }).first);
+        }).first;
+        assert(is_contiguous);
+#endif
 
         // assertion to check that each element is 1 byte long
         static_assert(
-            sizeof(typename std::iterator_traits<IteratorType>::value_type) == 1,
+            sizeof(typename iterator_traits<IteratorType>::value_type) == 1,
             "each element in the iterator range must have the size of 1 byte");
 
         const auto len = static_cast<size_t>(std::distance(first, last));
@@ -1610,7 +2544,7 @@ class input_adapter
     /// input adapter for contiguous container
     template<class ContiguousContainer, typename
              std::enable_if<not std::is_pointer<ContiguousContainer>::value and
-                            std::is_base_of<std::random_access_iterator_tag, typename std::iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value,
+                            std::is_base_of<std::random_access_iterator_tag, typename iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value,
                             int>::type = 0>
     input_adapter(const ContiguousContainer& c)
         : input_adapter(std::begin(c), std::end(c)) {}
@@ -1624,10 +2558,34 @@ class input_adapter
     /// the actual adapter
     input_adapter_t ia = nullptr;
 };
+}  // namespace detail
+}  // namespace nlohmann
 
-//////////////////////
-// lexer and parser //
-//////////////////////
+// #include <nlohmann/detail/input/lexer.hpp>
+
+
+#include <clocale> // localeconv
+#include <cstddef> // size_t
+#include <cstdlib> // strtof, strtod, strtold, strtoll, strtoull
+#include <cstdio> // snprintf
+#include <initializer_list> // initializer_list
+#include <string> // char_traits, string
+#include <vector> // vector
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/input/input_adapters.hpp>
+
+// #include <nlohmann/detail/input/position_t.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+///////////
+// lexer //
+///////////
 
 /*!
 @brief lexical analysis
@@ -1640,6 +2598,7 @@ class lexer
     using number_integer_t = typename BasicJsonType::number_integer_t;
     using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
     using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
 
   public:
     /// token types for the parser
@@ -1701,17 +2660,22 @@ class lexer
                 return "end of input";
             case token_type::literal_or_value:
                 return "'[', '{', or a literal";
+            // LCOV_EXCL_START
             default: // catch non-enum values
-                return "unknown token"; // LCOV_EXCL_LINE
+                return "unknown token";
+                // LCOV_EXCL_STOP
         }
     }
 
-    explicit lexer(detail::input_adapter_t adapter)
+    explicit lexer(detail::input_adapter_t&& adapter)
         : ia(std::move(adapter)), decimal_point_char(get_decimal_point()) {}
 
     // delete because of pointer members
     lexer(const lexer&) = delete;
+    lexer(lexer&&) = delete;
     lexer& operator=(lexer&) = delete;
+    lexer& operator=(lexer&&) = delete;
+    ~lexer() = default;
 
   private:
     /////////////////////
@@ -1819,9 +2783,10 @@ class lexer
     @brief scan a string literal
 
     This function scans a string according to Sect. 7 of RFC 7159. While
-    scanning, bytes are escaped and copied into buffer yytext. Then the function
-    returns successfully, yytext is *not* null-terminated (as it may contain \0
-    bytes), and yytext.size() is the number of bytes in the string.
+    scanning, bytes are escaped and copied into buffer token_buffer. Then the
+    function returns successfully, token_buffer is *not* null-terminated (as it
+    may contain \0 bytes), and token_buffer.size() is the number of bytes in the
+    string.
 
     @return token_type::value_string if string could be successfully scanned,
             token_type::parse_error otherwise
@@ -1831,7 +2796,7 @@ class lexer
     */
     token_type scan_string()
     {
-        // reset yytext (ignore opening quote)
+        // reset token_buffer (ignore opening quote)
         reset();
 
         // we entered the function by reading an open quote
@@ -1999,39 +2964,194 @@ class lexer
 
                 // invalid control characters
                 case 0x00:
+                {
+                    error_message = "invalid string: control character U+0000 (NUL) must be escaped to \\u0000";
+                    return token_type::parse_error;
+                }
+
                 case 0x01:
+                {
+                    error_message = "invalid string: control character U+0001 (SOH) must be escaped to \\u0001";
+                    return token_type::parse_error;
+                }
+
                 case 0x02:
+                {
+                    error_message = "invalid string: control character U+0002 (STX) must be escaped to \\u0002";
+                    return token_type::parse_error;
+                }
+
                 case 0x03:
+                {
+                    error_message = "invalid string: control character U+0003 (ETX) must be escaped to \\u0003";
+                    return token_type::parse_error;
+                }
+
                 case 0x04:
+                {
+                    error_message = "invalid string: control character U+0004 (EOT) must be escaped to \\u0004";
+                    return token_type::parse_error;
+                }
+
                 case 0x05:
+                {
+                    error_message = "invalid string: control character U+0005 (ENQ) must be escaped to \\u0005";
+                    return token_type::parse_error;
+                }
+
                 case 0x06:
+                {
+                    error_message = "invalid string: control character U+0006 (ACK) must be escaped to \\u0006";
+                    return token_type::parse_error;
+                }
+
                 case 0x07:
+                {
+                    error_message = "invalid string: control character U+0007 (BEL) must be escaped to \\u0007";
+                    return token_type::parse_error;
+                }
+
                 case 0x08:
+                {
+                    error_message = "invalid string: control character U+0008 (BS) must be escaped to \\u0008 or \\b";
+                    return token_type::parse_error;
+                }
+
                 case 0x09:
+                {
+                    error_message = "invalid string: control character U+0009 (HT) must be escaped to \\u0009 or \\t";
+                    return token_type::parse_error;
+                }
+
                 case 0x0A:
+                {
+                    error_message = "invalid string: control character U+000A (LF) must be escaped to \\u000A or \\n";
+                    return token_type::parse_error;
+                }
+
                 case 0x0B:
+                {
+                    error_message = "invalid string: control character U+000B (VT) must be escaped to \\u000B";
+                    return token_type::parse_error;
+                }
+
                 case 0x0C:
+                {
+                    error_message = "invalid string: control character U+000C (FF) must be escaped to \\u000C or \\f";
+                    return token_type::parse_error;
+                }
+
                 case 0x0D:
+                {
+                    error_message = "invalid string: control character U+000D (CR) must be escaped to \\u000D or \\r";
+                    return token_type::parse_error;
+                }
+
                 case 0x0E:
+                {
+                    error_message = "invalid string: control character U+000E (SO) must be escaped to \\u000E";
+                    return token_type::parse_error;
+                }
+
                 case 0x0F:
+                {
+                    error_message = "invalid string: control character U+000F (SI) must be escaped to \\u000F";
+                    return token_type::parse_error;
+                }
+
                 case 0x10:
+                {
+                    error_message = "invalid string: control character U+0010 (DLE) must be escaped to \\u0010";
+                    return token_type::parse_error;
+                }
+
                 case 0x11:
+                {
+                    error_message = "invalid string: control character U+0011 (DC1) must be escaped to \\u0011";
+                    return token_type::parse_error;
+                }
+
                 case 0x12:
+                {
+                    error_message = "invalid string: control character U+0012 (DC2) must be escaped to \\u0012";
+                    return token_type::parse_error;
+                }
+
                 case 0x13:
+                {
+                    error_message = "invalid string: control character U+0013 (DC3) must be escaped to \\u0013";
+                    return token_type::parse_error;
+                }
+
                 case 0x14:
+                {
+                    error_message = "invalid string: control character U+0014 (DC4) must be escaped to \\u0014";
+                    return token_type::parse_error;
+                }
+
                 case 0x15:
+                {
+                    error_message = "invalid string: control character U+0015 (NAK) must be escaped to \\u0015";
+                    return token_type::parse_error;
+                }
+
                 case 0x16:
+                {
+                    error_message = "invalid string: control character U+0016 (SYN) must be escaped to \\u0016";
+                    return token_type::parse_error;
+                }
+
                 case 0x17:
+                {
+                    error_message = "invalid string: control character U+0017 (ETB) must be escaped to \\u0017";
+                    return token_type::parse_error;
+                }
+
                 case 0x18:
+                {
+                    error_message = "invalid string: control character U+0018 (CAN) must be escaped to \\u0018";
+                    return token_type::parse_error;
+                }
+
                 case 0x19:
+                {
+                    error_message = "invalid string: control character U+0019 (EM) must be escaped to \\u0019";
+                    return token_type::parse_error;
+                }
+
                 case 0x1A:
+                {
+                    error_message = "invalid string: control character U+001A (SUB) must be escaped to \\u001A";
+                    return token_type::parse_error;
+                }
+
                 case 0x1B:
+                {
+                    error_message = "invalid string: control character U+001B (ESC) must be escaped to \\u001B";
+                    return token_type::parse_error;
+                }
+
                 case 0x1C:
+                {
+                    error_message = "invalid string: control character U+001C (FS) must be escaped to \\u001C";
+                    return token_type::parse_error;
+                }
+
                 case 0x1D:
+                {
+                    error_message = "invalid string: control character U+001D (GS) must be escaped to \\u001D";
+                    return token_type::parse_error;
+                }
+
                 case 0x1E:
+                {
+                    error_message = "invalid string: control character U+001E (RS) must be escaped to \\u001E";
+                    return token_type::parse_error;
+                }
+
                 case 0x1F:
                 {
-                    error_message = "invalid string: control character must be escaped";
+                    error_message = "invalid string: control character U+001F (US) must be escaped to \\u001F";
                     return token_type::parse_error;
                 }
 
@@ -2303,7 +3423,7 @@ class lexer
     contains cycles, but any cycle can be left when EOF is read. Therefore,
     the function is guaranteed to terminate.
 
-    During scanning, the read bytes are stored in yytext. This string is
+    During scanning, the read bytes are stored in token_buffer. This string is
     then converted to a signed integer, an unsigned integer, or a
     floating-point number.
 
@@ -2315,9 +3435,9 @@ class lexer
           locale's decimal point is used instead of `.` to work with the
           locale-dependent converters.
     */
-    token_type scan_number()
+    token_type scan_number()  // lgtm [cpp/use-of-goto]
     {
-        // reset yytext to store the number's bytes
+        // reset token_buffer to store the number's bytes
         reset();
 
         // the type of the parsed number; initially set to unsigned; will be
@@ -2353,11 +3473,13 @@ class lexer
                 goto scan_number_any1;
             }
 
+            // LCOV_EXCL_START
             default:
             {
                 // all other characters are rejected outside scan_number()
-                assert(false); // LCOV_EXCL_LINE
+                assert(false);
             }
+                // LCOV_EXCL_STOP
         }
 
 scan_number_minus:
@@ -2601,10 +3723,10 @@ scan_number_done:
         // try to parse integers first and fall back to floats
         if (number_type == token_type::value_unsigned)
         {
-            const auto x = std::strtoull(yytext.data(), &endptr, 10);
+            const auto x = std::strtoull(token_buffer.data(), &endptr, 10);
 
             // we checked the number format before
-            assert(endptr == yytext.data() + yytext.size());
+            assert(endptr == token_buffer.data() + token_buffer.size());
 
             if (errno == 0)
             {
@@ -2617,10 +3739,10 @@ scan_number_done:
         }
         else if (number_type == token_type::value_integer)
         {
-            const auto x = std::strtoll(yytext.data(), &endptr, 10);
+            const auto x = std::strtoll(token_buffer.data(), &endptr, 10);
 
             // we checked the number format before
-            assert(endptr == yytext.data() + yytext.size());
+            assert(endptr == token_buffer.data() + token_buffer.size());
 
             if (errno == 0)
             {
@@ -2634,10 +3756,10 @@ scan_number_done:
 
         // this code is reached if we parse a floating-point number or if an
         // integer conversion above failed
-        strtof(value_float, yytext.data(), &endptr);
+        strtof(value_float, token_buffer.data(), &endptr);
 
         // we checked the number format before
-        assert(endptr == yytext.data() + yytext.size());
+        assert(endptr == token_buffer.data() + token_buffer.size());
 
         return token_type::value_float;
     }
@@ -2666,10 +3788,10 @@ scan_number_done:
     // input management
     /////////////////////
 
-    /// reset yytext; current character is beginning of token
+    /// reset token_buffer; current character is beginning of token
     void reset() noexcept
     {
-        yytext.clear();
+        token_buffer.clear();
         token_string.clear();
         token_string.push_back(std::char_traits<char>::to_char_type(current));
     }
@@ -2686,31 +3808,71 @@ scan_number_done:
     */
     std::char_traits<char>::int_type get()
     {
-        ++chars_read;
-        current = ia->get_character();
+        ++position.chars_read_total;
+        ++position.chars_read_current_line;
+
+        if (next_unget)
+        {
+            // just reset the next_unget variable and work with current
+            next_unget = false;
+        }
+        else
+        {
+            current = ia->get_character();
+        }
+
         if (JSON_LIKELY(current != std::char_traits<char>::eof()))
         {
             token_string.push_back(std::char_traits<char>::to_char_type(current));
         }
+
+        if (current == '\n')
+        {
+            ++position.lines_read;
+            ++position.chars_read_current_line = 0;
+        }
+
         return current;
     }
 
-    /// unget current character (return it again on next get)
+    /*!
+    @brief unget current character (read it again on next get)
+
+    We implement unget by setting variable next_unget to true. The input is not
+    changed - we just simulate ungetting by modifying chars_read_total,
+    chars_read_current_line, and token_string. The next call to get() will
+    behave as if the unget character is read again.
+    */
     void unget()
     {
-        --chars_read;
+        next_unget = true;
+
+        --position.chars_read_total;
+
+        // in case we "unget" a newline, we have to also decrement the lines_read
+        if (position.chars_read_current_line == 0)
+        {
+            if (position.lines_read > 0)
+            {
+                --position.lines_read;
+            }
+        }
+        else
+        {
+            --position.chars_read_current_line;
+        }
+
         if (JSON_LIKELY(current != std::char_traits<char>::eof()))
         {
-            ia->unget_character();
             assert(token_string.size() != 0);
             token_string.pop_back();
         }
     }
 
-    /// add a character to yytext
+    /// add a character to token_buffer
     void add(int c)
     {
-        yytext.push_back(std::char_traits<char>::to_char_type(c));
+        token_buffer.push_back(std::char_traits<char>::to_char_type(c));
     }
 
   public:
@@ -2737,9 +3899,9 @@ scan_number_done:
     }
 
     /// return current string value (implicitly resets the token; useful only once)
-    std::string move_string()
+    string_t& get_string()
     {
-        return std::move(yytext);
+        return token_buffer;
     }
 
     /////////////////////
@@ -2747,9 +3909,9 @@ scan_number_done:
     /////////////////////
 
     /// return position of last read token
-    constexpr std::size_t get_position() const noexcept
+    constexpr position_t get_position() const noexcept
     {
-        return chars_read;
+        return position;
     }
 
     /// return the last read token (for errors only).  Will never contain EOF
@@ -2764,10 +3926,9 @@ scan_number_done:
             if ('\x00' <= c and c <= '\x1F')
             {
                 // escape control characters
-                std::stringstream ss;
-                ss << "<U+" << std::setw(4) << std::uppercase << std::setfill('0')
-                   << std::hex << static_cast<int>(c) << ">";
-                result += ss.str();
+                char cs[9];
+                (std::snprintf)(cs, 9, "<U+%.4X>", static_cast<unsigned char>(c));
+                result += cs;
             }
             else
             {
@@ -2789,8 +3950,33 @@ scan_number_done:
     // actual scanner
     /////////////////////
 
+    /*!
+    @brief skip the UTF-8 byte order mark
+    @return true iff there is no BOM or the correct BOM has been skipped
+    */
+    bool skip_bom()
+    {
+        if (get() == 0xEF)
+        {
+            // check if we completely parse the BOM
+            return get() == 0xBB and get() == 0xBF;
+        }
+
+        // the first character is not the beginning of the BOM; unget it to
+        // process is later
+        unget();
+        return true;
+    }
+
     token_type scan()
     {
+        // initially, skip the BOM
+        if (position.chars_read_total == 0 and not skip_bom())
+        {
+            error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given";
+            return token_type::parse_error;
+        }
+
         // read next character and ignore whitespace
         do
         {
@@ -2860,14 +4046,17 @@ scan_number_done:
     /// the current character
     std::char_traits<char>::int_type current = std::char_traits<char>::eof();
 
-    /// the number of characters read
-    std::size_t chars_read = 0;
+    /// whether the next get() call should just return current
+    bool next_unget = false;
+
+    /// the start position of the current token
+    position_t position;
 
     /// raw input token string (for error messages)
     std::vector<char> token_string {};
 
     /// buffer for variable-length tokens (numbers, strings)
-    std::string yytext {};
+    string_t token_buffer {};
 
     /// a description of occurred lexer errors
     const char* error_message = "";
@@ -2880,6 +4069,887 @@ scan_number_done:
     /// the decimal point
     const char decimal_point_char = '.';
 };
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/input/parser.hpp>
+
+
+#include <cassert> // assert
+#include <cmath> // isfinite
+#include <cstdint> // uint8_t
+#include <functional> // function
+#include <string> // string
+#include <utility> // move
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/is_sax.hpp>
+
+
+#include <cstdint> // size_t
+#include <utility> // declval
+
+// #include <nlohmann/detail/meta/detected.hpp>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+template <typename T>
+using null_function_t = decltype(std::declval<T&>().null());
+
+template <typename T>
+using boolean_function_t =
+    decltype(std::declval<T&>().boolean(std::declval<bool>()));
+
+template <typename T, typename Integer>
+using number_integer_function_t =
+    decltype(std::declval<T&>().number_integer(std::declval<Integer>()));
+
+template <typename T, typename Unsigned>
+using number_unsigned_function_t =
+    decltype(std::declval<T&>().number_unsigned(std::declval<Unsigned>()));
+
+template <typename T, typename Float, typename String>
+using number_float_function_t = decltype(std::declval<T&>().number_float(
+                                    std::declval<Float>(), std::declval<const String&>()));
+
+template <typename T, typename String>
+using string_function_t =
+    decltype(std::declval<T&>().string(std::declval<String&>()));
+
+template <typename T>
+using start_object_function_t =
+    decltype(std::declval<T&>().start_object(std::declval<std::size_t>()));
+
+template <typename T, typename String>
+using key_function_t =
+    decltype(std::declval<T&>().key(std::declval<String&>()));
+
+template <typename T>
+using end_object_function_t = decltype(std::declval<T&>().end_object());
+
+template <typename T>
+using start_array_function_t =
+    decltype(std::declval<T&>().start_array(std::declval<std::size_t>()));
+
+template <typename T>
+using end_array_function_t = decltype(std::declval<T&>().end_array());
+
+template <typename T, typename Exception>
+using parse_error_function_t = decltype(std::declval<T&>().parse_error(
+        std::declval<std::size_t>(), std::declval<const std::string&>(),
+        std::declval<const Exception&>()));
+
+template <typename SAX, typename BasicJsonType>
+struct is_sax
+{
+  private:
+    static_assert(is_basic_json<BasicJsonType>::value,
+                  "BasicJsonType must be of type basic_json<...>");
+
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+    using exception_t = typename BasicJsonType::exception;
+
+  public:
+    static constexpr bool value =
+        is_detected_exact<bool, null_function_t, SAX>::value &&
+        is_detected_exact<bool, boolean_function_t, SAX>::value &&
+        is_detected_exact<bool, number_integer_function_t, SAX,
+        number_integer_t>::value &&
+        is_detected_exact<bool, number_unsigned_function_t, SAX,
+        number_unsigned_t>::value &&
+        is_detected_exact<bool, number_float_function_t, SAX, number_float_t,
+        string_t>::value &&
+        is_detected_exact<bool, string_function_t, SAX, string_t>::value &&
+        is_detected_exact<bool, start_object_function_t, SAX>::value &&
+        is_detected_exact<bool, key_function_t, SAX, string_t>::value &&
+        is_detected_exact<bool, end_object_function_t, SAX>::value &&
+        is_detected_exact<bool, start_array_function_t, SAX>::value &&
+        is_detected_exact<bool, end_array_function_t, SAX>::value &&
+        is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value;
+};
+
+template <typename SAX, typename BasicJsonType>
+struct is_sax_static_asserts
+{
+  private:
+    static_assert(is_basic_json<BasicJsonType>::value,
+                  "BasicJsonType must be of type basic_json<...>");
+
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+    using exception_t = typename BasicJsonType::exception;
+
+  public:
+    static_assert(is_detected_exact<bool, null_function_t, SAX>::value,
+                  "Missing/invalid function: bool null()");
+    static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value,
+                  "Missing/invalid function: bool boolean(bool)");
+    static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value,
+                  "Missing/invalid function: bool boolean(bool)");
+    static_assert(
+        is_detected_exact<bool, number_integer_function_t, SAX,
+        number_integer_t>::value,
+        "Missing/invalid function: bool number_integer(number_integer_t)");
+    static_assert(
+        is_detected_exact<bool, number_unsigned_function_t, SAX,
+        number_unsigned_t>::value,
+        "Missing/invalid function: bool number_unsigned(number_unsigned_t)");
+    static_assert(is_detected_exact<bool, number_float_function_t, SAX,
+                  number_float_t, string_t>::value,
+                  "Missing/invalid function: bool number_float(number_float_t, const string_t&)");
+    static_assert(
+        is_detected_exact<bool, string_function_t, SAX, string_t>::value,
+        "Missing/invalid function: bool string(string_t&)");
+    static_assert(is_detected_exact<bool, start_object_function_t, SAX>::value,
+                  "Missing/invalid function: bool start_object(std::size_t)");
+    static_assert(is_detected_exact<bool, key_function_t, SAX, string_t>::value,
+                  "Missing/invalid function: bool key(string_t&)");
+    static_assert(is_detected_exact<bool, end_object_function_t, SAX>::value,
+                  "Missing/invalid function: bool end_object()");
+    static_assert(is_detected_exact<bool, start_array_function_t, SAX>::value,
+                  "Missing/invalid function: bool start_array(std::size_t)");
+    static_assert(is_detected_exact<bool, end_array_function_t, SAX>::value,
+                  "Missing/invalid function: bool end_array()");
+    static_assert(
+        is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value,
+        "Missing/invalid function: bool parse_error(std::size_t, const "
+        "std::string&, const exception&)");
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/input/input_adapters.hpp>
+
+// #include <nlohmann/detail/input/json_sax.hpp>
+
+
+#include <cstddef>
+#include <string>
+#include <vector>
+
+// #include <nlohmann/detail/input/parser.hpp>
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+
+namespace nlohmann
+{
+
+/*!
+@brief SAX interface
+
+This class describes the SAX interface used by @ref nlohmann::json::sax_parse.
+Each function is called in different situations while the input is parsed. The
+boolean return value informs the parser whether to continue processing the
+input.
+*/
+template<typename BasicJsonType>
+struct json_sax
+{
+    /// type for (signed) integers
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    /// type for unsigned integers
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    /// type for floating-point numbers
+    using number_float_t = typename BasicJsonType::number_float_t;
+    /// type for strings
+    using string_t = typename BasicJsonType::string_t;
+
+    /*!
+    @brief a null value was read
+    @return whether parsing should proceed
+    */
+    virtual bool null() = 0;
+
+    /*!
+    @brief a boolean value was read
+    @param[in] val  boolean value
+    @return whether parsing should proceed
+    */
+    virtual bool boolean(bool val) = 0;
+
+    /*!
+    @brief an integer number was read
+    @param[in] val  integer value
+    @return whether parsing should proceed
+    */
+    virtual bool number_integer(number_integer_t val) = 0;
+
+    /*!
+    @brief an unsigned integer number was read
+    @param[in] val  unsigned integer value
+    @return whether parsing should proceed
+    */
+    virtual bool number_unsigned(number_unsigned_t val) = 0;
+
+    /*!
+    @brief an floating-point number was read
+    @param[in] val  floating-point value
+    @param[in] s    raw token value
+    @return whether parsing should proceed
+    */
+    virtual bool number_float(number_float_t val, const string_t& s) = 0;
+
+    /*!
+    @brief a string was read
+    @param[in] val  string value
+    @return whether parsing should proceed
+    @note It is safe to move the passed string.
+    */
+    virtual bool string(string_t& val) = 0;
+
+    /*!
+    @brief the beginning of an object was read
+    @param[in] elements  number of object elements or -1 if unknown
+    @return whether parsing should proceed
+    @note binary formats may report the number of elements
+    */
+    virtual bool start_object(std::size_t elements) = 0;
+
+    /*!
+    @brief an object key was read
+    @param[in] val  object key
+    @return whether parsing should proceed
+    @note It is safe to move the passed string.
+    */
+    virtual bool key(string_t& val) = 0;
+
+    /*!
+    @brief the end of an object was read
+    @return whether parsing should proceed
+    */
+    virtual bool end_object() = 0;
+
+    /*!
+    @brief the beginning of an array was read
+    @param[in] elements  number of array elements or -1 if unknown
+    @return whether parsing should proceed
+    @note binary formats may report the number of elements
+    */
+    virtual bool start_array(std::size_t elements) = 0;
+
+    /*!
+    @brief the end of an array was read
+    @return whether parsing should proceed
+    */
+    virtual bool end_array() = 0;
+
+    /*!
+    @brief a parse error occurred
+    @param[in] position    the position in the input where the error occurs
+    @param[in] last_token  the last read token
+    @param[in] ex          an exception object describing the error
+    @return whether parsing should proceed (must return false)
+    */
+    virtual bool parse_error(std::size_t position,
+                             const std::string& last_token,
+                             const detail::exception& ex) = 0;
+
+    virtual ~json_sax() = default;
+};
+
+
+namespace detail
+{
+/*!
+@brief SAX implementation to create a JSON value from SAX events
+
+This class implements the @ref json_sax interface and processes the SAX events
+to create a JSON value which makes it basically a DOM parser. The structure or
+hierarchy of the JSON value is managed by the stack `ref_stack` which contains
+a pointer to the respective array or object for each recursion depth.
+
+After successful parsing, the value that is passed by reference to the
+constructor contains the parsed value.
+
+@tparam BasicJsonType  the JSON type
+*/
+template<typename BasicJsonType>
+class json_sax_dom_parser
+{
+  public:
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+
+    /*!
+    @param[in, out] r  reference to a JSON value that is manipulated while
+                       parsing
+    @param[in] allow_exceptions_  whether parse errors yield exceptions
+    */
+    explicit json_sax_dom_parser(BasicJsonType& r, const bool allow_exceptions_ = true)
+        : root(r), allow_exceptions(allow_exceptions_)
+    {}
+
+    bool null()
+    {
+        handle_value(nullptr);
+        return true;
+    }
+
+    bool boolean(bool val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool number_integer(number_integer_t val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool number_unsigned(number_unsigned_t val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool number_float(number_float_t val, const string_t& /*unused*/)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool string(string_t& val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool start_object(std::size_t len)
+    {
+        ref_stack.push_back(handle_value(BasicJsonType::value_t::object));
+
+        if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size()))
+        {
+            JSON_THROW(out_of_range::create(408,
+                                            "excessive object size: " + std::to_string(len)));
+        }
+
+        return true;
+    }
+
+    bool key(string_t& val)
+    {
+        // add null at given key and store the reference for later
+        object_element = &(ref_stack.back()->m_value.object->operator[](val));
+        return true;
+    }
+
+    bool end_object()
+    {
+        ref_stack.pop_back();
+        return true;
+    }
+
+    bool start_array(std::size_t len)
+    {
+        ref_stack.push_back(handle_value(BasicJsonType::value_t::array));
+
+        if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size()))
+        {
+            JSON_THROW(out_of_range::create(408,
+                                            "excessive array size: " + std::to_string(len)));
+        }
+
+        return true;
+    }
+
+    bool end_array()
+    {
+        ref_stack.pop_back();
+        return true;
+    }
+
+    bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/,
+                     const detail::exception& ex)
+    {
+        errored = true;
+        if (allow_exceptions)
+        {
+            // determine the proper exception type from the id
+            switch ((ex.id / 100) % 100)
+            {
+                case 1:
+                    JSON_THROW(*reinterpret_cast<const detail::parse_error*>(&ex));
+                case 4:
+                    JSON_THROW(*reinterpret_cast<const detail::out_of_range*>(&ex));
+                // LCOV_EXCL_START
+                case 2:
+                    JSON_THROW(*reinterpret_cast<const detail::invalid_iterator*>(&ex));
+                case 3:
+                    JSON_THROW(*reinterpret_cast<const detail::type_error*>(&ex));
+                case 5:
+                    JSON_THROW(*reinterpret_cast<const detail::other_error*>(&ex));
+                default:
+                    assert(false);
+                    // LCOV_EXCL_STOP
+            }
+        }
+        return false;
+    }
+
+    constexpr bool is_errored() const
+    {
+        return errored;
+    }
+
+  private:
+    /*!
+    @invariant If the ref stack is empty, then the passed value will be the new
+               root.
+    @invariant If the ref stack contains a value, then it is an array or an
+               object to which we can add elements
+    */
+    template<typename Value>
+    BasicJsonType* handle_value(Value&& v)
+    {
+        if (ref_stack.empty())
+        {
+            root = BasicJsonType(std::forward<Value>(v));
+            return &root;
+        }
+
+        assert(ref_stack.back()->is_array() or ref_stack.back()->is_object());
+
+        if (ref_stack.back()->is_array())
+        {
+            ref_stack.back()->m_value.array->emplace_back(std::forward<Value>(v));
+            return &(ref_stack.back()->m_value.array->back());
+        }
+        else
+        {
+            assert(object_element);
+            *object_element = BasicJsonType(std::forward<Value>(v));
+            return object_element;
+        }
+    }
+
+    /// the parsed JSON value
+    BasicJsonType& root;
+    /// stack to model hierarchy of values
+    std::vector<BasicJsonType*> ref_stack;
+    /// helper to hold the reference for the next object element
+    BasicJsonType* object_element = nullptr;
+    /// whether a syntax error occurred
+    bool errored = false;
+    /// whether to throw exceptions in case of errors
+    const bool allow_exceptions = true;
+};
+
+template<typename BasicJsonType>
+class json_sax_dom_callback_parser
+{
+  public:
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+    using parser_callback_t = typename BasicJsonType::parser_callback_t;
+    using parse_event_t = typename BasicJsonType::parse_event_t;
+
+    json_sax_dom_callback_parser(BasicJsonType& r,
+                                 const parser_callback_t cb,
+                                 const bool allow_exceptions_ = true)
+        : root(r), callback(cb), allow_exceptions(allow_exceptions_)
+    {
+        keep_stack.push_back(true);
+    }
+
+    bool null()
+    {
+        handle_value(nullptr);
+        return true;
+    }
+
+    bool boolean(bool val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool number_integer(number_integer_t val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool number_unsigned(number_unsigned_t val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool number_float(number_float_t val, const string_t& /*unused*/)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool string(string_t& val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool start_object(std::size_t len)
+    {
+        // check callback for object start
+        const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::object_start, discarded);
+        keep_stack.push_back(keep);
+
+        auto val = handle_value(BasicJsonType::value_t::object, true);
+        ref_stack.push_back(val.second);
+
+        // check object limit
+        if (ref_stack.back())
+        {
+            if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size()))
+            {
+                JSON_THROW(out_of_range::create(408,
+                                                "excessive object size: " + std::to_string(len)));
+            }
+        }
+
+        return true;
+    }
+
+    bool key(string_t& val)
+    {
+        BasicJsonType k = BasicJsonType(val);
+
+        // check callback for key
+        const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::key, k);
+        key_keep_stack.push_back(keep);
+
+        // add discarded value at given key and store the reference for later
+        if (keep and ref_stack.back())
+        {
+            object_element = &(ref_stack.back()->m_value.object->operator[](val) = discarded);
+        }
+
+        return true;
+    }
+
+    bool end_object()
+    {
+        if (ref_stack.back())
+        {
+            if (not callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::object_end, *ref_stack.back()))
+            {
+                // discard object
+                *ref_stack.back() = discarded;
+            }
+        }
+
+        assert(not ref_stack.empty());
+        assert(not keep_stack.empty());
+        ref_stack.pop_back();
+        keep_stack.pop_back();
+
+        if (not ref_stack.empty() and ref_stack.back())
+        {
+            // remove discarded value
+            if (ref_stack.back()->is_object())
+            {
+                for (auto it = ref_stack.back()->begin(); it != ref_stack.back()->end(); ++it)
+                {
+                    if (it->is_discarded())
+                    {
+                        ref_stack.back()->erase(it);
+                        break;
+                    }
+                }
+            }
+        }
+
+        return true;
+    }
+
+    bool start_array(std::size_t len)
+    {
+        const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::array_start, discarded);
+        keep_stack.push_back(keep);
+
+        auto val = handle_value(BasicJsonType::value_t::array, true);
+        ref_stack.push_back(val.second);
+
+        // check array limit
+        if (ref_stack.back())
+        {
+            if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size()))
+            {
+                JSON_THROW(out_of_range::create(408,
+                                                "excessive array size: " + std::to_string(len)));
+            }
+        }
+
+        return true;
+    }
+
+    bool end_array()
+    {
+        bool keep = true;
+
+        if (ref_stack.back())
+        {
+            keep = callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::array_end, *ref_stack.back());
+            if (not keep)
+            {
+                // discard array
+                *ref_stack.back() = discarded;
+            }
+        }
+
+        assert(not ref_stack.empty());
+        assert(not keep_stack.empty());
+        ref_stack.pop_back();
+        keep_stack.pop_back();
+
+        // remove discarded value
+        if (not keep and not ref_stack.empty())
+        {
+            if (ref_stack.back()->is_array())
+            {
+                ref_stack.back()->m_value.array->pop_back();
+            }
+        }
+
+        return true;
+    }
+
+    bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/,
+                     const detail::exception& ex)
+    {
+        errored = true;
+        if (allow_exceptions)
+        {
+            // determine the proper exception type from the id
+            switch ((ex.id / 100) % 100)
+            {
+                case 1:
+                    JSON_THROW(*reinterpret_cast<const detail::parse_error*>(&ex));
+                case 4:
+                    JSON_THROW(*reinterpret_cast<const detail::out_of_range*>(&ex));
+                // LCOV_EXCL_START
+                case 2:
+                    JSON_THROW(*reinterpret_cast<const detail::invalid_iterator*>(&ex));
+                case 3:
+                    JSON_THROW(*reinterpret_cast<const detail::type_error*>(&ex));
+                case 5:
+                    JSON_THROW(*reinterpret_cast<const detail::other_error*>(&ex));
+                default:
+                    assert(false);
+                    // LCOV_EXCL_STOP
+            }
+        }
+        return false;
+    }
+
+    constexpr bool is_errored() const
+    {
+        return errored;
+    }
+
+  private:
+    /*!
+    @param[in] v  value to add to the JSON value we build during parsing
+    @param[in] skip_callback  whether we should skip calling the callback
+               function; this is required after start_array() and
+               start_object() SAX events, because otherwise we would call the
+               callback function with an empty array or object, respectively.
+
+    @invariant If the ref stack is empty, then the passed value will be the new
+               root.
+    @invariant If the ref stack contains a value, then it is an array or an
+               object to which we can add elements
+
+    @return pair of boolean (whether value should be kept) and pointer (to the
+            passed value in the ref_stack hierarchy; nullptr if not kept)
+    */
+    template<typename Value>
+    std::pair<bool, BasicJsonType*> handle_value(Value&& v, const bool skip_callback = false)
+    {
+        assert(not keep_stack.empty());
+
+        // do not handle this value if we know it would be added to a discarded
+        // container
+        if (not keep_stack.back())
+        {
+            return {false, nullptr};
+        }
+
+        // create value
+        auto value = BasicJsonType(std::forward<Value>(v));
+
+        // check callback
+        const bool keep = skip_callback or callback(static_cast<int>(ref_stack.size()), parse_event_t::value, value);
+
+        // do not handle this value if we just learnt it shall be discarded
+        if (not keep)
+        {
+            return {false, nullptr};
+        }
+
+        if (ref_stack.empty())
+        {
+            root = std::move(value);
+            return {true, &root};
+        }
+
+        // skip this value if we already decided to skip the parent
+        // (https://github.com/nlohmann/json/issues/971#issuecomment-413678360)
+        if (not ref_stack.back())
+        {
+            return {false, nullptr};
+        }
+
+        // we now only expect arrays and objects
+        assert(ref_stack.back()->is_array() or ref_stack.back()->is_object());
+
+        if (ref_stack.back()->is_array())
+        {
+            ref_stack.back()->m_value.array->push_back(std::move(value));
+            return {true, &(ref_stack.back()->m_value.array->back())};
+        }
+        else
+        {
+            // check if we should store an element for the current key
+            assert(not key_keep_stack.empty());
+            const bool store_element = key_keep_stack.back();
+            key_keep_stack.pop_back();
+
+            if (not store_element)
+            {
+                return {false, nullptr};
+            }
+
+            assert(object_element);
+            *object_element = std::move(value);
+            return {true, object_element};
+        }
+    }
+
+    /// the parsed JSON value
+    BasicJsonType& root;
+    /// stack to model hierarchy of values
+    std::vector<BasicJsonType*> ref_stack;
+    /// stack to manage which values to keep
+    std::vector<bool> keep_stack;
+    /// stack to manage which object keys to keep
+    std::vector<bool> key_keep_stack;
+    /// helper to hold the reference for the next object element
+    BasicJsonType* object_element = nullptr;
+    /// whether a syntax error occurred
+    bool errored = false;
+    /// callback function
+    const parser_callback_t callback = nullptr;
+    /// whether to throw exceptions in case of errors
+    const bool allow_exceptions = true;
+    /// a discarded value for the callback
+    BasicJsonType discarded = BasicJsonType::value_t::discarded;
+};
+
+template<typename BasicJsonType>
+class json_sax_acceptor
+{
+  public:
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+
+    bool null()
+    {
+        return true;
+    }
+
+    bool boolean(bool /*unused*/)
+    {
+        return true;
+    }
+
+    bool number_integer(number_integer_t /*unused*/)
+    {
+        return true;
+    }
+
+    bool number_unsigned(number_unsigned_t /*unused*/)
+    {
+        return true;
+    }
+
+    bool number_float(number_float_t /*unused*/, const string_t& /*unused*/)
+    {
+        return true;
+    }
+
+    bool string(string_t& /*unused*/)
+    {
+        return true;
+    }
+
+    bool start_object(std::size_t  /*unused*/ = std::size_t(-1))
+    {
+        return true;
+    }
+
+    bool key(string_t& /*unused*/)
+    {
+        return true;
+    }
+
+    bool end_object()
+    {
+        return true;
+    }
+
+    bool start_array(std::size_t  /*unused*/ = std::size_t(-1))
+    {
+        return true;
+    }
+
+    bool end_array()
+    {
+        return true;
+    }
+
+    bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const detail::exception& /*unused*/)
+    {
+        return false;
+    }
+};
+}  // namespace detail
+
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/input/lexer.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+////////////
+// parser //
+////////////
 
 /*!
 @brief syntax analysis
@@ -2892,6 +4962,7 @@ class parser
     using number_integer_t = typename BasicJsonType::number_integer_t;
     using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
     using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
     using lexer_t = lexer<BasicJsonType>;
     using token_type = typename lexer_t::token_type;
 
@@ -2916,11 +4987,14 @@ class parser
         std::function<bool(int depth, parse_event_t event, BasicJsonType& parsed)>;
 
     /// a parser reading from an input adapter
-    explicit parser(detail::input_adapter_t adapter,
+    explicit parser(detail::input_adapter_t&& adapter,
                     const parser_callback_t cb = nullptr,
                     const bool allow_exceptions_ = true)
-        : callback(cb), m_lexer(adapter), allow_exceptions(allow_exceptions_)
-    {}
+        : callback(cb), m_lexer(std::move(adapter)), allow_exceptions(allow_exceptions_)
+    {
+        // read first token
+        get_token();
+    }
 
     /*!
     @brief public parser interface
@@ -2934,31 +5008,56 @@ class parser
     */
     void parse(const bool strict, BasicJsonType& result)
     {
-        // read first token
-        get_token();
+        if (callback)
+        {
+            json_sax_dom_callback_parser<BasicJsonType> sdp(result, callback, allow_exceptions);
+            sax_parse_internal(&sdp);
+            result.assert_invariant();
 
-        parse_internal(true, result);
-        result.assert_invariant();
+            // in strict mode, input must be completely read
+            if (strict and (get_token() != token_type::end_of_input))
+            {
+                sdp.parse_error(m_lexer.get_position(),
+                                m_lexer.get_token_string(),
+                                parse_error::create(101, m_lexer.get_position(),
+                                                    exception_message(token_type::end_of_input, "value")));
+            }
 
-        // in strict mode, input must be completely read
-        if (strict)
-        {
-            get_token();
-            expect(token_type::end_of_input);
-        }
+            // in case of an error, return discarded value
+            if (sdp.is_errored())
+            {
+                result = value_t::discarded;
+                return;
+            }
 
-        // in case of an error, return discarded value
-        if (errored)
-        {
-            result = value_t::discarded;
-            return;
+            // set top-level value to null if it was discarded by the callback
+            // function
+            if (result.is_discarded())
+            {
+                result = nullptr;
+            }
         }
-
-        // set top-level value to null if it was discarded by the callback
-        // function
-        if (result.is_discarded())
+        else
         {
-            result = nullptr;
+            json_sax_dom_parser<BasicJsonType> sdp(result, allow_exceptions);
+            sax_parse_internal(&sdp);
+            result.assert_invariant();
+
+            // in strict mode, input must be completely read
+            if (strict and (get_token() != token_type::end_of_input))
+            {
+                sdp.parse_error(m_lexer.get_position(),
+                                m_lexer.get_token_string(),
+                                parse_error::create(101, m_lexer.get_position(),
+                                                    exception_message(token_type::end_of_input, "value")));
+            }
+
+            // in case of an error, return discarded value
+            if (sdp.is_errored())
+            {
+                result = value_t::discarded;
+                return;
+            }
         }
     }
 
@@ -2970,413 +5069,317 @@ class parser
     */
     bool accept(const bool strict = true)
     {
-        // read first token
-        get_token();
+        json_sax_acceptor<BasicJsonType> sax_acceptor;
+        return sax_parse(&sax_acceptor, strict);
+    }
 
-        if (not accept_internal())
+    template <typename SAX>
+    bool sax_parse(SAX* sax, const bool strict = true)
+    {
+        (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {};
+        const bool result = sax_parse_internal(sax);
+
+        // strict mode: next byte must be EOF
+        if (result and strict and (get_token() != token_type::end_of_input))
         {
-            return false;
+            return sax->parse_error(m_lexer.get_position(),
+                                    m_lexer.get_token_string(),
+                                    parse_error::create(101, m_lexer.get_position(),
+                                            exception_message(token_type::end_of_input, "value")));
         }
 
-        // strict => last token must be EOF
-        return not strict or (get_token() == token_type::end_of_input);
+        return result;
     }
 
   private:
-    /*!
-    @brief the actual parser
-    @throw parse_error.101 in case of an unexpected token
-    @throw parse_error.102 if to_unicode fails or surrogate error
-    @throw parse_error.103 if to_unicode fails
-    */
-    void parse_internal(bool keep, BasicJsonType& result)
+    template <typename SAX>
+    bool sax_parse_internal(SAX* sax)
     {
-        // never parse after a parse error was detected
-        assert(not errored);
+        // stack to remember the hierarchy of structured values we are parsing
+        // true = array; false = object
+        std::vector<bool> states;
+        // value to avoid a goto (see comment where set to true)
+        bool skip_to_state_evaluation = false;
 
-        // start with a discarded value
-        if (not result.is_discarded())
-        {
-            result.m_value.destroy(result.m_type);
-            result.m_type = value_t::discarded;
-        }
-
-        switch (last_token)
+        while (true)
         {
-            case token_type::begin_object:
+            if (not skip_to_state_evaluation)
             {
-                if (keep)
+                // invariant: get_token() was called before each iteration
+                switch (last_token)
                 {
-                    if (callback)
+                    case token_type::begin_object:
                     {
-                        keep = callback(depth++, parse_event_t::object_start, result);
-                    }
+                        if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1))))
+                        {
+                            return false;
+                        }
 
-                    if (not callback or keep)
-                    {
-                        // explicitly set result to object to cope with {}
-                        result.m_type = value_t::object;
-                        result.m_value = value_t::object;
-                    }
-                }
+                        // closing } -> we are done
+                        if (get_token() == token_type::end_object)
+                        {
+                            if (JSON_UNLIKELY(not sax->end_object()))
+                            {
+                                return false;
+                            }
+                            break;
+                        }
 
-                // read next token
-                get_token();
+                        // parse key
+                        if (JSON_UNLIKELY(last_token != token_type::value_string))
+                        {
+                            return sax->parse_error(m_lexer.get_position(),
+                                                    m_lexer.get_token_string(),
+                                                    parse_error::create(101, m_lexer.get_position(),
+                                                            exception_message(token_type::value_string, "object key")));
+                        }
+                        if (JSON_UNLIKELY(not sax->key(m_lexer.get_string())))
+                        {
+                            return false;
+                        }
 
-                // closing } -> we are done
-                if (last_token == token_type::end_object)
-                {
-                    if (keep and callback and not callback(--depth, parse_event_t::object_end, result))
-                    {
-                        result.m_value.destroy(result.m_type);
-                        result.m_type = value_t::discarded;
-                    }
-                    break;
-                }
+                        // parse separator (:)
+                        if (JSON_UNLIKELY(get_token() != token_type::name_separator))
+                        {
+                            return sax->parse_error(m_lexer.get_position(),
+                                                    m_lexer.get_token_string(),
+                                                    parse_error::create(101, m_lexer.get_position(),
+                                                            exception_message(token_type::name_separator, "object separator")));
+                        }
 
-                // parse values
-                std::string key;
-                BasicJsonType value;
-                while (true)
-                {
-                    // store key
-                    if (not expect(token_type::value_string))
-                    {
-                        return;
+                        // remember we are now inside an object
+                        states.push_back(false);
+
+                        // parse values
+                        get_token();
+                        continue;
                     }
-                    key = m_lexer.move_string();
 
-                    bool keep_tag = false;
-                    if (keep)
+                    case token_type::begin_array:
                     {
-                        if (callback)
+                        if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1))))
                         {
-                            BasicJsonType k(key);
-                            keep_tag = callback(depth, parse_event_t::key, k);
+                            return false;
                         }
-                        else
+
+                        // closing ] -> we are done
+                        if (get_token() == token_type::end_array)
                         {
-                            keep_tag = true;
+                            if (JSON_UNLIKELY(not sax->end_array()))
+                            {
+                                return false;
+                            }
+                            break;
                         }
-                    }
-
-                    // parse separator (:)
-                    get_token();
-                    if (not expect(token_type::name_separator))
-                    {
-                        return;
-                    }
 
-                    // parse and add value
-                    get_token();
-                    value.m_value.destroy(value.m_type);
-                    value.m_type = value_t::discarded;
-                    parse_internal(keep, value);
+                        // remember we are now inside an array
+                        states.push_back(true);
 
-                    if (JSON_UNLIKELY(errored))
-                    {
-                        return;
+                        // parse values (no need to call get_token)
+                        continue;
                     }
 
-                    if (keep and keep_tag and not value.is_discarded())
+                    case token_type::value_float:
                     {
-                        result.m_value.object->emplace(std::move(key), std::move(value));
-                    }
+                        const auto res = m_lexer.get_number_float();
 
-                    // comma -> next value
-                    get_token();
-                    if (last_token == token_type::value_separator)
-                    {
-                        get_token();
-                        continue;
+                        if (JSON_UNLIKELY(not std::isfinite(res)))
+                        {
+                            return sax->parse_error(m_lexer.get_position(),
+                                                    m_lexer.get_token_string(),
+                                                    out_of_range::create(406, "number overflow parsing '" + m_lexer.get_token_string() + "'"));
+                        }
+                        else
+                        {
+                            if (JSON_UNLIKELY(not sax->number_float(res, m_lexer.get_string())))
+                            {
+                                return false;
+                            }
+                            break;
+                        }
                     }
 
-                    // closing }
-                    if (not expect(token_type::end_object))
+                    case token_type::literal_false:
                     {
-                        return;
+                        if (JSON_UNLIKELY(not sax->boolean(false)))
+                        {
+                            return false;
+                        }
+                        break;
                     }
-                    break;
-                }
 
-                if (keep and callback and not callback(--depth, parse_event_t::object_end, result))
-                {
-                    result.m_value.destroy(result.m_type);
-                    result.m_type = value_t::discarded;
-                }
-                break;
-            }
-
-            case token_type::begin_array:
-            {
-                if (keep)
-                {
-                    if (callback)
+                    case token_type::literal_null:
                     {
-                        keep = callback(depth++, parse_event_t::array_start, result);
+                        if (JSON_UNLIKELY(not sax->null()))
+                        {
+                            return false;
+                        }
+                        break;
                     }
 
-                    if (not callback or keep)
+                    case token_type::literal_true:
                     {
-                        // explicitly set result to array to cope with []
-                        result.m_type = value_t::array;
-                        result.m_value = value_t::array;
+                        if (JSON_UNLIKELY(not sax->boolean(true)))
+                        {
+                            return false;
+                        }
+                        break;
                     }
-                }
-
-                // read next token
-                get_token();
 
-                // closing ] -> we are done
-                if (last_token == token_type::end_array)
-                {
-                    if (callback and not callback(--depth, parse_event_t::array_end, result))
+                    case token_type::value_integer:
                     {
-                        result.m_value.destroy(result.m_type);
-                        result.m_type = value_t::discarded;
+                        if (JSON_UNLIKELY(not sax->number_integer(m_lexer.get_number_integer())))
+                        {
+                            return false;
+                        }
+                        break;
                     }
-                    break;
-                }
 
-                // parse values
-                BasicJsonType value;
-                while (true)
-                {
-                    // parse value
-                    value.m_value.destroy(value.m_type);
-                    value.m_type = value_t::discarded;
-                    parse_internal(keep, value);
-
-                    if (JSON_UNLIKELY(errored))
+                    case token_type::value_string:
                     {
-                        return;
+                        if (JSON_UNLIKELY(not sax->string(m_lexer.get_string())))
+                        {
+                            return false;
+                        }
+                        break;
                     }
 
-                    if (keep and not value.is_discarded())
+                    case token_type::value_unsigned:
                     {
-                        result.m_value.array->push_back(std::move(value));
+                        if (JSON_UNLIKELY(not sax->number_unsigned(m_lexer.get_number_unsigned())))
+                        {
+                            return false;
+                        }
+                        break;
                     }
 
-                    // comma -> next value
-                    get_token();
-                    if (last_token == token_type::value_separator)
+                    case token_type::parse_error:
                     {
-                        get_token();
-                        continue;
+                        // using "uninitialized" to avoid "expected" message
+                        return sax->parse_error(m_lexer.get_position(),
+                                                m_lexer.get_token_string(),
+                                                parse_error::create(101, m_lexer.get_position(),
+                                                        exception_message(token_type::uninitialized, "value")));
                     }
 
-                    // closing ]
-                    if (not expect(token_type::end_array))
+                    default: // the last token was unexpected
                     {
-                        return;
+                        return sax->parse_error(m_lexer.get_position(),
+                                                m_lexer.get_token_string(),
+                                                parse_error::create(101, m_lexer.get_position(),
+                                                        exception_message(token_type::literal_or_value, "value")));
                     }
-                    break;
-                }
-
-                if (keep and callback and not callback(--depth, parse_event_t::array_end, result))
-                {
-                    result.m_value.destroy(result.m_type);
-                    result.m_type = value_t::discarded;
                 }
-                break;
-            }
-
-            case token_type::literal_null:
-            {
-                result.m_type = value_t::null;
-                break;
-            }
-
-            case token_type::value_string:
-            {
-                result.m_type = value_t::string;
-                result.m_value = m_lexer.move_string();
-                break;
-            }
-
-            case token_type::literal_true:
-            {
-                result.m_type = value_t::boolean;
-                result.m_value = true;
-                break;
-            }
-
-            case token_type::literal_false:
-            {
-                result.m_type = value_t::boolean;
-                result.m_value = false;
-                break;
             }
-
-            case token_type::value_unsigned:
+            else
             {
-                result.m_type = value_t::number_unsigned;
-                result.m_value = m_lexer.get_number_unsigned();
-                break;
+                skip_to_state_evaluation = false;
             }
 
-            case token_type::value_integer:
+            // we reached this line after we successfully parsed a value
+            if (states.empty())
             {
-                result.m_type = value_t::number_integer;
-                result.m_value = m_lexer.get_number_integer();
-                break;
+                // empty stack: we reached the end of the hierarchy: done
+                return true;
             }
-
-            case token_type::value_float:
+            else
             {
-                result.m_type = value_t::number_float;
-                result.m_value = m_lexer.get_number_float();
-
-                // throw in case of infinity or NAN
-                if (JSON_UNLIKELY(not std::isfinite(result.m_value.number_float)))
+                if (states.back())  // array
                 {
-                    if (allow_exceptions)
+                    // comma -> next value
+                    if (get_token() == token_type::value_separator)
                     {
-                        JSON_THROW(out_of_range::create(406, "number overflow parsing '" +
-                                                        m_lexer.get_token_string() + "'"));
+                        // parse a new value
+                        get_token();
+                        continue;
                     }
-                    expect(token_type::uninitialized);
-                }
-                break;
-            }
-
-            case token_type::parse_error:
-            {
-                // using "uninitialized" to avoid "expected" message
-                if (not expect(token_type::uninitialized))
-                {
-                    return;
-                }
-                break; // LCOV_EXCL_LINE
-            }
-
-            default:
-            {
-                // the last token was unexpected; we expected a value
-                if (not expect(token_type::literal_or_value))
-                {
-                    return;
-                }
-                break; // LCOV_EXCL_LINE
-            }
-        }
-
-        if (keep and callback and not callback(depth, parse_event_t::value, result))
-        {
-            result.m_type = value_t::discarded;
-        }
-    }
-
-    /*!
-    @brief the actual acceptor
-
-    @invariant 1. The last token is not yet processed. Therefore, the caller
-                  of this function must make sure a token has been read.
-               2. When this function returns, the last token is processed.
-                  That is, the last read character was already considered.
-
-    This invariant makes sure that no token needs to be "unput".
-    */
-    bool accept_internal()
-    {
-        switch (last_token)
-        {
-            case token_type::begin_object:
-            {
-                // read next token
-                get_token();
 
-                // closing } -> we are done
-                if (last_token == token_type::end_object)
-                {
-                    return true;
-                }
-
-                // parse values
-                while (true)
-                {
-                    // parse key
-                    if (last_token != token_type::value_string)
+                    // closing ]
+                    if (JSON_LIKELY(last_token == token_type::end_array))
                     {
-                        return false;
-                    }
+                        if (JSON_UNLIKELY(not sax->end_array()))
+                        {
+                            return false;
+                        }
 
-                    // parse separator (:)
-                    get_token();
-                    if (last_token != token_type::name_separator)
-                    {
-                        return false;
+                        // We are done with this array. Before we can parse a
+                        // new value, we need to evaluate the new state first.
+                        // By setting skip_to_state_evaluation to false, we
+                        // are effectively jumping to the beginning of this if.
+                        assert(not states.empty());
+                        states.pop_back();
+                        skip_to_state_evaluation = true;
+                        continue;
                     }
-
-                    // parse value
-                    get_token();
-                    if (not accept_internal())
+                    else
                     {
-                        return false;
+                        return sax->parse_error(m_lexer.get_position(),
+                                                m_lexer.get_token_string(),
+                                                parse_error::create(101, m_lexer.get_position(),
+                                                        exception_message(token_type::end_array, "array")));
                     }
-
+                }
+                else  // object
+                {
                     // comma -> next value
-                    get_token();
-                    if (last_token == token_type::value_separator)
+                    if (get_token() == token_type::value_separator)
                     {
+                        // parse key
+                        if (JSON_UNLIKELY(get_token() != token_type::value_string))
+                        {
+                            return sax->parse_error(m_lexer.get_position(),
+                                                    m_lexer.get_token_string(),
+                                                    parse_error::create(101, m_lexer.get_position(),
+                                                            exception_message(token_type::value_string, "object key")));
+                        }
+                        else
+                        {
+                            if (JSON_UNLIKELY(not sax->key(m_lexer.get_string())))
+                            {
+                                return false;
+                            }
+                        }
+
+                        // parse separator (:)
+                        if (JSON_UNLIKELY(get_token() != token_type::name_separator))
+                        {
+                            return sax->parse_error(m_lexer.get_position(),
+                                                    m_lexer.get_token_string(),
+                                                    parse_error::create(101, m_lexer.get_position(),
+                                                            exception_message(token_type::name_separator, "object separator")));
+                        }
+
+                        // parse values
                         get_token();
                         continue;
                     }
 
                     // closing }
-                    return (last_token == token_type::end_object);
-                }
-            }
-
-            case token_type::begin_array:
-            {
-                // read next token
-                get_token();
-
-                // closing ] -> we are done
-                if (last_token == token_type::end_array)
-                {
-                    return true;
-                }
-
-                // parse values
-                while (true)
-                {
-                    // parse value
-                    if (not accept_internal())
+                    if (JSON_LIKELY(last_token == token_type::end_object))
                     {
-                        return false;
-                    }
+                        if (JSON_UNLIKELY(not sax->end_object()))
+                        {
+                            return false;
+                        }
 
-                    // comma -> next value
-                    get_token();
-                    if (last_token == token_type::value_separator)
-                    {
-                        get_token();
+                        // We are done with this object. Before we can parse a
+                        // new value, we need to evaluate the new state first.
+                        // By setting skip_to_state_evaluation to false, we
+                        // are effectively jumping to the beginning of this if.
+                        assert(not states.empty());
+                        states.pop_back();
+                        skip_to_state_evaluation = true;
                         continue;
                     }
-
-                    // closing ]
-                    return (last_token == token_type::end_array);
+                    else
+                    {
+                        return sax->parse_error(m_lexer.get_position(),
+                                                m_lexer.get_token_string(),
+                                                parse_error::create(101, m_lexer.get_position(),
+                                                        exception_message(token_type::end_object, "object")));
+                    }
                 }
             }
-
-            case token_type::value_float:
-            {
-                // reject infinity or NAN
-                return std::isfinite(m_lexer.get_number_float());
-            }
-
-            case token_type::literal_false:
-            case token_type::literal_null:
-            case token_type::literal_true:
-            case token_type::value_integer:
-            case token_type::value_string:
-            case token_type::value_unsigned:
-                return true;
-
-            default: // the last token was unexpected
-                return false;
         }
     }
 
@@ -3386,31 +5389,17 @@ class parser
         return (last_token = m_lexer.scan());
     }
 
-    /*!
-    @throw parse_error.101 if expected token did not occur
-    */
-    bool expect(token_type t)
+    std::string exception_message(const token_type expected, const std::string& context)
     {
-        if (JSON_UNLIKELY(t != last_token))
+        std::string error_msg = "syntax error ";
+
+        if (not context.empty())
         {
-            errored = true;
-            expected = t;
-            if (allow_exceptions)
-            {
-                throw_exception();
-            }
-            else
-            {
-                return false;
-            }
+            error_msg += "while parsing " + context + " ";
         }
 
-        return true;
-    }
+        error_msg += "- ";
 
-    [[noreturn]] void throw_exception() const
-    {
-        std::string error_msg = "syntax error - ";
         if (last_token == token_type::parse_error)
         {
             error_msg += std::string(m_lexer.get_error_message()) + "; last read: '" +
@@ -3426,31 +5415,33 @@ class parser
             error_msg += "; expected " + std::string(lexer_t::token_type_name(expected));
         }
 
-        JSON_THROW(parse_error::create(101, m_lexer.get_position(), error_msg));
+        return error_msg;
     }
 
   private:
-    /// current level of recursion
-    int depth = 0;
     /// callback function
     const parser_callback_t callback = nullptr;
     /// the type of the last read token
     token_type last_token = token_type::uninitialized;
     /// the lexer
     lexer_t m_lexer;
-    /// whether a syntax error occurred
-    bool errored = false;
-    /// possible reason for the syntax error
-    token_type expected = token_type::uninitialized;
     /// whether to throw exceptions in case of errors
     const bool allow_exceptions = true;
 };
+}  // namespace detail
+}  // namespace nlohmann
 
-///////////////
-// iterators //
-///////////////
+// #include <nlohmann/detail/iterators/primitive_iterator.hpp>
 
-/*!
+
+#include <cstddef> // ptrdiff_t
+#include <limits>  // numeric_limits
+
+namespace nlohmann
+{
+namespace detail
+{
+/*
 @brief an iterator for primitive JSON types
 
 This class models an iterator for primitive JSON types (boolean, number,
@@ -3461,9 +5452,15 @@ end_value (`1`) models past the end.
 */
 class primitive_iterator_t
 {
-  public:
+  private:
     using difference_type = std::ptrdiff_t;
+    static constexpr difference_type begin_value = 0;
+    static constexpr difference_type end_value = begin_value + 1;
 
+    /// iterator as signed integer type
+    difference_type m_it = (std::numeric_limits<std::ptrdiff_t>::min)();
+
+  public:
     constexpr difference_type get_value() const noexcept
     {
         return m_it;
@@ -3503,10 +5500,10 @@ class primitive_iterator_t
         return lhs.m_it < rhs.m_it;
     }
 
-    primitive_iterator_t operator+(difference_type i)
+    primitive_iterator_t operator+(difference_type n) noexcept
     {
         auto result = *this;
-        result += i;
+        result += n;
         return result;
     }
 
@@ -3515,57 +5512,57 @@ class primitive_iterator_t
         return lhs.m_it - rhs.m_it;
     }
 
-    friend std::ostream& operator<<(std::ostream& os, primitive_iterator_t it)
-    {
-        return os << it.m_it;
-    }
-
-    primitive_iterator_t& operator++()
+    primitive_iterator_t& operator++() noexcept
     {
         ++m_it;
         return *this;
     }
 
-    primitive_iterator_t const operator++(int)
+    primitive_iterator_t const operator++(int) noexcept
     {
         auto result = *this;
-        m_it++;
+        ++m_it;
         return result;
     }
 
-    primitive_iterator_t& operator--()
+    primitive_iterator_t& operator--() noexcept
     {
         --m_it;
         return *this;
     }
 
-    primitive_iterator_t const operator--(int)
+    primitive_iterator_t const operator--(int) noexcept
     {
         auto result = *this;
-        m_it--;
+        --m_it;
         return result;
     }
 
-    primitive_iterator_t& operator+=(difference_type n)
+    primitive_iterator_t& operator+=(difference_type n) noexcept
     {
         m_it += n;
         return *this;
     }
 
-    primitive_iterator_t& operator-=(difference_type n)
+    primitive_iterator_t& operator-=(difference_type n) noexcept
     {
         m_it -= n;
         return *this;
     }
+};
+}  // namespace detail
+}  // namespace nlohmann
 
-  private:
-    static constexpr difference_type begin_value = 0;
-    static constexpr difference_type end_value = begin_value + 1;
+// #include <nlohmann/detail/iterators/internal_iterator.hpp>
+
+
+// #include <nlohmann/detail/iterators/primitive_iterator.hpp>
 
-    /// iterator as signed integer type
-    difference_type m_it = (std::numeric_limits<std::ptrdiff_t>::min)();
-};
 
+namespace nlohmann
+{
+namespace detail
+{
 /*!
 @brief an iterator value
 
@@ -3581,26 +5578,50 @@ template<typename BasicJsonType> struct internal_iterator
     /// generic iterator for all other types
     primitive_iterator_t primitive_iterator {};
 };
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/iterators/iter_impl.hpp>
+
+
+#include <ciso646> // not
+#include <iterator> // iterator, random_access_iterator_tag, bidirectional_iterator_tag, advance, next
+#include <type_traits> // conditional, is_const, remove_const
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/iterators/internal_iterator.hpp>
 
+// #include <nlohmann/detail/iterators/primitive_iterator.hpp>
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+// forward declare, to be able to friend it later on
 template<typename IteratorType> class iteration_proxy;
+template<typename IteratorType> class iteration_proxy_value;
 
 /*!
 @brief a template for a bidirectional iterator for the @ref basic_json class
-
 This class implements a both iterators (iterator and const_iterator) for the
 @ref basic_json class.
-
 @note An iterator is called *initialized* when a pointer to a JSON value has
       been set (e.g., by a constructor or a copy assignment). If the iterator is
       default-constructed, it is *uninitialized* and most methods are undefined.
       **The library uses assertions to detect calls on uninitialized iterators.**
-
 @requirement The class satisfies the following concept requirements:
 -
-[BidirectionalIterator](http://en.cppreference.com/w/cpp/concept/BidirectionalIterator):
+[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator):
   The iterator that can be moved can be moved in both directions (i.e.
   incremented and decremented).
-
 @since version 1.0.0, simplified in version 2.0.9, change to bidirectional
        iterators in version 3.0.0 (see https://github.com/nlohmann/json/issues/593)
 */
@@ -3611,6 +5632,7 @@ class iter_impl
     friend iter_impl<typename std::conditional<std::is_const<BasicJsonType>::value, typename std::remove_const<BasicJsonType>::type, const BasicJsonType>::type>;
     friend BasicJsonType;
     friend iteration_proxy<iter_impl>;
+    friend iteration_proxy_value<iter_impl>;
 
     using object_t = typename BasicJsonType::object_t;
     using array_t = typename BasicJsonType::array_t;
@@ -4149,7 +6171,7 @@ class iter_impl
     @brief  return the key of an object iterator
     @pre The iterator is initialized; i.e. `m_object != nullptr`.
     */
-    typename object_t::key_type key() const
+    const typename object_t::key_type& key() const
     {
         assert(m_object != nullptr);
 
@@ -4174,94 +6196,26 @@ class iter_impl
     /// associated JSON instance
     pointer m_object = nullptr;
     /// the actual iterator of the associated instance
-    internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it = {};
+    internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it;
 };
+}  // namespace detail
+} // namespace nlohmann
+// #include <nlohmann/detail/iterators/iteration_proxy.hpp>
 
-/// proxy class for the iterator_wrapper functions
-template<typename IteratorType> class iteration_proxy
-{
-  private:
-    /// helper class for iteration
-    class iteration_proxy_internal
-    {
-      private:
-        /// the iterator
-        IteratorType anchor;
-        /// an index for arrays (used to create key names)
-        std::size_t array_index = 0;
-
-      public:
-        explicit iteration_proxy_internal(IteratorType it) noexcept : anchor(it) {}
-
-        /// dereference operator (needed for range-based for)
-        iteration_proxy_internal& operator*()
-        {
-            return *this;
-        }
-
-        /// increment operator (needed for range-based for)
-        iteration_proxy_internal& operator++()
-        {
-            ++anchor;
-            ++array_index;
-
-            return *this;
-        }
-
-        /// inequality operator (needed for range-based for)
-        bool operator!=(const iteration_proxy_internal& o) const noexcept
-        {
-            return anchor != o.anchor;
-        }
-
-        /// return key of the iterator
-        std::string key() const
-        {
-            assert(anchor.m_object != nullptr);
-
-            switch (anchor.m_object->type())
-            {
-                // use integer array index as key
-                case value_t::array:
-                    return std::to_string(array_index);
-
-                // use key from the object
-                case value_t::object:
-                    return anchor.key();
-
-                // use an empty key for all primitive types
-                default:
-                    return "";
-            }
-        }
-
-        /// return value of the iterator
-        typename IteratorType::reference value() const
-        {
-            return anchor.value();
-        }
-    };
-
-    /// the container to iterate
-    typename IteratorType::reference container;
+// #include <nlohmann/detail/iterators/json_reverse_iterator.hpp>
 
-  public:
-    /// construct iteration proxy from a container
-    explicit iteration_proxy(typename IteratorType::reference cont)
-        : container(cont) {}
 
-    /// return iterator begin (needed for range-based for)
-    iteration_proxy_internal begin() noexcept
-    {
-        return iteration_proxy_internal(container.begin());
-    }
+#include <cstddef> // ptrdiff_t
+#include <iterator> // reverse_iterator
+#include <utility> // declval
 
-    /// return iterator end (needed for range-based for)
-    iteration_proxy_internal end() noexcept
-    {
-        return iteration_proxy_internal(container.end());
-    }
-};
+namespace nlohmann
+{
+namespace detail
+{
+//////////////////////
+// reverse_iterator //
+//////////////////////
 
 /*!
 @brief a template for a reverse iterator class
@@ -4272,10 +6226,10 @@ create @ref const_reverse_iterator).
 
 @requirement The class satisfies the following concept requirements:
 -
-[BidirectionalIterator](http://en.cppreference.com/w/cpp/concept/BidirectionalIterator):
+[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator):
   The iterator that can be moved can be moved in both directions (i.e.
   incremented and decremented).
-- [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator):
+- [OutputIterator](https://en.cppreference.com/w/cpp/named_req/OutputIterator):
   It is possible to write to the pointed-to element (only if @a Base is
   @ref iterator).
 
@@ -4292,11 +6246,11 @@ class json_reverse_iterator : public std::reverse_iterator<Base>
     using reference = typename Base::reference;
 
     /// create reverse iterator from iterator
-    json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept
+    explicit json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept
         : base_iterator(it) {}
 
     /// create reverse iterator from base class
-    json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {}
+    explicit json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {}
 
     /// post-increment (it++)
     json_reverse_iterator const operator++(int)
@@ -4366,11 +6320,25 @@ class json_reverse_iterator : public std::reverse_iterator<Base>
         return it.operator * ();
     }
 };
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/output/output_adapters.hpp>
 
-/////////////////////
-// output adapters //
-/////////////////////
 
+#include <algorithm> // copy
+#include <cstddef> // size_t
+#include <ios> // streamsize
+#include <iterator> // back_inserter
+#include <memory> // shared_ptr, make_shared
+#include <ostream> // basic_ostream
+#include <string> // basic_string
+#include <vector> // vector
+
+namespace nlohmann
+{
+namespace detail
+{
 /// abstract output adapter interface
 template<typename CharType> struct output_adapter_protocol
 {
@@ -4388,7 +6356,9 @@ template<typename CharType>
 class output_vector_adapter : public output_adapter_protocol<CharType>
 {
   public:
-    explicit output_vector_adapter(std::vector<CharType>& vec) : v(vec) {}
+    explicit output_vector_adapter(std::vector<CharType>& vec) noexcept
+        : v(vec)
+    {}
 
     void write_character(CharType c) override
     {
@@ -4409,7 +6379,9 @@ template<typename CharType>
 class output_stream_adapter : public output_adapter_protocol<CharType>
 {
   public:
-    explicit output_stream_adapter(std::basic_ostream<CharType>& s) : stream(s) {}
+    explicit output_stream_adapter(std::basic_ostream<CharType>& s) noexcept
+        : stream(s)
+    {}
 
     void write_character(CharType c) override
     {
@@ -4426,11 +6398,13 @@ class output_stream_adapter : public output_adapter_protocol<CharType>
 };
 
 /// output adapter for basic_string
-template<typename CharType>
+template<typename CharType, typename StringType = std::basic_string<CharType>>
 class output_string_adapter : public output_adapter_protocol<CharType>
 {
   public:
-    explicit output_string_adapter(std::basic_string<CharType>& s) : str(s) {}
+    explicit output_string_adapter(StringType& s) noexcept
+        : str(s)
+    {}
 
     void write_character(CharType c) override
     {
@@ -4443,10 +6417,10 @@ class output_string_adapter : public output_adapter_protocol<CharType>
     }
 
   private:
-    std::basic_string<CharType>& str;
+    StringType& str;
 };
 
-template<typename CharType>
+template<typename CharType, typename StringType = std::basic_string<CharType>>
 class output_adapter
 {
   public:
@@ -4456,8 +6430,8 @@ class output_adapter
     output_adapter(std::basic_ostream<CharType>& s)
         : oa(std::make_shared<output_stream_adapter<CharType>>(s)) {}
 
-    output_adapter(std::basic_string<CharType>& s)
-        : oa(std::make_shared<output_string_adapter<CharType>>(s)) {}
+    output_adapter(StringType& s)
+        : oa(std::make_shared<output_string_adapter<CharType, StringType>>(s)) {}
 
     operator output_adapter_t<CharType>()
     {
@@ -4467,19 +6441,57 @@ class output_adapter
   private:
     output_adapter_t<CharType> oa = nullptr;
 };
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/input/binary_reader.hpp>
 
-//////////////////////////////
-// binary reader and writer //
-//////////////////////////////
+
+#include <algorithm> // generate_n
+#include <array> // array
+#include <cassert> // assert
+#include <cmath> // ldexp
+#include <cstddef> // size_t
+#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t
+#include <cstdio> // snprintf
+#include <cstring> // memcpy
+#include <iterator> // back_inserter
+#include <limits> // numeric_limits
+#include <string> // char_traits, string
+#include <utility> // make_pair, move
+
+// #include <nlohmann/detail/input/input_adapters.hpp>
+
+// #include <nlohmann/detail/input/json_sax.hpp>
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/is_sax.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+///////////////////
+// binary reader //
+///////////////////
 
 /*!
-@brief deserialization of CBOR and MessagePack values
+@brief deserialization of CBOR, MessagePack, and UBJSON values
 */
-template<typename BasicJsonType>
+template<typename BasicJsonType, typename SAX = json_sax_dom_parser<BasicJsonType>>
 class binary_reader
 {
     using number_integer_t = typename BasicJsonType::number_integer_t;
     using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+    using json_sax_t = SAX;
 
   public:
     /*!
@@ -4489,49 +6501,68 @@ class binary_reader
     */
     explicit binary_reader(input_adapter_t adapter) : ia(std::move(adapter))
     {
+        (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {};
         assert(ia);
     }
 
     /*!
-    @brief create a JSON value from CBOR input
-
+    @param[in] format  the binary format to parse
+    @param[in] sax_    a SAX event processor
     @param[in] strict  whether to expect the input to be consumed completed
-    @return JSON value created from CBOR input
 
-    @throw parse_error.110 if input ended unexpectedly or the end of file was
-                           not reached when @a strict was set to true
-    @throw parse_error.112 if unsupported byte was read
+    @return
     */
-    BasicJsonType parse_cbor(const bool strict)
+    bool sax_parse(const input_format_t format,
+                   json_sax_t* sax_,
+                   const bool strict = true)
     {
-        const auto res = parse_cbor_internal();
-        if (strict)
+        sax = sax_;
+        bool result = false;
+
+        switch (format)
         {
-            get();
-            check_eof(true);
-        }
-        return res;
-    }
+            case input_format_t::bson:
+                result = parse_bson_internal();
+                break;
 
-    /*!
-    @brief create a JSON value from MessagePack input
+            case input_format_t::cbor:
+                result = parse_cbor_internal();
+                break;
 
-    @param[in] strict  whether to expect the input to be consumed completed
-    @return JSON value created from MessagePack input
+            case input_format_t::msgpack:
+                result = parse_msgpack_internal();
+                break;
 
-    @throw parse_error.110 if input ended unexpectedly or the end of file was
-                           not reached when @a strict was set to true
-    @throw parse_error.112 if unsupported byte was read
-    */
-    BasicJsonType parse_msgpack(const bool strict)
-    {
-        const auto res = parse_msgpack_internal();
-        if (strict)
+            case input_format_t::ubjson:
+                result = parse_ubjson_internal();
+                break;
+
+            // LCOV_EXCL_START
+            default:
+                assert(false);
+                // LCOV_EXCL_STOP
+        }
+
+        // strict mode: next byte must be EOF
+        if (result and strict)
         {
-            get();
-            check_eof(true);
+            if (format == input_format_t::ubjson)
+            {
+                get_ignore_noop();
+            }
+            else
+            {
+                get();
+            }
+
+            if (JSON_UNLIKELY(current != std::char_traits<char>::eof()))
+            {
+                return sax->parse_error(chars_read, get_token_string(),
+                                        parse_error::create(110, chars_read, exception_message(format, "expected end of input; last byte: 0x" + get_token_string(), "value")));
+            }
         }
-        return res;
+
+        return result;
     }
 
     /*!
@@ -4547,18 +6578,239 @@ class binary_reader
     }
 
   private:
+    //////////
+    // BSON //
+    //////////
+
+    /*!
+    @brief Reads in a BSON-object and passes it to the SAX-parser.
+    @return whether a valid BSON-value was passed to the SAX parser
+    */
+    bool parse_bson_internal()
+    {
+        std::int32_t document_size;
+        get_number<std::int32_t, true>(input_format_t::bson, document_size);
+
+        if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1))))
+        {
+            return false;
+        }
+
+        if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/false)))
+        {
+            return false;
+        }
+
+        return sax->end_object();
+    }
+
+    /*!
+    @brief Parses a C-style string from the BSON input.
+    @param[in, out] result  A reference to the string variable where the read
+                            string is to be stored.
+    @return `true` if the \x00-byte indicating the end of the string was
+             encountered before the EOF; false` indicates an unexpected EOF.
+    */
+    bool get_bson_cstr(string_t& result)
+    {
+        auto out = std::back_inserter(result);
+        while (true)
+        {
+            get();
+            if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "cstring")))
+            {
+                return false;
+            }
+            if (current == 0x00)
+            {
+                return true;
+            }
+            *out++ = static_cast<char>(current);
+        }
+
+        return true;
+    }
+
+    /*!
+    @brief Parses a zero-terminated string of length @a len from the BSON
+           input.
+    @param[in] len  The length (including the zero-byte at the end) of the
+                    string to be read.
+    @param[in, out] result  A reference to the string variable where the read
+                            string is to be stored.
+    @tparam NumberType The type of the length @a len
+    @pre len >= 1
+    @return `true` if the string was successfully parsed
+    */
+    template<typename NumberType>
+    bool get_bson_string(const NumberType len, string_t& result)
+    {
+        if (JSON_UNLIKELY(len < 1))
+        {
+            auto last_token = get_token_string();
+            return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bson, "string length must be at least 1, is " + std::to_string(len), "string")));
+        }
+
+        return get_string(input_format_t::bson, len - static_cast<NumberType>(1), result) and get() != std::char_traits<char>::eof();
+    }
+
+    /*!
+    @brief Read a BSON document element of the given @a element_type.
+    @param[in] element_type The BSON element type, c.f. http://bsonspec.org/spec.html
+    @param[in] element_type_parse_position The position in the input stream,
+               where the `element_type` was read.
+    @warning Not all BSON element types are supported yet. An unsupported
+             @a element_type will give rise to a parse_error.114:
+             Unsupported BSON record type 0x...
+    @return whether a valid BSON-object/array was passed to the SAX parser
+    */
+    bool parse_bson_element_internal(const int element_type,
+                                     const std::size_t element_type_parse_position)
+    {
+        switch (element_type)
+        {
+            case 0x01: // double
+            {
+                double number;
+                return get_number<double, true>(input_format_t::bson, number) and sax->number_float(static_cast<number_float_t>(number), "");
+            }
+
+            case 0x02: // string
+            {
+                std::int32_t len;
+                string_t value;
+                return get_number<std::int32_t, true>(input_format_t::bson, len) and get_bson_string(len, value) and sax->string(value);
+            }
+
+            case 0x03: // object
+            {
+                return parse_bson_internal();
+            }
+
+            case 0x04: // array
+            {
+                return parse_bson_array();
+            }
+
+            case 0x08: // boolean
+            {
+                return sax->boolean(get() != 0);
+            }
+
+            case 0x0A: // null
+            {
+                return sax->null();
+            }
+
+            case 0x10: // int32
+            {
+                std::int32_t value;
+                return get_number<std::int32_t, true>(input_format_t::bson, value) and sax->number_integer(value);
+            }
+
+            case 0x12: // int64
+            {
+                std::int64_t value;
+                return get_number<std::int64_t, true>(input_format_t::bson, value) and sax->number_integer(value);
+            }
+
+            default: // anything else not supported (yet)
+            {
+                char cr[3];
+                (std::snprintf)(cr, sizeof(cr), "%.2hhX", static_cast<unsigned char>(element_type));
+                return sax->parse_error(element_type_parse_position, std::string(cr), parse_error::create(114, element_type_parse_position, "Unsupported BSON record type 0x" + std::string(cr)));
+            }
+        }
+    }
+
+    /*!
+    @brief Read a BSON element list (as specified in the BSON-spec)
+
+    The same binary layout is used for objects and arrays, hence it must be
+    indicated with the argument @a is_array which one is expected
+    (true --> array, false --> object).
+
+    @param[in] is_array Determines if the element list being read is to be
+                        treated as an object (@a is_array == false), or as an
+                        array (@a is_array == true).
+    @return whether a valid BSON-object/array was passed to the SAX parser
+    */
+    bool parse_bson_element_list(const bool is_array)
+    {
+        string_t key;
+        while (int element_type = get())
+        {
+            if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "element list")))
+            {
+                return false;
+            }
+
+            const std::size_t element_type_parse_position = chars_read;
+            if (JSON_UNLIKELY(not get_bson_cstr(key)))
+            {
+                return false;
+            }
+
+            if (not is_array)
+            {
+                if (not sax->key(key))
+                {
+                    return false;
+                }
+            }
+
+            if (JSON_UNLIKELY(not parse_bson_element_internal(element_type, element_type_parse_position)))
+            {
+                return false;
+            }
+
+            // get_bson_cstr only appends
+            key.clear();
+        }
+
+        return true;
+    }
+
+    /*!
+    @brief Reads an array from the BSON input and passes it to the SAX-parser.
+    @return whether a valid BSON-array was passed to the SAX parser
+    */
+    bool parse_bson_array()
+    {
+        std::int32_t document_size;
+        get_number<std::int32_t, true>(input_format_t::bson, document_size);
+
+        if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1))))
+        {
+            return false;
+        }
+
+        if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/true)))
+        {
+            return false;
+        }
+
+        return sax->end_array();
+    }
+
+    //////////
+    // CBOR //
+    //////////
+
     /*!
     @param[in] get_char  whether a new character should be retrieved from the
                          input (true, default) or whether the last read
                          character should be considered instead
+
+    @return whether a valid CBOR value was passed to the SAX parser
     */
-    BasicJsonType parse_cbor_internal(const bool get_char = true)
+    bool parse_cbor_internal(const bool get_char = true)
     {
         switch (get_char ? get() : current)
         {
             // EOF
             case std::char_traits<char>::eof():
-                JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input"));
+                return unexpect_eof(input_format_t::cbor, "value");
 
             // Integer 0x00..0x17 (0..23)
             case 0x00:
@@ -4585,19 +6837,31 @@ class binary_reader
             case 0x15:
             case 0x16:
             case 0x17:
-                return static_cast<number_unsigned_t>(current);
+                return sax->number_unsigned(static_cast<number_unsigned_t>(current));
 
             case 0x18: // Unsigned integer (one-byte uint8_t follows)
-                return get_number<uint8_t>();
+            {
+                uint8_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_unsigned(number);
+            }
 
             case 0x19: // Unsigned integer (two-byte uint16_t follows)
-                return get_number<uint16_t>();
+            {
+                uint16_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_unsigned(number);
+            }
 
             case 0x1A: // Unsigned integer (four-byte uint32_t follows)
-                return get_number<uint32_t>();
+            {
+                uint32_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_unsigned(number);
+            }
 
             case 0x1B: // Unsigned integer (eight-byte uint64_t follows)
-                return get_number<uint64_t>();
+            {
+                uint64_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_unsigned(number);
+            }
 
             // Negative integer -1-0x00..-1-0x17 (-1..-24)
             case 0x20:
@@ -4624,28 +6888,31 @@ class binary_reader
             case 0x35:
             case 0x36:
             case 0x37:
-                return static_cast<int8_t>(0x20 - 1 - current);
+                return sax->number_integer(static_cast<int8_t>(0x20 - 1 - current));
 
             case 0x38: // Negative integer (one-byte uint8_t follows)
             {
-                // must be uint8_t !
-                return static_cast<number_integer_t>(-1) - get_number<uint8_t>();
+                uint8_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number);
             }
 
             case 0x39: // Negative integer -1-n (two-byte uint16_t follows)
             {
-                return static_cast<number_integer_t>(-1) - get_number<uint16_t>();
+                uint16_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number);
             }
 
             case 0x3A: // Negative integer -1-n (four-byte uint32_t follows)
             {
-                return static_cast<number_integer_t>(-1) - get_number<uint32_t>();
+                uint32_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number);
             }
 
             case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows)
             {
-                return static_cast<number_integer_t>(-1) -
-                       static_cast<number_integer_t>(get_number<uint64_t>());
+                uint64_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1)
+                        - static_cast<number_integer_t>(number));
             }
 
             // UTF-8 string (0x00..0x17 bytes follow)
@@ -4679,7 +6946,8 @@ class binary_reader
             case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow)
             case 0x7F: // UTF-8 string (indefinite length)
             {
-                return get_cbor_string();
+                string_t s;
+                return get_cbor_string(s) and sax->string(s);
             }
 
             // array (0x00..0x17 data items follow)
@@ -4707,39 +6975,34 @@ class binary_reader
             case 0x95:
             case 0x96:
             case 0x97:
-            {
-                return get_cbor_array(current & 0x1F);
-            }
+                return get_cbor_array(static_cast<std::size_t>(current & 0x1F));
 
             case 0x98: // array (one-byte uint8_t for n follows)
             {
-                return get_cbor_array(get_number<uint8_t>());
+                uint8_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len));
             }
 
             case 0x99: // array (two-byte uint16_t for n follow)
             {
-                return get_cbor_array(get_number<uint16_t>());
+                uint16_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len));
             }
 
             case 0x9A: // array (four-byte uint32_t for n follow)
             {
-                return get_cbor_array(get_number<uint32_t>());
+                uint32_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len));
             }
 
             case 0x9B: // array (eight-byte uint64_t for n follow)
             {
-                return get_cbor_array(get_number<uint64_t>());
+                uint64_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len));
             }
 
             case 0x9F: // array (indefinite length)
-            {
-                BasicJsonType result = value_t::array;
-                while (get() != 0xFF)
-                {
-                    result.push_back(parse_cbor_internal(false));
-                }
-                return result;
-            }
+                return get_cbor_array(std::size_t(-1));
 
             // map (0x00..0x17 pairs of data items follow)
             case 0xA0:
@@ -4766,62 +7029,59 @@ class binary_reader
             case 0xB5:
             case 0xB6:
             case 0xB7:
-            {
-                return get_cbor_object(current & 0x1F);
-            }
+                return get_cbor_object(static_cast<std::size_t>(current & 0x1F));
 
             case 0xB8: // map (one-byte uint8_t for n follows)
             {
-                return get_cbor_object(get_number<uint8_t>());
+                uint8_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len));
             }
 
             case 0xB9: // map (two-byte uint16_t for n follow)
             {
-                return get_cbor_object(get_number<uint16_t>());
+                uint16_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len));
             }
 
             case 0xBA: // map (four-byte uint32_t for n follow)
             {
-                return get_cbor_object(get_number<uint32_t>());
+                uint32_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len));
             }
 
             case 0xBB: // map (eight-byte uint64_t for n follow)
             {
-                return get_cbor_object(get_number<uint64_t>());
+                uint64_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len));
             }
 
             case 0xBF: // map (indefinite length)
-            {
-                BasicJsonType result = value_t::object;
-                while (get() != 0xFF)
-                {
-                    auto key = get_cbor_string();
-                    result[key] = parse_cbor_internal();
-                }
-                return result;
-            }
+                return get_cbor_object(std::size_t(-1));
 
             case 0xF4: // false
-            {
-                return false;
-            }
+                return sax->boolean(false);
 
             case 0xF5: // true
-            {
-                return true;
-            }
+                return sax->boolean(true);
 
             case 0xF6: // null
-            {
-                return value_t::null;
-            }
+                return sax->null();
 
             case 0xF9: // Half-Precision Float (two-byte IEEE 754)
             {
-                const int byte1 = get();
-                check_eof();
-                const int byte2 = get();
-                check_eof();
+                const int byte1_raw = get();
+                if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number")))
+                {
+                    return false;
+                }
+                const int byte2_raw = get();
+                if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number")))
+                {
+                    return false;
+                }
+
+                const auto byte1 = static_cast<unsigned char>(byte1_raw);
+                const auto byte2 = static_cast<unsigned char>(byte2_raw);
 
                 // code from RFC 7049, Appendix D, Figure 3:
                 // As half-precision floating-point numbers were only added
@@ -4832,51 +7092,244 @@ class binary_reader
                 // half-precision floating-point numbers in the C language
                 // is shown in Fig. 3.
                 const int half = (byte1 << 8) + byte2;
-                const int exp = (half >> 10) & 0x1F;
-                const int mant = half & 0x3FF;
-                double val;
-                if (exp == 0)
-                {
-                    val = std::ldexp(mant, -24);
-                }
-                else if (exp != 31)
+                const double val = [&half]
                 {
-                    val = std::ldexp(mant + 1024, exp - 25);
-                }
-                else
-                {
-                    val = (mant == 0) ? std::numeric_limits<double>::infinity()
-                          : std::numeric_limits<double>::quiet_NaN();
-                }
-                return (half & 0x8000) != 0 ? -val : val;
+                    const int exp = (half >> 10) & 0x1F;
+                    const int mant = half & 0x3FF;
+                    assert(0 <= exp and exp <= 32);
+                    assert(0 <= mant and mant <= 1024);
+                    switch (exp)
+                    {
+                        case 0:
+                            return std::ldexp(mant, -24);
+                        case 31:
+                            return (mant == 0)
+                            ? std::numeric_limits<double>::infinity()
+                            : std::numeric_limits<double>::quiet_NaN();
+                        default:
+                            return std::ldexp(mant + 1024, exp - 25);
+                    }
+                }();
+                return sax->number_float((half & 0x8000) != 0
+                                         ? static_cast<number_float_t>(-val)
+                                         : static_cast<number_float_t>(val), "");
             }
 
             case 0xFA: // Single-Precision Float (four-byte IEEE 754)
             {
-                return get_number<float>();
+                float number;
+                return get_number(input_format_t::cbor, number) and sax->number_float(static_cast<number_float_t>(number), "");
             }
 
             case 0xFB: // Double-Precision Float (eight-byte IEEE 754)
             {
-                return get_number<double>();
+                double number;
+                return get_number(input_format_t::cbor, number) and sax->number_float(static_cast<number_float_t>(number), "");
             }
 
             default: // anything else (0xFF is handled inside the other types)
             {
-                std::stringstream ss;
-                ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current;
-                JSON_THROW(parse_error::create(112, chars_read, "error reading CBOR; last byte: 0x" + ss.str()));
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::cbor, "invalid byte: 0x" + last_token, "value")));
             }
         }
     }
 
-    BasicJsonType parse_msgpack_internal()
+    /*!
+    @brief reads a CBOR string
+
+    This function first reads starting bytes to determine the expected
+    string length and then copies this number of bytes into a string.
+    Additionally, CBOR's strings with indefinite lengths are supported.
+
+    @param[out] result  created string
+
+    @return whether string creation completed
+    */
+    bool get_cbor_string(string_t& result)
+    {
+        if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "string")))
+        {
+            return false;
+        }
+
+        switch (current)
+        {
+            // UTF-8 string (0x00..0x17 bytes follow)
+            case 0x60:
+            case 0x61:
+            case 0x62:
+            case 0x63:
+            case 0x64:
+            case 0x65:
+            case 0x66:
+            case 0x67:
+            case 0x68:
+            case 0x69:
+            case 0x6A:
+            case 0x6B:
+            case 0x6C:
+            case 0x6D:
+            case 0x6E:
+            case 0x6F:
+            case 0x70:
+            case 0x71:
+            case 0x72:
+            case 0x73:
+            case 0x74:
+            case 0x75:
+            case 0x76:
+            case 0x77:
+            {
+                return get_string(input_format_t::cbor, current & 0x1F, result);
+            }
+
+            case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
+            {
+                uint8_t len;
+                return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result);
+            }
+
+            case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
+            {
+                uint16_t len;
+                return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result);
+            }
+
+            case 0x7A: // UTF-8 string (four-byte uint32_t for n follow)
+            {
+                uint32_t len;
+                return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result);
+            }
+
+            case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow)
+            {
+                uint64_t len;
+                return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result);
+            }
+
+            case 0x7F: // UTF-8 string (indefinite length)
+            {
+                while (get() != 0xFF)
+                {
+                    string_t chunk;
+                    if (not get_cbor_string(chunk))
+                    {
+                        return false;
+                    }
+                    result.append(chunk);
+                }
+                return true;
+            }
+
+            default:
+            {
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::cbor, "expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x" + last_token, "string")));
+            }
+        }
+    }
+
+    /*!
+    @param[in] len  the length of the array or std::size_t(-1) for an
+                    array of indefinite size
+    @return whether array creation completed
+    */
+    bool get_cbor_array(const std::size_t len)
+    {
+        if (JSON_UNLIKELY(not sax->start_array(len)))
+        {
+            return false;
+        }
+
+        if (len != std::size_t(-1))
+        {
+            for (std::size_t i = 0; i < len; ++i)
+            {
+                if (JSON_UNLIKELY(not parse_cbor_internal()))
+                {
+                    return false;
+                }
+            }
+        }
+        else
+        {
+            while (get() != 0xFF)
+            {
+                if (JSON_UNLIKELY(not parse_cbor_internal(false)))
+                {
+                    return false;
+                }
+            }
+        }
+
+        return sax->end_array();
+    }
+
+    /*!
+    @param[in] len  the length of the object or std::size_t(-1) for an
+                    object of indefinite size
+    @return whether object creation completed
+    */
+    bool get_cbor_object(const std::size_t len)
+    {
+        if (not JSON_UNLIKELY(sax->start_object(len)))
+        {
+            return false;
+        }
+
+        string_t key;
+        if (len != std::size_t(-1))
+        {
+            for (std::size_t i = 0; i < len; ++i)
+            {
+                get();
+                if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key)))
+                {
+                    return false;
+                }
+
+                if (JSON_UNLIKELY(not parse_cbor_internal()))
+                {
+                    return false;
+                }
+                key.clear();
+            }
+        }
+        else
+        {
+            while (get() != 0xFF)
+            {
+                if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key)))
+                {
+                    return false;
+                }
+
+                if (JSON_UNLIKELY(not parse_cbor_internal()))
+                {
+                    return false;
+                }
+                key.clear();
+            }
+        }
+
+        return sax->end_object();
+    }
+
+    /////////////
+    // MsgPack //
+    /////////////
+
+    /*!
+    @return whether a valid MessagePack value was passed to the SAX parser
+    */
+    bool parse_msgpack_internal()
     {
         switch (get())
         {
             // EOF
             case std::char_traits<char>::eof():
-                JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input"));
+                return unexpect_eof(input_format_t::msgpack, "value");
 
             // positive fixint
             case 0x00:
@@ -5007,7 +7460,7 @@ class binary_reader
             case 0x7D:
             case 0x7E:
             case 0x7F:
-                return static_cast<number_unsigned_t>(current);
+                return sax->number_unsigned(static_cast<number_unsigned_t>(current));
 
             // fixmap
             case 0x80:
@@ -5026,9 +7479,7 @@ class binary_reader
             case 0x8D:
             case 0x8E:
             case 0x8F:
-            {
-                return get_msgpack_object(current & 0x0F);
-            }
+                return get_msgpack_object(static_cast<std::size_t>(current & 0x0F));
 
             // fixarray
             case 0x90:
@@ -5047,9 +7498,7 @@ class binary_reader
             case 0x9D:
             case 0x9E:
             case 0x9F:
-            {
-                return get_msgpack_array(current & 0x0F);
-            }
+                return get_msgpack_array(static_cast<std::size_t>(current & 0x0F));
 
             // fixstr
             case 0xA0:
@@ -5084,73 +7533,113 @@ class binary_reader
             case 0xBD:
             case 0xBE:
             case 0xBF:
-                return get_msgpack_string();
+            {
+                string_t s;
+                return get_msgpack_string(s) and sax->string(s);
+            }
 
             case 0xC0: // nil
-                return value_t::null;
+                return sax->null();
 
             case 0xC2: // false
-                return false;
+                return sax->boolean(false);
 
             case 0xC3: // true
-                return true;
+                return sax->boolean(true);
 
             case 0xCA: // float 32
-                return get_number<float>();
+            {
+                float number;
+                return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast<number_float_t>(number), "");
+            }
 
             case 0xCB: // float 64
-                return get_number<double>();
+            {
+                double number;
+                return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast<number_float_t>(number), "");
+            }
 
             case 0xCC: // uint 8
-                return get_number<uint8_t>();
+            {
+                uint8_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number);
+            }
 
             case 0xCD: // uint 16
-                return get_number<uint16_t>();
+            {
+                uint16_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number);
+            }
 
             case 0xCE: // uint 32
-                return get_number<uint32_t>();
+            {
+                uint32_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number);
+            }
 
             case 0xCF: // uint 64
-                return get_number<uint64_t>();
+            {
+                uint64_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number);
+            }
 
             case 0xD0: // int 8
-                return get_number<int8_t>();
+            {
+                int8_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_integer(number);
+            }
 
             case 0xD1: // int 16
-                return get_number<int16_t>();
+            {
+                int16_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_integer(number);
+            }
 
             case 0xD2: // int 32
-                return get_number<int32_t>();
+            {
+                int32_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_integer(number);
+            }
 
             case 0xD3: // int 64
-                return get_number<int64_t>();
+            {
+                int64_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_integer(number);
+            }
 
             case 0xD9: // str 8
             case 0xDA: // str 16
             case 0xDB: // str 32
-                return get_msgpack_string();
+            {
+                string_t s;
+                return get_msgpack_string(s) and sax->string(s);
+            }
 
             case 0xDC: // array 16
             {
-                return get_msgpack_array(get_number<uint16_t>());
+                uint16_t len;
+                return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast<std::size_t>(len));
             }
 
             case 0xDD: // array 32
             {
-                return get_msgpack_array(get_number<uint32_t>());
+                uint32_t len;
+                return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast<std::size_t>(len));
             }
 
             case 0xDE: // map 16
             {
-                return get_msgpack_object(get_number<uint16_t>());
+                uint16_t len;
+                return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast<std::size_t>(len));
             }
 
             case 0xDF: // map 32
             {
-                return get_msgpack_object(get_number<uint32_t>());
+                uint32_t len;
+                return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast<std::size_t>(len));
             }
 
-            // positive fixint
+            // negative fixint
             case 0xE0:
             case 0xE1:
             case 0xE2:
@@ -5183,338 +7672,749 @@ class binary_reader
             case 0xFD:
             case 0xFE:
             case 0xFF:
-                return static_cast<int8_t>(current);
+                return sax->number_integer(static_cast<int8_t>(current));
 
             default: // anything else
             {
-                std::stringstream ss;
-                ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current;
-                JSON_THROW(parse_error::create(112, chars_read,
-                                               "error reading MessagePack; last byte: 0x" + ss.str()));
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::msgpack, "invalid byte: 0x" + last_token, "value")));
             }
         }
     }
 
     /*!
-    @brief get next character from the input
+    @brief reads a MessagePack string
 
-    This function provides the interface to the used input adapter. It does
-    not throw in case the input reached EOF, but returns a -'ve valued
-    `std::char_traits<char>::eof()` in that case.
+    This function first reads starting bytes to determine the expected
+    string length and then copies this number of bytes into a string.
 
-    @return character read from the input
+    @param[out] result  created string
+
+    @return whether string creation completed
     */
-    int get()
+    bool get_msgpack_string(string_t& result)
     {
-        ++chars_read;
-        return (current = ia->get_character());
-    }
+        if (JSON_UNLIKELY(not unexpect_eof(input_format_t::msgpack, "string")))
+        {
+            return false;
+        }
 
-    /*
-    @brief read a number from the input
+        switch (current)
+        {
+            // fixstr
+            case 0xA0:
+            case 0xA1:
+            case 0xA2:
+            case 0xA3:
+            case 0xA4:
+            case 0xA5:
+            case 0xA6:
+            case 0xA7:
+            case 0xA8:
+            case 0xA9:
+            case 0xAA:
+            case 0xAB:
+            case 0xAC:
+            case 0xAD:
+            case 0xAE:
+            case 0xAF:
+            case 0xB0:
+            case 0xB1:
+            case 0xB2:
+            case 0xB3:
+            case 0xB4:
+            case 0xB5:
+            case 0xB6:
+            case 0xB7:
+            case 0xB8:
+            case 0xB9:
+            case 0xBA:
+            case 0xBB:
+            case 0xBC:
+            case 0xBD:
+            case 0xBE:
+            case 0xBF:
+            {
+                return get_string(input_format_t::msgpack, current & 0x1F, result);
+            }
 
-    @tparam NumberType the type of the number
+            case 0xD9: // str 8
+            {
+                uint8_t len;
+                return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result);
+            }
 
-    @return number of type @a NumberType
+            case 0xDA: // str 16
+            {
+                uint16_t len;
+                return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result);
+            }
 
-    @note This function needs to respect the system's endianess, because
-          bytes in CBOR and MessagePack are stored in network order (big
-          endian) and therefore need reordering on little endian systems.
+            case 0xDB: // str 32
+            {
+                uint32_t len;
+                return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result);
+            }
+
+            default:
+            {
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::msgpack, "expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x" + last_token, "string")));
+            }
+        }
+    }
 
-    @throw parse_error.110 if input has less than `sizeof(NumberType)` bytes
+    /*!
+    @param[in] len  the length of the array
+    @return whether array creation completed
     */
-    template<typename NumberType> NumberType get_number()
+    bool get_msgpack_array(const std::size_t len)
     {
-        // step 1: read input into array with system's byte order
-        std::array<uint8_t, sizeof(NumberType)> vec;
-        for (std::size_t i = 0; i < sizeof(NumberType); ++i)
+        if (JSON_UNLIKELY(not sax->start_array(len)))
         {
-            get();
-            check_eof();
+            return false;
+        }
 
-            // reverse byte order prior to conversion if necessary
-            if (is_little_endian)
-            {
-                vec[sizeof(NumberType) - i - 1] = static_cast<uint8_t>(current);
-            }
-            else
+        for (std::size_t i = 0; i < len; ++i)
+        {
+            if (JSON_UNLIKELY(not parse_msgpack_internal()))
             {
-                vec[i] = static_cast<uint8_t>(current); // LCOV_EXCL_LINE
+                return false;
             }
         }
 
-        // step 2: convert array into number of type T and return
-        NumberType result;
-        std::memcpy(&result, vec.data(), sizeof(NumberType));
-        return result;
+        return sax->end_array();
     }
 
     /*!
-    @brief create a string by reading characters from the input
+    @param[in] len  the length of the object
+    @return whether object creation completed
+    */
+    bool get_msgpack_object(const std::size_t len)
+    {
+        if (JSON_UNLIKELY(not sax->start_object(len)))
+        {
+            return false;
+        }
 
-    @param[in] len number of bytes to read
+        string_t key;
+        for (std::size_t i = 0; i < len; ++i)
+        {
+            get();
+            if (JSON_UNLIKELY(not get_msgpack_string(key) or not sax->key(key)))
+            {
+                return false;
+            }
 
-    @note We can not reserve @a len bytes for the result, because @a len
-          may be too large. Usually, @ref check_eof() detects the end of
-          the input before we run out of string memory.
+            if (JSON_UNLIKELY(not parse_msgpack_internal()))
+            {
+                return false;
+            }
+            key.clear();
+        }
 
-    @return string created by reading @a len bytes
+        return sax->end_object();
+    }
 
-    @throw parse_error.110 if input has less than @a len bytes
+    ////////////
+    // UBJSON //
+    ////////////
+
+    /*!
+    @param[in] get_char  whether a new character should be retrieved from the
+                         input (true, default) or whether the last read
+                         character should be considered instead
+
+    @return whether a valid UBJSON value was passed to the SAX parser
     */
-    template<typename NumberType>
-    std::string get_string(const NumberType len)
+    bool parse_ubjson_internal(const bool get_char = true)
     {
-        std::string result;
-        std::generate_n(std::back_inserter(result), len, [this]()
-        {
-            get();
-            check_eof();
-            return static_cast<char>(current);
-        });
-        return result;
+        return get_ubjson_value(get_char ? get_ignore_noop() : current);
     }
 
     /*!
-    @brief reads a CBOR string
+    @brief reads a UBJSON string
 
-    This function first reads starting bytes to determine the expected
-    string length and then copies this number of bytes into a string.
-    Additionally, CBOR's strings with indefinite lengths are supported.
+    This function is either called after reading the 'S' byte explicitly
+    indicating a string, or in case of an object key where the 'S' byte can be
+    left out.
 
-    @return string
+    @param[out] result   created string
+    @param[in] get_char  whether a new character should be retrieved from the
+                         input (true, default) or whether the last read
+                         character should be considered instead
 
-    @throw parse_error.110 if input ended
-    @throw parse_error.113 if an unexpected byte is read
+    @return whether string creation completed
     */
-    std::string get_cbor_string()
+    bool get_ubjson_string(string_t& result, const bool get_char = true)
     {
-        check_eof();
+        if (get_char)
+        {
+            get();  // TODO: may we ignore N here?
+        }
+
+        if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value")))
+        {
+            return false;
+        }
 
         switch (current)
         {
-            // UTF-8 string (0x00..0x17 bytes follow)
-            case 0x60:
-            case 0x61:
-            case 0x62:
-            case 0x63:
-            case 0x64:
-            case 0x65:
-            case 0x66:
-            case 0x67:
-            case 0x68:
-            case 0x69:
-            case 0x6A:
-            case 0x6B:
-            case 0x6C:
-            case 0x6D:
-            case 0x6E:
-            case 0x6F:
-            case 0x70:
-            case 0x71:
-            case 0x72:
-            case 0x73:
-            case 0x74:
-            case 0x75:
-            case 0x76:
-            case 0x77:
+            case 'U':
             {
-                return get_string(current & 0x1F);
+                uint8_t len;
+                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
             }
 
-            case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
+            case 'i':
             {
-                return get_string(get_number<uint8_t>());
+                int8_t len;
+                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
             }
 
-            case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
+            case 'I':
             {
-                return get_string(get_number<uint16_t>());
+                int16_t len;
+                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
             }
 
-            case 0x7A: // UTF-8 string (four-byte uint32_t for n follow)
+            case 'l':
             {
-                return get_string(get_number<uint32_t>());
+                int32_t len;
+                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
             }
 
-            case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow)
+            case 'L':
             {
-                return get_string(get_number<uint64_t>());
+                int64_t len;
+                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
             }
 
-            case 0x7F: // UTF-8 string (indefinite length)
+            default:
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L); last byte: 0x" + last_token, "string")));
+        }
+    }
+
+    /*!
+    @param[out] result  determined size
+    @return whether size determination completed
+    */
+    bool get_ubjson_size_value(std::size_t& result)
+    {
+        switch (get_ignore_noop())
+        {
+            case 'U':
             {
-                std::string result;
-                while (get() != 0xFF)
+                uint8_t number;
+                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
+                {
+                    return false;
+                }
+                result = static_cast<std::size_t>(number);
+                return true;
+            }
+
+            case 'i':
+            {
+                int8_t number;
+                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
                 {
-                    check_eof();
-                    result.push_back(static_cast<char>(current));
+                    return false;
                 }
-                return result;
+                result = static_cast<std::size_t>(number);
+                return true;
+            }
+
+            case 'I':
+            {
+                int16_t number;
+                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
+                {
+                    return false;
+                }
+                result = static_cast<std::size_t>(number);
+                return true;
+            }
+
+            case 'l':
+            {
+                int32_t number;
+                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
+                {
+                    return false;
+                }
+                result = static_cast<std::size_t>(number);
+                return true;
+            }
+
+            case 'L':
+            {
+                int64_t number;
+                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
+                {
+                    return false;
+                }
+                result = static_cast<std::size_t>(number);
+                return true;
             }
 
             default:
             {
-                std::stringstream ss;
-                ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current;
-                JSON_THROW(parse_error::create(113, chars_read, "expected a CBOR string; last byte: 0x" + ss.str()));
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token, "size")));
             }
         }
     }
 
-    template<typename NumberType>
-    BasicJsonType get_cbor_array(const NumberType len)
+    /*!
+    @brief determine the type and size for a container
+
+    In the optimized UBJSON format, a type and a size can be provided to allow
+    for a more compact representation.
+
+    @param[out] result  pair of the size and the type
+
+    @return whether pair creation completed
+    */
+    bool get_ubjson_size_type(std::pair<std::size_t, int>& result)
     {
-        BasicJsonType result = value_t::array;
-        std::generate_n(std::back_inserter(*result.m_value.array), len, [this]()
+        result.first = string_t::npos; // size
+        result.second = 0; // type
+
+        get_ignore_noop();
+
+        if (current == '$')
         {
-            return parse_cbor_internal();
-        });
-        return result;
-    }
+            result.second = get();  // must not ignore 'N', because 'N' maybe the type
+            if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "type")))
+            {
+                return false;
+            }
 
-    template<typename NumberType>
-    BasicJsonType get_cbor_object(const NumberType len)
-    {
-        BasicJsonType result = value_t::object;
-        std::generate_n(std::inserter(*result.m_value.object,
-                                      result.m_value.object->end()),
-                        len, [this]()
+            get_ignore_noop();
+            if (JSON_UNLIKELY(current != '#'))
+            {
+                if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value")))
+                {
+                    return false;
+                }
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "expected '#' after type information; last byte: 0x" + last_token, "size")));
+            }
+
+            return get_ubjson_size_value(result.first);
+        }
+        else if (current == '#')
         {
-            get();
-            auto key = get_cbor_string();
-            auto val = parse_cbor_internal();
-            return std::make_pair(std::move(key), std::move(val));
-        });
-        return result;
+            return get_ubjson_size_value(result.first);
+        }
+        return true;
     }
 
     /*!
-    @brief reads a MessagePack string
+    @param prefix  the previously read or set type prefix
+    @return whether value creation completed
+    */
+    bool get_ubjson_value(const int prefix)
+    {
+        switch (prefix)
+        {
+            case std::char_traits<char>::eof():  // EOF
+                return unexpect_eof(input_format_t::ubjson, "value");
 
-    This function first reads starting bytes to determine the expected
-    string length and then copies this number of bytes into a string.
+            case 'T':  // true
+                return sax->boolean(true);
+            case 'F':  // false
+                return sax->boolean(false);
+
+            case 'Z':  // null
+                return sax->null();
 
-    @return string
+            case 'U':
+            {
+                uint8_t number;
+                return get_number(input_format_t::ubjson, number) and sax->number_unsigned(number);
+            }
+
+            case 'i':
+            {
+                int8_t number;
+                return get_number(input_format_t::ubjson, number) and sax->number_integer(number);
+            }
 
-    @throw parse_error.110 if input ended
-    @throw parse_error.113 if an unexpected byte is read
+            case 'I':
+            {
+                int16_t number;
+                return get_number(input_format_t::ubjson, number) and sax->number_integer(number);
+            }
+
+            case 'l':
+            {
+                int32_t number;
+                return get_number(input_format_t::ubjson, number) and sax->number_integer(number);
+            }
+
+            case 'L':
+            {
+                int64_t number;
+                return get_number(input_format_t::ubjson, number) and sax->number_integer(number);
+            }
+
+            case 'd':
+            {
+                float number;
+                return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast<number_float_t>(number), "");
+            }
+
+            case 'D':
+            {
+                double number;
+                return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast<number_float_t>(number), "");
+            }
+
+            case 'C':  // char
+            {
+                get();
+                if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "char")))
+                {
+                    return false;
+                }
+                if (JSON_UNLIKELY(current > 127))
+                {
+                    auto last_token = get_token_string();
+                    return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "byte after 'C' must be in range 0x00..0x7F; last byte: 0x" + last_token, "char")));
+                }
+                string_t s(1, static_cast<char>(current));
+                return sax->string(s);
+            }
+
+            case 'S':  // string
+            {
+                string_t s;
+                return get_ubjson_string(s) and sax->string(s);
+            }
+
+            case '[':  // array
+                return get_ubjson_array();
+
+            case '{':  // object
+                return get_ubjson_object();
+
+            default: // anything else
+            {
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "invalid byte: 0x" + last_token, "value")));
+            }
+        }
+    }
+
+    /*!
+    @return whether array creation completed
     */
-    std::string get_msgpack_string()
+    bool get_ubjson_array()
     {
-        check_eof();
+        std::pair<std::size_t, int> size_and_type;
+        if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type)))
+        {
+            return false;
+        }
 
-        switch (current)
+        if (size_and_type.first != string_t::npos)
         {
-            // fixstr
-            case 0xA0:
-            case 0xA1:
-            case 0xA2:
-            case 0xA3:
-            case 0xA4:
-            case 0xA5:
-            case 0xA6:
-            case 0xA7:
-            case 0xA8:
-            case 0xA9:
-            case 0xAA:
-            case 0xAB:
-            case 0xAC:
-            case 0xAD:
-            case 0xAE:
-            case 0xAF:
-            case 0xB0:
-            case 0xB1:
-            case 0xB2:
-            case 0xB3:
-            case 0xB4:
-            case 0xB5:
-            case 0xB6:
-            case 0xB7:
-            case 0xB8:
-            case 0xB9:
-            case 0xBA:
-            case 0xBB:
-            case 0xBC:
-            case 0xBD:
-            case 0xBE:
-            case 0xBF:
+            if (JSON_UNLIKELY(not sax->start_array(size_and_type.first)))
             {
-                return get_string(current & 0x1F);
+                return false;
             }
 
-            case 0xD9: // str 8
+            if (size_and_type.second != 0)
             {
-                return get_string(get_number<uint8_t>());
+                if (size_and_type.second != 'N')
+                {
+                    for (std::size_t i = 0; i < size_and_type.first; ++i)
+                    {
+                        if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second)))
+                        {
+                            return false;
+                        }
+                    }
+                }
+            }
+            else
+            {
+                for (std::size_t i = 0; i < size_and_type.first; ++i)
+                {
+                    if (JSON_UNLIKELY(not parse_ubjson_internal()))
+                    {
+                        return false;
+                    }
+                }
+            }
+        }
+        else
+        {
+            if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1))))
+            {
+                return false;
             }
 
-            case 0xDA: // str 16
+            while (current != ']')
             {
-                return get_string(get_number<uint16_t>());
+                if (JSON_UNLIKELY(not parse_ubjson_internal(false)))
+                {
+                    return false;
+                }
+                get_ignore_noop();
             }
+        }
 
-            case 0xDB: // str 32
+        return sax->end_array();
+    }
+
+    /*!
+    @return whether object creation completed
+    */
+    bool get_ubjson_object()
+    {
+        std::pair<std::size_t, int> size_and_type;
+        if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type)))
+        {
+            return false;
+        }
+
+        string_t key;
+        if (size_and_type.first != string_t::npos)
+        {
+            if (JSON_UNLIKELY(not sax->start_object(size_and_type.first)))
             {
-                return get_string(get_number<uint32_t>());
+                return false;
             }
 
-            default:
+            if (size_and_type.second != 0)
             {
-                std::stringstream ss;
-                ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current;
-                JSON_THROW(parse_error::create(113, chars_read,
-                                               "expected a MessagePack string; last byte: 0x" + ss.str()));
+                for (std::size_t i = 0; i < size_and_type.first; ++i)
+                {
+                    if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key)))
+                    {
+                        return false;
+                    }
+                    if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second)))
+                    {
+                        return false;
+                    }
+                    key.clear();
+                }
+            }
+            else
+            {
+                for (std::size_t i = 0; i < size_and_type.first; ++i)
+                {
+                    if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key)))
+                    {
+                        return false;
+                    }
+                    if (JSON_UNLIKELY(not parse_ubjson_internal()))
+                    {
+                        return false;
+                    }
+                    key.clear();
+                }
+            }
+        }
+        else
+        {
+            if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1))))
+            {
+                return false;
+            }
+
+            while (current != '}')
+            {
+                if (JSON_UNLIKELY(not get_ubjson_string(key, false) or not sax->key(key)))
+                {
+                    return false;
+                }
+                if (JSON_UNLIKELY(not parse_ubjson_internal()))
+                {
+                    return false;
+                }
+                get_ignore_noop();
+                key.clear();
             }
         }
+
+        return sax->end_object();
     }
 
-    template<typename NumberType>
-    BasicJsonType get_msgpack_array(const NumberType len)
+    ///////////////////////
+    // Utility functions //
+    ///////////////////////
+
+    /*!
+    @brief get next character from the input
+
+    This function provides the interface to the used input adapter. It does
+    not throw in case the input reached EOF, but returns a -'ve valued
+    `std::char_traits<char>::eof()` in that case.
+
+    @return character read from the input
+    */
+    int get()
     {
-        BasicJsonType result = value_t::array;
-        std::generate_n(std::back_inserter(*result.m_value.array), len, [this]()
-        {
-            return parse_msgpack_internal();
-        });
-        return result;
+        ++chars_read;
+        return (current = ia->get_character());
     }
 
-    template<typename NumberType>
-    BasicJsonType get_msgpack_object(const NumberType len)
+    /*!
+    @return character read from the input after ignoring all 'N' entries
+    */
+    int get_ignore_noop()
     {
-        BasicJsonType result = value_t::object;
-        std::generate_n(std::inserter(*result.m_value.object,
-                                      result.m_value.object->end()),
-                        len, [this]()
+        do
         {
             get();
-            auto key = get_msgpack_string();
-            auto val = parse_msgpack_internal();
-            return std::make_pair(std::move(key), std::move(val));
-        });
-        return result;
+        }
+        while (current == 'N');
+
+        return current;
     }
 
-    /*!
-    @brief check if input ended
-    @throw parse_error.110 if input ended
+    /*
+    @brief read a number from the input
+
+    @tparam NumberType the type of the number
+    @param[in] format   the current format (for diagnostics)
+    @param[out] result  number of type @a NumberType
+
+    @return whether conversion completed
+
+    @note This function needs to respect the system's endianess, because
+          bytes in CBOR, MessagePack, and UBJSON are stored in network order
+          (big endian) and therefore need reordering on little endian systems.
     */
-    void check_eof(const bool expect_eof = false) const
+    template<typename NumberType, bool InputIsLittleEndian = false>
+    bool get_number(const input_format_t format, NumberType& result)
     {
-        if (expect_eof)
+        // step 1: read input into array with system's byte order
+        std::array<uint8_t, sizeof(NumberType)> vec;
+        for (std::size_t i = 0; i < sizeof(NumberType); ++i)
         {
-            if (JSON_UNLIKELY(current != std::char_traits<char>::eof()))
+            get();
+            if (JSON_UNLIKELY(not unexpect_eof(format, "number")))
             {
-                JSON_THROW(parse_error::create(110, chars_read, "expected end of input"));
+                return false;
+            }
+
+            // reverse byte order prior to conversion if necessary
+            if (is_little_endian && !InputIsLittleEndian)
+            {
+                vec[sizeof(NumberType) - i - 1] = static_cast<uint8_t>(current);
+            }
+            else
+            {
+                vec[i] = static_cast<uint8_t>(current); // LCOV_EXCL_LINE
             }
         }
-        else
+
+        // step 2: convert array into number of type T and return
+        std::memcpy(&result, vec.data(), sizeof(NumberType));
+        return true;
+    }
+
+    /*!
+    @brief create a string by reading characters from the input
+
+    @tparam NumberType the type of the number
+    @param[in] format the current format (for diagnostics)
+    @param[in] len number of characters to read
+    @param[out] result string created by reading @a len bytes
+
+    @return whether string creation completed
+
+    @note We can not reserve @a len bytes for the result, because @a len
+          may be too large. Usually, @ref unexpect_eof() detects the end of
+          the input before we run out of string memory.
+    */
+    template<typename NumberType>
+    bool get_string(const input_format_t format,
+                    const NumberType len,
+                    string_t& result)
+    {
+        bool success = true;
+        std::generate_n(std::back_inserter(result), len, [this, &success, &format]()
         {
-            if (JSON_UNLIKELY(current == std::char_traits<char>::eof()))
+            get();
+            if (JSON_UNLIKELY(not unexpect_eof(format, "string")))
             {
-                JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input"));
+                success = false;
             }
+            return static_cast<char>(current);
+        });
+        return success;
+    }
+
+    /*!
+    @param[in] format   the current format (for diagnostics)
+    @param[in] context  further context information (for diagnostics)
+    @return whether the last read character is not EOF
+    */
+    bool unexpect_eof(const input_format_t format, const char* context) const
+    {
+        if (JSON_UNLIKELY(current == std::char_traits<char>::eof()))
+        {
+            return sax->parse_error(chars_read, "<end of file>",
+                                    parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context)));
+        }
+        return true;
+    }
+
+    /*!
+    @return a string representation of the last read byte
+    */
+    std::string get_token_string() const
+    {
+        char cr[3];
+        (std::snprintf)(cr, 3, "%.2hhX", static_cast<unsigned char>(current));
+        return std::string{cr};
+    }
+
+    /*!
+    @param[in] format   the current format
+    @param[in] detail   a detailed error message
+    @param[in] context  further contect information
+    @return a message string to use in the parse_error exceptions
+    */
+    std::string exception_message(const input_format_t format,
+                                  const std::string& detail,
+                                  const std::string& context) const
+    {
+        std::string error_msg = "syntax error while parsing ";
+
+        switch (format)
+        {
+            case input_format_t::cbor:
+                error_msg += "CBOR";
+                break;
+
+            case input_format_t::msgpack:
+                error_msg += "MessagePack";
+                break;
+
+            case input_format_t::ubjson:
+                error_msg += "UBJSON";
+                break;
+
+            case input_format_t::bson:
+                error_msg += "BSON";
+                break;
+
+            // LCOV_EXCL_START
+            default:
+                assert(false);
+                // LCOV_EXCL_STOP
         }
+
+        return error_msg + " " + context + ": " + detail;
     }
 
   private:
@@ -5529,7 +8429,34 @@ class binary_reader
 
     /// whether we can assume little endianess
     const bool is_little_endian = little_endianess();
+
+    /// the SAX parser
+    json_sax_t* sax = nullptr;
 };
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/output/binary_writer.hpp>
+
+
+#include <algorithm> // reverse
+#include <array> // array
+#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t
+#include <cstring> // memcpy
+#include <limits> // numeric_limits
+
+// #include <nlohmann/detail/input/binary_reader.hpp>
+
+// #include <nlohmann/detail/output/output_adapters.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+///////////////////
+// binary writer //
+///////////////////
 
 /*!
 @brief serialization to CBOR and MessagePack values
@@ -5537,6 +8464,8 @@ class binary_reader
 template<typename BasicJsonType, typename CharType>
 class binary_writer
 {
+    using string_t = typename BasicJsonType::string_t;
+
   public:
     /*!
     @brief create a binary writer
@@ -5549,7 +8478,28 @@ class binary_writer
     }
 
     /*!
-    @brief[in] j  JSON value to serialize
+    @param[in] j  JSON value to serialize
+    @pre       j.type() == value_t::object
+    */
+    void write_bson(const BasicJsonType& j)
+    {
+        switch (j.type())
+        {
+            case value_t::object:
+            {
+                write_bson_object(*j.m_value.object);
+                break;
+            }
+
+            default:
+            {
+                JSON_THROW(type_error::create(317, "to serialize to BSON, top-level type must be object, but is " + std::string(j.type_name())));
+            }
+        }
+    }
+
+    /*!
+    @param[in] j  JSON value to serialize
     */
     void write_cbor(const BasicJsonType& j)
     {
@@ -5557,15 +8507,15 @@ class binary_writer
         {
             case value_t::null:
             {
-                oa->write_character(static_cast<CharType>(0xF6));
+                oa->write_character(to_char_type(0xF6));
                 break;
             }
 
             case value_t::boolean:
             {
                 oa->write_character(j.m_value.boolean
-                                    ? static_cast<CharType>(0xF5)
-                                    : static_cast<CharType>(0xF4));
+                                    ? to_char_type(0xF5)
+                                    : to_char_type(0xF4));
                 break;
             }
 
@@ -5582,22 +8532,22 @@ class binary_writer
                     }
                     else if (j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)())
                     {
-                        oa->write_character(static_cast<CharType>(0x18));
+                        oa->write_character(to_char_type(0x18));
                         write_number(static_cast<uint8_t>(j.m_value.number_integer));
                     }
                     else if (j.m_value.number_integer <= (std::numeric_limits<uint16_t>::max)())
                     {
-                        oa->write_character(static_cast<CharType>(0x19));
+                        oa->write_character(to_char_type(0x19));
                         write_number(static_cast<uint16_t>(j.m_value.number_integer));
                     }
                     else if (j.m_value.number_integer <= (std::numeric_limits<uint32_t>::max)())
                     {
-                        oa->write_character(static_cast<CharType>(0x1A));
+                        oa->write_character(to_char_type(0x1A));
                         write_number(static_cast<uint32_t>(j.m_value.number_integer));
                     }
                     else
                     {
-                        oa->write_character(static_cast<CharType>(0x1B));
+                        oa->write_character(to_char_type(0x1B));
                         write_number(static_cast<uint64_t>(j.m_value.number_integer));
                     }
                 }
@@ -5612,22 +8562,22 @@ class binary_writer
                     }
                     else if (positive_number <= (std::numeric_limits<uint8_t>::max)())
                     {
-                        oa->write_character(static_cast<CharType>(0x38));
+                        oa->write_character(to_char_type(0x38));
                         write_number(static_cast<uint8_t>(positive_number));
                     }
                     else if (positive_number <= (std::numeric_limits<uint16_t>::max)())
                     {
-                        oa->write_character(static_cast<CharType>(0x39));
+                        oa->write_character(to_char_type(0x39));
                         write_number(static_cast<uint16_t>(positive_number));
                     }
                     else if (positive_number <= (std::numeric_limits<uint32_t>::max)())
                     {
-                        oa->write_character(static_cast<CharType>(0x3A));
+                        oa->write_character(to_char_type(0x3A));
                         write_number(static_cast<uint32_t>(positive_number));
                     }
                     else
                     {
-                        oa->write_character(static_cast<CharType>(0x3B));
+                        oa->write_character(to_char_type(0x3B));
                         write_number(static_cast<uint64_t>(positive_number));
                     }
                 }
@@ -5642,30 +8592,30 @@ class binary_writer
                 }
                 else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
                 {
-                    oa->write_character(static_cast<CharType>(0x18));
+                    oa->write_character(to_char_type(0x18));
                     write_number(static_cast<uint8_t>(j.m_value.number_unsigned));
                 }
                 else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
                 {
-                    oa->write_character(static_cast<CharType>(0x19));
+                    oa->write_character(to_char_type(0x19));
                     write_number(static_cast<uint16_t>(j.m_value.number_unsigned));
                 }
                 else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
                 {
-                    oa->write_character(static_cast<CharType>(0x1A));
+                    oa->write_character(to_char_type(0x1A));
                     write_number(static_cast<uint32_t>(j.m_value.number_unsigned));
                 }
                 else
                 {
-                    oa->write_character(static_cast<CharType>(0x1B));
+                    oa->write_character(to_char_type(0x1B));
                     write_number(static_cast<uint64_t>(j.m_value.number_unsigned));
                 }
                 break;
             }
 
-            case value_t::number_float: // Double-Precision Float
+            case value_t::number_float:
             {
-                oa->write_character(static_cast<CharType>(0xFB));
+                oa->write_character(get_cbor_float_prefix(j.m_value.number_float));
                 write_number(j.m_value.number_float);
                 break;
             }
@@ -5678,25 +8628,25 @@ class binary_writer
                 {
                     write_number(static_cast<uint8_t>(0x60 + N));
                 }
-                else if (N <= 0xFF)
+                else if (N <= (std::numeric_limits<uint8_t>::max)())
                 {
-                    oa->write_character(static_cast<CharType>(0x78));
+                    oa->write_character(to_char_type(0x78));
                     write_number(static_cast<uint8_t>(N));
                 }
-                else if (N <= 0xFFFF)
+                else if (N <= (std::numeric_limits<uint16_t>::max)())
                 {
-                    oa->write_character(static_cast<CharType>(0x79));
+                    oa->write_character(to_char_type(0x79));
                     write_number(static_cast<uint16_t>(N));
                 }
-                else if (N <= 0xFFFFFFFF)
+                else if (N <= (std::numeric_limits<uint32_t>::max)())
                 {
-                    oa->write_character(static_cast<CharType>(0x7A));
+                    oa->write_character(to_char_type(0x7A));
                     write_number(static_cast<uint32_t>(N));
                 }
                 // LCOV_EXCL_START
-                else if (N <= 0xFFFFFFFFFFFFFFFF)
+                else if (N <= (std::numeric_limits<uint64_t>::max)())
                 {
-                    oa->write_character(static_cast<CharType>(0x7B));
+                    oa->write_character(to_char_type(0x7B));
                     write_number(static_cast<uint64_t>(N));
                 }
                 // LCOV_EXCL_STOP
@@ -5716,25 +8666,25 @@ class binary_writer
                 {
                     write_number(static_cast<uint8_t>(0x80 + N));
                 }
-                else if (N <= 0xFF)
+                else if (N <= (std::numeric_limits<uint8_t>::max)())
                 {
-                    oa->write_character(static_cast<CharType>(0x98));
+                    oa->write_character(to_char_type(0x98));
                     write_number(static_cast<uint8_t>(N));
                 }
-                else if (N <= 0xFFFF)
+                else if (N <= (std::numeric_limits<uint16_t>::max)())
                 {
-                    oa->write_character(static_cast<CharType>(0x99));
+                    oa->write_character(to_char_type(0x99));
                     write_number(static_cast<uint16_t>(N));
                 }
-                else if (N <= 0xFFFFFFFF)
+                else if (N <= (std::numeric_limits<uint32_t>::max)())
                 {
-                    oa->write_character(static_cast<CharType>(0x9A));
+                    oa->write_character(to_char_type(0x9A));
                     write_number(static_cast<uint32_t>(N));
                 }
                 // LCOV_EXCL_START
-                else if (N <= 0xFFFFFFFFFFFFFFFF)
+                else if (N <= (std::numeric_limits<uint64_t>::max)())
                 {
-                    oa->write_character(static_cast<CharType>(0x9B));
+                    oa->write_character(to_char_type(0x9B));
                     write_number(static_cast<uint64_t>(N));
                 }
                 // LCOV_EXCL_STOP
@@ -5755,25 +8705,25 @@ class binary_writer
                 {
                     write_number(static_cast<uint8_t>(0xA0 + N));
                 }
-                else if (N <= 0xFF)
+                else if (N <= (std::numeric_limits<uint8_t>::max)())
                 {
-                    oa->write_character(static_cast<CharType>(0xB8));
+                    oa->write_character(to_char_type(0xB8));
                     write_number(static_cast<uint8_t>(N));
                 }
-                else if (N <= 0xFFFF)
+                else if (N <= (std::numeric_limits<uint16_t>::max)())
                 {
-                    oa->write_character(static_cast<CharType>(0xB9));
+                    oa->write_character(to_char_type(0xB9));
                     write_number(static_cast<uint16_t>(N));
                 }
-                else if (N <= 0xFFFFFFFF)
+                else if (N <= (std::numeric_limits<uint32_t>::max)())
                 {
-                    oa->write_character(static_cast<CharType>(0xBA));
+                    oa->write_character(to_char_type(0xBA));
                     write_number(static_cast<uint32_t>(N));
                 }
                 // LCOV_EXCL_START
-                else if (N <= 0xFFFFFFFFFFFFFFFF)
+                else if (N <= (std::numeric_limits<uint64_t>::max)())
                 {
-                    oa->write_character(static_cast<CharType>(0xBB));
+                    oa->write_character(to_char_type(0xBB));
                     write_number(static_cast<uint64_t>(N));
                 }
                 // LCOV_EXCL_STOP
@@ -5793,7 +8743,7 @@ class binary_writer
     }
 
     /*!
-    @brief[in] j  JSON value to serialize
+    @param[in] j  JSON value to serialize
     */
     void write_msgpack(const BasicJsonType& j)
     {
@@ -5801,15 +8751,15 @@ class binary_writer
         {
             case value_t::null: // nil
             {
-                oa->write_character(static_cast<CharType>(0xC0));
+                oa->write_character(to_char_type(0xC0));
                 break;
             }
 
             case value_t::boolean: // true and false
             {
                 oa->write_character(j.m_value.boolean
-                                    ? static_cast<CharType>(0xC3)
-                                    : static_cast<CharType>(0xC2));
+                                    ? to_char_type(0xC3)
+                                    : to_char_type(0xC2));
                 break;
             }
 
@@ -5828,25 +8778,25 @@ class binary_writer
                     else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
                     {
                         // uint 8
-                        oa->write_character(static_cast<CharType>(0xCC));
+                        oa->write_character(to_char_type(0xCC));
                         write_number(static_cast<uint8_t>(j.m_value.number_integer));
                     }
                     else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
                     {
                         // uint 16
-                        oa->write_character(static_cast<CharType>(0xCD));
+                        oa->write_character(to_char_type(0xCD));
                         write_number(static_cast<uint16_t>(j.m_value.number_integer));
                     }
                     else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
                     {
                         // uint 32
-                        oa->write_character(static_cast<CharType>(0xCE));
+                        oa->write_character(to_char_type(0xCE));
                         write_number(static_cast<uint32_t>(j.m_value.number_integer));
                     }
                     else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
                     {
                         // uint 64
-                        oa->write_character(static_cast<CharType>(0xCF));
+                        oa->write_character(to_char_type(0xCF));
                         write_number(static_cast<uint64_t>(j.m_value.number_integer));
                     }
                 }
@@ -5861,28 +8811,28 @@ class binary_writer
                              j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)())
                     {
                         // int 8
-                        oa->write_character(static_cast<CharType>(0xD0));
+                        oa->write_character(to_char_type(0xD0));
                         write_number(static_cast<int8_t>(j.m_value.number_integer));
                     }
                     else if (j.m_value.number_integer >= (std::numeric_limits<int16_t>::min)() and
                              j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)())
                     {
                         // int 16
-                        oa->write_character(static_cast<CharType>(0xD1));
+                        oa->write_character(to_char_type(0xD1));
                         write_number(static_cast<int16_t>(j.m_value.number_integer));
                     }
                     else if (j.m_value.number_integer >= (std::numeric_limits<int32_t>::min)() and
                              j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)())
                     {
                         // int 32
-                        oa->write_character(static_cast<CharType>(0xD2));
+                        oa->write_character(to_char_type(0xD2));
                         write_number(static_cast<int32_t>(j.m_value.number_integer));
                     }
                     else if (j.m_value.number_integer >= (std::numeric_limits<int64_t>::min)() and
                              j.m_value.number_integer <= (std::numeric_limits<int64_t>::max)())
                     {
                         // int 64
-                        oa->write_character(static_cast<CharType>(0xD3));
+                        oa->write_character(to_char_type(0xD3));
                         write_number(static_cast<int64_t>(j.m_value.number_integer));
                     }
                 }
@@ -5899,33 +8849,33 @@ class binary_writer
                 else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
                 {
                     // uint 8
-                    oa->write_character(static_cast<CharType>(0xCC));
+                    oa->write_character(to_char_type(0xCC));
                     write_number(static_cast<uint8_t>(j.m_value.number_integer));
                 }
                 else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
                 {
                     // uint 16
-                    oa->write_character(static_cast<CharType>(0xCD));
+                    oa->write_character(to_char_type(0xCD));
                     write_number(static_cast<uint16_t>(j.m_value.number_integer));
                 }
                 else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
                 {
                     // uint 32
-                    oa->write_character(static_cast<CharType>(0xCE));
+                    oa->write_character(to_char_type(0xCE));
                     write_number(static_cast<uint32_t>(j.m_value.number_integer));
                 }
                 else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
                 {
                     // uint 64
-                    oa->write_character(static_cast<CharType>(0xCF));
+                    oa->write_character(to_char_type(0xCF));
                     write_number(static_cast<uint64_t>(j.m_value.number_integer));
                 }
                 break;
             }
 
-            case value_t::number_float: // float 64
+            case value_t::number_float:
             {
-                oa->write_character(static_cast<CharType>(0xCB));
+                oa->write_character(get_msgpack_float_prefix(j.m_value.number_float));
                 write_number(j.m_value.number_float);
                 break;
             }
@@ -5939,22 +8889,22 @@ class binary_writer
                     // fixstr
                     write_number(static_cast<uint8_t>(0xA0 | N));
                 }
-                else if (N <= 255)
+                else if (N <= (std::numeric_limits<uint8_t>::max)())
                 {
                     // str 8
-                    oa->write_character(static_cast<CharType>(0xD9));
+                    oa->write_character(to_char_type(0xD9));
                     write_number(static_cast<uint8_t>(N));
                 }
-                else if (N <= 65535)
+                else if (N <= (std::numeric_limits<uint16_t>::max)())
                 {
                     // str 16
-                    oa->write_character(static_cast<CharType>(0xDA));
+                    oa->write_character(to_char_type(0xDA));
                     write_number(static_cast<uint16_t>(N));
                 }
-                else if (N <= 4294967295)
+                else if (N <= (std::numeric_limits<uint32_t>::max)())
                 {
                     // str 32
-                    oa->write_character(static_cast<CharType>(0xDB));
+                    oa->write_character(to_char_type(0xDB));
                     write_number(static_cast<uint32_t>(N));
                 }
 
@@ -5974,16 +8924,16 @@ class binary_writer
                     // fixarray
                     write_number(static_cast<uint8_t>(0x90 | N));
                 }
-                else if (N <= 0xFFFF)
+                else if (N <= (std::numeric_limits<uint16_t>::max)())
                 {
                     // array 16
-                    oa->write_character(static_cast<CharType>(0xDC));
+                    oa->write_character(to_char_type(0xDC));
                     write_number(static_cast<uint16_t>(N));
                 }
-                else if (N <= 0xFFFFFFFF)
+                else if (N <= (std::numeric_limits<uint32_t>::max)())
                 {
                     // array 32
-                    oa->write_character(static_cast<CharType>(0xDD));
+                    oa->write_character(to_char_type(0xDD));
                     write_number(static_cast<uint32_t>(N));
                 }
 
@@ -6004,16 +8954,16 @@ class binary_writer
                     // fixmap
                     write_number(static_cast<uint8_t>(0x80 | (N & 0xF)));
                 }
-                else if (N <= 65535)
+                else if (N <= (std::numeric_limits<uint16_t>::max)())
                 {
                     // map 16
-                    oa->write_character(static_cast<CharType>(0xDE));
+                    oa->write_character(to_char_type(0xDE));
                     write_number(static_cast<uint16_t>(N));
                 }
-                else if (N <= 4294967295)
+                else if (N <= (std::numeric_limits<uint32_t>::max)())
                 {
                     // map 32
-                    oa->write_character(static_cast<CharType>(0xDF));
+                    oa->write_character(to_char_type(0xDF));
                     write_number(static_cast<uint32_t>(N));
                 }
 
@@ -6031,25 +8981,745 @@ class binary_writer
         }
     }
 
+    /*!
+    @param[in] j  JSON value to serialize
+    @param[in] use_count   whether to use '#' prefixes (optimized format)
+    @param[in] use_type    whether to use '$' prefixes (optimized format)
+    @param[in] add_prefix  whether prefixes need to be used for this value
+    */
+    void write_ubjson(const BasicJsonType& j, const bool use_count,
+                      const bool use_type, const bool add_prefix = true)
+    {
+        switch (j.type())
+        {
+            case value_t::null:
+            {
+                if (add_prefix)
+                {
+                    oa->write_character(to_char_type('Z'));
+                }
+                break;
+            }
+
+            case value_t::boolean:
+            {
+                if (add_prefix)
+                {
+                    oa->write_character(j.m_value.boolean
+                                        ? to_char_type('T')
+                                        : to_char_type('F'));
+                }
+                break;
+            }
+
+            case value_t::number_integer:
+            {
+                write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix);
+                break;
+            }
+
+            case value_t::number_unsigned:
+            {
+                write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix);
+                break;
+            }
+
+            case value_t::number_float:
+            {
+                write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix);
+                break;
+            }
+
+            case value_t::string:
+            {
+                if (add_prefix)
+                {
+                    oa->write_character(to_char_type('S'));
+                }
+                write_number_with_ubjson_prefix(j.m_value.string->size(), true);
+                oa->write_characters(
+                    reinterpret_cast<const CharType*>(j.m_value.string->c_str()),
+                    j.m_value.string->size());
+                break;
+            }
+
+            case value_t::array:
+            {
+                if (add_prefix)
+                {
+                    oa->write_character(to_char_type('['));
+                }
+
+                bool prefix_required = true;
+                if (use_type and not j.m_value.array->empty())
+                {
+                    assert(use_count);
+                    const CharType first_prefix = ubjson_prefix(j.front());
+                    const bool same_prefix = std::all_of(j.begin() + 1, j.end(),
+                                                         [this, first_prefix](const BasicJsonType & v)
+                    {
+                        return ubjson_prefix(v) == first_prefix;
+                    });
+
+                    if (same_prefix)
+                    {
+                        prefix_required = false;
+                        oa->write_character(to_char_type('$'));
+                        oa->write_character(first_prefix);
+                    }
+                }
+
+                if (use_count)
+                {
+                    oa->write_character(to_char_type('#'));
+                    write_number_with_ubjson_prefix(j.m_value.array->size(), true);
+                }
+
+                for (const auto& el : *j.m_value.array)
+                {
+                    write_ubjson(el, use_count, use_type, prefix_required);
+                }
+
+                if (not use_count)
+                {
+                    oa->write_character(to_char_type(']'));
+                }
+
+                break;
+            }
+
+            case value_t::object:
+            {
+                if (add_prefix)
+                {
+                    oa->write_character(to_char_type('{'));
+                }
+
+                bool prefix_required = true;
+                if (use_type and not j.m_value.object->empty())
+                {
+                    assert(use_count);
+                    const CharType first_prefix = ubjson_prefix(j.front());
+                    const bool same_prefix = std::all_of(j.begin(), j.end(),
+                                                         [this, first_prefix](const BasicJsonType & v)
+                    {
+                        return ubjson_prefix(v) == first_prefix;
+                    });
+
+                    if (same_prefix)
+                    {
+                        prefix_required = false;
+                        oa->write_character(to_char_type('$'));
+                        oa->write_character(first_prefix);
+                    }
+                }
+
+                if (use_count)
+                {
+                    oa->write_character(to_char_type('#'));
+                    write_number_with_ubjson_prefix(j.m_value.object->size(), true);
+                }
+
+                for (const auto& el : *j.m_value.object)
+                {
+                    write_number_with_ubjson_prefix(el.first.size(), true);
+                    oa->write_characters(
+                        reinterpret_cast<const CharType*>(el.first.c_str()),
+                        el.first.size());
+                    write_ubjson(el.second, use_count, use_type, prefix_required);
+                }
+
+                if (not use_count)
+                {
+                    oa->write_character(to_char_type('}'));
+                }
+
+                break;
+            }
+
+            default:
+                break;
+        }
+    }
+
   private:
+    //////////
+    // BSON //
+    //////////
+
+    /*!
+    @return The size of a BSON document entry header, including the id marker
+            and the entry name size (and its null-terminator).
+    */
+    static std::size_t calc_bson_entry_header_size(const string_t& name)
+    {
+        const auto it = name.find(static_cast<typename string_t::value_type>(0));
+        if (JSON_UNLIKELY(it != BasicJsonType::string_t::npos))
+        {
+            JSON_THROW(out_of_range::create(409,
+                                            "BSON key cannot contain code point U+0000 (at byte " + std::to_string(it) + ")"));
+        }
+
+        return /*id*/ 1ul + name.size() + /*zero-terminator*/1u;
+    }
+
+    /*!
+    @brief Writes the given @a element_type and @a name to the output adapter
+    */
+    void write_bson_entry_header(const string_t& name,
+                                 const std::uint8_t element_type)
+    {
+        oa->write_character(to_char_type(element_type)); // boolean
+        oa->write_characters(
+            reinterpret_cast<const CharType*>(name.c_str()),
+            name.size() + 1u);
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and boolean value @a value
+    */
+    void write_bson_boolean(const string_t& name,
+                            const bool value)
+    {
+        write_bson_entry_header(name, 0x08);
+        oa->write_character(value ? to_char_type(0x01) : to_char_type(0x00));
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and double value @a value
+    */
+    void write_bson_double(const string_t& name,
+                           const double value)
+    {
+        write_bson_entry_header(name, 0x01);
+        write_number<double, true>(value);
+    }
+
+    /*!
+    @return The size of the BSON-encoded string in @a value
+    */
+    static std::size_t calc_bson_string_size(const string_t& value)
+    {
+        return sizeof(std::int32_t) + value.size() + 1ul;
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and string value @a value
+    */
+    void write_bson_string(const string_t& name,
+                           const string_t& value)
+    {
+        write_bson_entry_header(name, 0x02);
+
+        write_number<std::int32_t, true>(static_cast<std::int32_t>(value.size() + 1ul));
+        oa->write_characters(
+            reinterpret_cast<const CharType*>(value.c_str()),
+            value.size() + 1);
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and null value
+    */
+    void write_bson_null(const string_t& name)
+    {
+        write_bson_entry_header(name, 0x0A);
+    }
+
+    /*!
+    @return The size of the BSON-encoded integer @a value
+    */
+    static std::size_t calc_bson_integer_size(const std::int64_t value)
+    {
+        if ((std::numeric_limits<std::int32_t>::min)() <= value and value <= (std::numeric_limits<std::int32_t>::max)())
+        {
+            return sizeof(std::int32_t);
+        }
+        else
+        {
+            return sizeof(std::int64_t);
+        }
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and integer @a value
+    */
+    void write_bson_integer(const string_t& name,
+                            const std::int64_t value)
+    {
+        if ((std::numeric_limits<std::int32_t>::min)() <= value and value <= (std::numeric_limits<std::int32_t>::max)())
+        {
+            write_bson_entry_header(name, 0x10); // int32
+            write_number<std::int32_t, true>(static_cast<std::int32_t>(value));
+        }
+        else
+        {
+            write_bson_entry_header(name, 0x12); // int64
+            write_number<std::int64_t, true>(static_cast<std::int64_t>(value));
+        }
+    }
+
+    /*!
+    @return The size of the BSON-encoded unsigned integer in @a j
+    */
+    static constexpr std::size_t calc_bson_unsigned_size(const std::uint64_t value) noexcept
+    {
+        return (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)()))
+               ? sizeof(std::int32_t)
+               : sizeof(std::int64_t);
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and unsigned @a value
+    */
+    void write_bson_unsigned(const string_t& name,
+                             const std::uint64_t value)
+    {
+        if (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)()))
+        {
+            write_bson_entry_header(name, 0x10 /* int32 */);
+            write_number<std::int32_t, true>(static_cast<std::int32_t>(value));
+        }
+        else if (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()))
+        {
+            write_bson_entry_header(name, 0x12 /* int64 */);
+            write_number<std::int64_t, true>(static_cast<std::int64_t>(value));
+        }
+        else
+        {
+            JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(value) + " cannot be represented by BSON as it does not fit int64"));
+        }
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and object @a value
+    */
+    void write_bson_object_entry(const string_t& name,
+                                 const typename BasicJsonType::object_t& value)
+    {
+        write_bson_entry_header(name, 0x03); // object
+        write_bson_object(value);
+    }
+
+    /*!
+    @return The size of the BSON-encoded array @a value
+    */
+    static std::size_t calc_bson_array_size(const typename BasicJsonType::array_t& value)
+    {
+        std::size_t embedded_document_size = 0ul;
+        std::size_t array_index = 0ul;
+
+        for (const auto& el : value)
+        {
+            embedded_document_size += calc_bson_element_size(std::to_string(array_index++), el);
+        }
+
+        return sizeof(std::int32_t) + embedded_document_size + 1ul;
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and array @a value
+    */
+    void write_bson_array(const string_t& name,
+                          const typename BasicJsonType::array_t& value)
+    {
+        write_bson_entry_header(name, 0x04); // array
+        write_number<std::int32_t, true>(static_cast<std::int32_t>(calc_bson_array_size(value)));
+
+        std::size_t array_index = 0ul;
+
+        for (const auto& el : value)
+        {
+            write_bson_element(std::to_string(array_index++), el);
+        }
+
+        oa->write_character(to_char_type(0x00));
+    }
+
+    /*!
+    @brief Calculates the size necessary to serialize the JSON value @a j with its @a name
+    @return The calculated size for the BSON document entry for @a j with the given @a name.
+    */
+    static std::size_t calc_bson_element_size(const string_t& name,
+            const BasicJsonType& j)
+    {
+        const auto header_size = calc_bson_entry_header_size(name);
+        switch (j.type())
+        {
+            case value_t::object:
+                return header_size + calc_bson_object_size(*j.m_value.object);
+
+            case value_t::array:
+                return header_size + calc_bson_array_size(*j.m_value.array);
+
+            case value_t::boolean:
+                return header_size + 1ul;
+
+            case value_t::number_float:
+                return header_size + 8ul;
+
+            case value_t::number_integer:
+                return header_size + calc_bson_integer_size(j.m_value.number_integer);
+
+            case value_t::number_unsigned:
+                return header_size + calc_bson_unsigned_size(j.m_value.number_unsigned);
+
+            case value_t::string:
+                return header_size + calc_bson_string_size(*j.m_value.string);
+
+            case value_t::null:
+                return header_size + 0ul;
+
+            // LCOV_EXCL_START
+            default:
+                assert(false);
+                return 0ul;
+                // LCOV_EXCL_STOP
+        };
+    }
+
+    /*!
+    @brief Serializes the JSON value @a j to BSON and associates it with the
+           key @a name.
+    @param name The name to associate with the JSON entity @a j within the
+                current BSON document
+    @return The size of the BSON entry
+    */
+    void write_bson_element(const string_t& name,
+                            const BasicJsonType& j)
+    {
+        switch (j.type())
+        {
+            case value_t::object:
+                return write_bson_object_entry(name, *j.m_value.object);
+
+            case value_t::array:
+                return write_bson_array(name, *j.m_value.array);
+
+            case value_t::boolean:
+                return write_bson_boolean(name, j.m_value.boolean);
+
+            case value_t::number_float:
+                return write_bson_double(name, j.m_value.number_float);
+
+            case value_t::number_integer:
+                return write_bson_integer(name, j.m_value.number_integer);
+
+            case value_t::number_unsigned:
+                return write_bson_unsigned(name, j.m_value.number_unsigned);
+
+            case value_t::string:
+                return write_bson_string(name, *j.m_value.string);
+
+            case value_t::null:
+                return write_bson_null(name);
+
+            // LCOV_EXCL_START
+            default:
+                assert(false);
+                return;
+                // LCOV_EXCL_STOP
+        };
+    }
+
+    /*!
+    @brief Calculates the size of the BSON serialization of the given
+           JSON-object @a j.
+    @param[in] j  JSON value to serialize
+    @pre       j.type() == value_t::object
+    */
+    static std::size_t calc_bson_object_size(const typename BasicJsonType::object_t& value)
+    {
+        std::size_t document_size = std::accumulate(value.begin(), value.end(), 0ul,
+                                    [](size_t result, const typename BasicJsonType::object_t::value_type & el)
+        {
+            return result += calc_bson_element_size(el.first, el.second);
+        });
+
+        return sizeof(std::int32_t) + document_size + 1ul;
+    }
+
+    /*!
+    @param[in] j  JSON value to serialize
+    @pre       j.type() == value_t::object
+    */
+    void write_bson_object(const typename BasicJsonType::object_t& value)
+    {
+        write_number<std::int32_t, true>(static_cast<std::int32_t>(calc_bson_object_size(value)));
+
+        for (const auto& el : value)
+        {
+            write_bson_element(el.first, el.second);
+        }
+
+        oa->write_character(to_char_type(0x00));
+    }
+
+    //////////
+    // CBOR //
+    //////////
+
+    static constexpr CharType get_cbor_float_prefix(float /*unused*/)
+    {
+        return to_char_type(0xFA);  // Single-Precision Float
+    }
+
+    static constexpr CharType get_cbor_float_prefix(double /*unused*/)
+    {
+        return to_char_type(0xFB);  // Double-Precision Float
+    }
+
+    /////////////
+    // MsgPack //
+    /////////////
+
+    static constexpr CharType get_msgpack_float_prefix(float /*unused*/)
+    {
+        return to_char_type(0xCA);  // float 32
+    }
+
+    static constexpr CharType get_msgpack_float_prefix(double /*unused*/)
+    {
+        return to_char_type(0xCB);  // float 64
+    }
+
+    ////////////
+    // UBJSON //
+    ////////////
+
+    // UBJSON: write number (floating point)
+    template<typename NumberType, typename std::enable_if<
+                 std::is_floating_point<NumberType>::value, int>::type = 0>
+    void write_number_with_ubjson_prefix(const NumberType n,
+                                         const bool add_prefix)
+    {
+        if (add_prefix)
+        {
+            oa->write_character(get_ubjson_float_prefix(n));
+        }
+        write_number(n);
+    }
+
+    // UBJSON: write number (unsigned integer)
+    template<typename NumberType, typename std::enable_if<
+                 std::is_unsigned<NumberType>::value, int>::type = 0>
+    void write_number_with_ubjson_prefix(const NumberType n,
+                                         const bool add_prefix)
+    {
+        if (n <= static_cast<uint64_t>((std::numeric_limits<int8_t>::max)()))
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('i'));  // int8
+            }
+            write_number(static_cast<uint8_t>(n));
+        }
+        else if (n <= (std::numeric_limits<uint8_t>::max)())
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('U'));  // uint8
+            }
+            write_number(static_cast<uint8_t>(n));
+        }
+        else if (n <= static_cast<uint64_t>((std::numeric_limits<int16_t>::max)()))
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('I'));  // int16
+            }
+            write_number(static_cast<int16_t>(n));
+        }
+        else if (n <= static_cast<uint64_t>((std::numeric_limits<int32_t>::max)()))
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('l'));  // int32
+            }
+            write_number(static_cast<int32_t>(n));
+        }
+        else if (n <= static_cast<uint64_t>((std::numeric_limits<int64_t>::max)()))
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('L'));  // int64
+            }
+            write_number(static_cast<int64_t>(n));
+        }
+        else
+        {
+            JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64"));
+        }
+    }
+
+    // UBJSON: write number (signed integer)
+    template<typename NumberType, typename std::enable_if<
+                 std::is_signed<NumberType>::value and
+                 not std::is_floating_point<NumberType>::value, int>::type = 0>
+    void write_number_with_ubjson_prefix(const NumberType n,
+                                         const bool add_prefix)
+    {
+        if ((std::numeric_limits<int8_t>::min)() <= n and n <= (std::numeric_limits<int8_t>::max)())
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('i'));  // int8
+            }
+            write_number(static_cast<int8_t>(n));
+        }
+        else if (static_cast<int64_t>((std::numeric_limits<uint8_t>::min)()) <= n and n <= static_cast<int64_t>((std::numeric_limits<uint8_t>::max)()))
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('U'));  // uint8
+            }
+            write_number(static_cast<uint8_t>(n));
+        }
+        else if ((std::numeric_limits<int16_t>::min)() <= n and n <= (std::numeric_limits<int16_t>::max)())
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('I'));  // int16
+            }
+            write_number(static_cast<int16_t>(n));
+        }
+        else if ((std::numeric_limits<int32_t>::min)() <= n and n <= (std::numeric_limits<int32_t>::max)())
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('l'));  // int32
+            }
+            write_number(static_cast<int32_t>(n));
+        }
+        else if ((std::numeric_limits<int64_t>::min)() <= n and n <= (std::numeric_limits<int64_t>::max)())
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('L'));  // int64
+            }
+            write_number(static_cast<int64_t>(n));
+        }
+        // LCOV_EXCL_START
+        else
+        {
+            JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64"));
+        }
+        // LCOV_EXCL_STOP
+    }
+
+    /*!
+    @brief determine the type prefix of container values
+
+    @note This function does not need to be 100% accurate when it comes to
+          integer limits. In case a number exceeds the limits of int64_t,
+          this will be detected by a later call to function
+          write_number_with_ubjson_prefix. Therefore, we return 'L' for any
+          value that does not fit the previous limits.
+    */
+    CharType ubjson_prefix(const BasicJsonType& j) const noexcept
+    {
+        switch (j.type())
+        {
+            case value_t::null:
+                return 'Z';
+
+            case value_t::boolean:
+                return j.m_value.boolean ? 'T' : 'F';
+
+            case value_t::number_integer:
+            {
+                if ((std::numeric_limits<int8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)())
+                {
+                    return 'i';
+                }
+                if ((std::numeric_limits<uint8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    return 'U';
+                }
+                if ((std::numeric_limits<int16_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)())
+                {
+                    return 'I';
+                }
+                if ((std::numeric_limits<int32_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)())
+                {
+                    return 'l';
+                }
+                // no check and assume int64_t (see note above)
+                return 'L';
+            }
+
+            case value_t::number_unsigned:
+            {
+                if (j.m_value.number_unsigned <= (std::numeric_limits<int8_t>::max)())
+                {
+                    return 'i';
+                }
+                if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    return 'U';
+                }
+                if (j.m_value.number_unsigned <= (std::numeric_limits<int16_t>::max)())
+                {
+                    return 'I';
+                }
+                if (j.m_value.number_unsigned <= (std::numeric_limits<int32_t>::max)())
+                {
+                    return 'l';
+                }
+                // no check and assume int64_t (see note above)
+                return 'L';
+            }
+
+            case value_t::number_float:
+                return get_ubjson_float_prefix(j.m_value.number_float);
+
+            case value_t::string:
+                return 'S';
+
+            case value_t::array:
+                return '[';
+
+            case value_t::object:
+                return '{';
+
+            default:  // discarded values
+                return 'N';
+        }
+    }
+
+    static constexpr CharType get_ubjson_float_prefix(float /*unused*/)
+    {
+        return 'd';  // float 32
+    }
+
+    static constexpr CharType get_ubjson_float_prefix(double /*unused*/)
+    {
+        return 'D';  // float 64
+    }
+
+    ///////////////////////
+    // Utility functions //
+    ///////////////////////
+
     /*
     @brief write a number to output input
-
     @param[in] n number of type @a NumberType
     @tparam NumberType the type of the number
+    @tparam OutputIsLittleEndian Set to true if output data is
+                                 required to be little endian
 
     @note This function needs to respect the system's endianess, because bytes
-          in CBOR and MessagePack are stored in network order (big endian) and
-          therefore need reordering on little endian systems.
+          in CBOR, MessagePack, and UBJSON are stored in network order (big
+          endian) and therefore need reordering on little endian systems.
     */
-    template<typename NumberType> void write_number(NumberType n)
+    template<typename NumberType, bool OutputIsLittleEndian = false>
+    void write_number(const NumberType n)
     {
         // step 1: write number to array of length NumberType
         std::array<CharType, sizeof(NumberType)> vec;
         std::memcpy(vec.data(), &n, sizeof(NumberType));
 
         // step 2: write array to output (with possible reordering)
-        if (is_little_endian)
+        if (is_little_endian and not OutputIsLittleEndian)
         {
             // reverse byte order prior to conversion if necessary
             std::reverse(vec.begin(), vec.end());
@@ -6058,6 +9728,47 @@ class binary_writer
         oa->write_characters(vec.data(), sizeof(NumberType));
     }
 
+  public:
+    // The following to_char_type functions are implement the conversion
+    // between uint8_t and CharType. In case CharType is not unsigned,
+    // such a conversion is required to allow values greater than 128.
+    // See <https://github.com/nlohmann/json/issues/1286> for a discussion.
+    template < typename C = CharType,
+               enable_if_t < std::is_signed<C>::value and std::is_signed<char>::value > * = nullptr >
+    static constexpr CharType to_char_type(std::uint8_t x) noexcept
+    {
+        return *reinterpret_cast<char*>(&x);
+    }
+
+    template < typename C = CharType,
+               enable_if_t < std::is_signed<C>::value and std::is_unsigned<char>::value > * = nullptr >
+    static CharType to_char_type(std::uint8_t x) noexcept
+    {
+        static_assert(sizeof(std::uint8_t) == sizeof(CharType), "size of CharType must be equal to std::uint8_t");
+        static_assert(std::is_pod<CharType>::value, "CharType must be POD");
+        CharType result;
+        std::memcpy(&result, &x, sizeof(x));
+        return result;
+    }
+
+    template<typename C = CharType,
+             enable_if_t<std::is_unsigned<C>::value>* = nullptr>
+    static constexpr CharType to_char_type(std::uint8_t x) noexcept
+    {
+        return x;
+    }
+
+    template < typename InputCharType, typename C = CharType,
+               enable_if_t <
+                   std::is_signed<C>::value and
+                   std::is_signed<char>::value and
+                   std::is_same<char, typename std::remove_cv<InputCharType>::type>::value
+                   > * = nullptr >
+    static constexpr CharType to_char_type(InputCharType x) noexcept
+    {
+        return x;
+    }
+
   private:
     /// whether we can assume little endianess
     const bool is_little_endian = binary_reader<BasicJsonType>::little_endianess();
@@ -6065,11 +9776,1150 @@ class binary_writer
     /// the output
     output_adapter_t<CharType> oa = nullptr;
 };
+}  // namespace detail
+}  // namespace nlohmann
 
+// #include <nlohmann/detail/output/serializer.hpp>
+
+
+#include <algorithm> // reverse, remove, fill, find, none_of
+#include <array> // array
+#include <cassert> // assert
+#include <ciso646> // and, or
+#include <clocale> // localeconv, lconv
+#include <cmath> // labs, isfinite, isnan, signbit
+#include <cstddef> // size_t, ptrdiff_t
+#include <cstdint> // uint8_t
+#include <cstdio> // snprintf
+#include <limits> // numeric_limits
+#include <string> // string
+#include <type_traits> // is_same
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/conversions/to_chars.hpp>
+
+
+#include <cassert> // assert
+#include <ciso646> // or, and, not
+#include <cmath>   // signbit, isfinite
+#include <cstdint> // intN_t, uintN_t
+#include <cstring> // memcpy, memmove
+
+namespace nlohmann
+{
+namespace detail
+{
+
+/*!
+@brief implements the Grisu2 algorithm for binary to decimal floating-point
+conversion.
+
+This implementation is a slightly modified version of the reference
+implementation which may be obtained from
+http://florian.loitsch.com/publications (bench.tar.gz).
+
+The code is distributed under the MIT license, Copyright (c) 2009 Florian Loitsch.
+
+For a detailed description of the algorithm see:
+
+[1] Loitsch, "Printing Floating-Point Numbers Quickly and Accurately with
+    Integers", Proceedings of the ACM SIGPLAN 2010 Conference on Programming
+    Language Design and Implementation, PLDI 2010
+[2] Burger, Dybvig, "Printing Floating-Point Numbers Quickly and Accurately",
+    Proceedings of the ACM SIGPLAN 1996 Conference on Programming Language
+    Design and Implementation, PLDI 1996
+*/
+namespace dtoa_impl
+{
+
+template <typename Target, typename Source>
+Target reinterpret_bits(const Source source)
+{
+    static_assert(sizeof(Target) == sizeof(Source), "size mismatch");
+
+    Target target;
+    std::memcpy(&target, &source, sizeof(Source));
+    return target;
+}
+
+struct diyfp // f * 2^e
+{
+    static constexpr int kPrecision = 64; // = q
+
+    uint64_t f = 0;
+    int e = 0;
+
+    constexpr diyfp(uint64_t f_, int e_) noexcept : f(f_), e(e_) {}
+
+    /*!
+    @brief returns x - y
+    @pre x.e == y.e and x.f >= y.f
+    */
+    static diyfp sub(const diyfp& x, const diyfp& y) noexcept
+    {
+        assert(x.e == y.e);
+        assert(x.f >= y.f);
+
+        return {x.f - y.f, x.e};
+    }
+
+    /*!
+    @brief returns x * y
+    @note The result is rounded. (Only the upper q bits are returned.)
+    */
+    static diyfp mul(const diyfp& x, const diyfp& y) noexcept
+    {
+        static_assert(kPrecision == 64, "internal error");
+
+        // Computes:
+        //  f = round((x.f * y.f) / 2^q)
+        //  e = x.e + y.e + q
+
+        // Emulate the 64-bit * 64-bit multiplication:
+        //
+        // p = u * v
+        //   = (u_lo + 2^32 u_hi) (v_lo + 2^32 v_hi)
+        //   = (u_lo v_lo         ) + 2^32 ((u_lo v_hi         ) + (u_hi v_lo         )) + 2^64 (u_hi v_hi         )
+        //   = (p0                ) + 2^32 ((p1                ) + (p2                )) + 2^64 (p3                )
+        //   = (p0_lo + 2^32 p0_hi) + 2^32 ((p1_lo + 2^32 p1_hi) + (p2_lo + 2^32 p2_hi)) + 2^64 (p3                )
+        //   = (p0_lo             ) + 2^32 (p0_hi + p1_lo + p2_lo                      ) + 2^64 (p1_hi + p2_hi + p3)
+        //   = (p0_lo             ) + 2^32 (Q                                          ) + 2^64 (H                 )
+        //   = (p0_lo             ) + 2^32 (Q_lo + 2^32 Q_hi                           ) + 2^64 (H                 )
+        //
+        // (Since Q might be larger than 2^32 - 1)
+        //
+        //   = (p0_lo + 2^32 Q_lo) + 2^64 (Q_hi + H)
+        //
+        // (Q_hi + H does not overflow a 64-bit int)
+        //
+        //   = p_lo + 2^64 p_hi
+
+        const uint64_t u_lo = x.f & 0xFFFFFFFF;
+        const uint64_t u_hi = x.f >> 32;
+        const uint64_t v_lo = y.f & 0xFFFFFFFF;
+        const uint64_t v_hi = y.f >> 32;
+
+        const uint64_t p0 = u_lo * v_lo;
+        const uint64_t p1 = u_lo * v_hi;
+        const uint64_t p2 = u_hi * v_lo;
+        const uint64_t p3 = u_hi * v_hi;
+
+        const uint64_t p0_hi = p0 >> 32;
+        const uint64_t p1_lo = p1 & 0xFFFFFFFF;
+        const uint64_t p1_hi = p1 >> 32;
+        const uint64_t p2_lo = p2 & 0xFFFFFFFF;
+        const uint64_t p2_hi = p2 >> 32;
+
+        uint64_t Q = p0_hi + p1_lo + p2_lo;
+
+        // The full product might now be computed as
+        //
+        // p_hi = p3 + p2_hi + p1_hi + (Q >> 32)
+        // p_lo = p0_lo + (Q << 32)
+        //
+        // But in this particular case here, the full p_lo is not required.
+        // Effectively we only need to add the highest bit in p_lo to p_hi (and
+        // Q_hi + 1 does not overflow).
+
+        Q += uint64_t{1} << (64 - 32 - 1); // round, ties up
+
+        const uint64_t h = p3 + p2_hi + p1_hi + (Q >> 32);
+
+        return {h, x.e + y.e + 64};
+    }
+
+    /*!
+    @brief normalize x such that the significand is >= 2^(q-1)
+    @pre x.f != 0
+    */
+    static diyfp normalize(diyfp x) noexcept
+    {
+        assert(x.f != 0);
+
+        while ((x.f >> 63) == 0)
+        {
+            x.f <<= 1;
+            x.e--;
+        }
+
+        return x;
+    }
+
+    /*!
+    @brief normalize x such that the result has the exponent E
+    @pre e >= x.e and the upper e - x.e bits of x.f must be zero.
+    */
+    static diyfp normalize_to(const diyfp& x, const int target_exponent) noexcept
+    {
+        const int delta = x.e - target_exponent;
+
+        assert(delta >= 0);
+        assert(((x.f << delta) >> delta) == x.f);
+
+        return {x.f << delta, target_exponent};
+    }
+};
+
+struct boundaries
+{
+    diyfp w;
+    diyfp minus;
+    diyfp plus;
+};
+
+/*!
+Compute the (normalized) diyfp representing the input number 'value' and its
+boundaries.
+
+@pre value must be finite and positive
+*/
+template <typename FloatType>
+boundaries compute_boundaries(FloatType value)
+{
+    assert(std::isfinite(value));
+    assert(value > 0);
+
+    // Convert the IEEE representation into a diyfp.
+    //
+    // If v is denormal:
+    //      value = 0.F * 2^(1 - bias) = (          F) * 2^(1 - bias - (p-1))
+    // If v is normalized:
+    //      value = 1.F * 2^(E - bias) = (2^(p-1) + F) * 2^(E - bias - (p-1))
+
+    static_assert(std::numeric_limits<FloatType>::is_iec559,
+                  "internal error: dtoa_short requires an IEEE-754 floating-point implementation");
+
+    constexpr int      kPrecision = std::numeric_limits<FloatType>::digits; // = p (includes the hidden bit)
+    constexpr int      kBias      = std::numeric_limits<FloatType>::max_exponent - 1 + (kPrecision - 1);
+    constexpr int      kMinExp    = 1 - kBias;
+    constexpr uint64_t kHiddenBit = uint64_t{1} << (kPrecision - 1); // = 2^(p-1)
+
+    using bits_type = typename std::conditional< kPrecision == 24, uint32_t, uint64_t >::type;
+
+    const uint64_t bits = reinterpret_bits<bits_type>(value);
+    const uint64_t E = bits >> (kPrecision - 1);
+    const uint64_t F = bits & (kHiddenBit - 1);
+
+    const bool is_denormal = (E == 0);
+    const diyfp v = is_denormal
+                    ? diyfp(F, kMinExp)
+                    : diyfp(F + kHiddenBit, static_cast<int>(E) - kBias);
+
+    // Compute the boundaries m- and m+ of the floating-point value
+    // v = f * 2^e.
+    //
+    // Determine v- and v+, the floating-point predecessor and successor if v,
+    // respectively.
+    //
+    //      v- = v - 2^e        if f != 2^(p-1) or e == e_min                (A)
+    //         = v - 2^(e-1)    if f == 2^(p-1) and e > e_min                (B)
+    //
+    //      v+ = v + 2^e
+    //
+    // Let m- = (v- + v) / 2 and m+ = (v + v+) / 2. All real numbers _strictly_
+    // between m- and m+ round to v, regardless of how the input rounding
+    // algorithm breaks ties.
+    //
+    //      ---+-------------+-------------+-------------+-------------+---  (A)
+    //         v-            m-            v             m+            v+
+    //
+    //      -----------------+------+------+-------------+-------------+---  (B)
+    //                       v-     m-     v             m+            v+
+
+    const bool lower_boundary_is_closer = (F == 0 and E > 1);
+    const diyfp m_plus = diyfp(2 * v.f + 1, v.e - 1);
+    const diyfp m_minus = lower_boundary_is_closer
+                          ? diyfp(4 * v.f - 1, v.e - 2)  // (B)
+                          : diyfp(2 * v.f - 1, v.e - 1); // (A)
+
+    // Determine the normalized w+ = m+.
+    const diyfp w_plus = diyfp::normalize(m_plus);
+
+    // Determine w- = m- such that e_(w-) = e_(w+).
+    const diyfp w_minus = diyfp::normalize_to(m_minus, w_plus.e);
+
+    return {diyfp::normalize(v), w_minus, w_plus};
+}
+
+// Given normalized diyfp w, Grisu needs to find a (normalized) cached
+// power-of-ten c, such that the exponent of the product c * w = f * 2^e lies
+// within a certain range [alpha, gamma] (Definition 3.2 from [1])
+//
+//      alpha <= e = e_c + e_w + q <= gamma
+//
+// or
+//
+//      f_c * f_w * 2^alpha <= f_c 2^(e_c) * f_w 2^(e_w) * 2^q
+//                          <= f_c * f_w * 2^gamma
+//
+// Since c and w are normalized, i.e. 2^(q-1) <= f < 2^q, this implies
+//
+//      2^(q-1) * 2^(q-1) * 2^alpha <= c * w * 2^q < 2^q * 2^q * 2^gamma
+//
+// or
+//
+//      2^(q - 2 + alpha) <= c * w < 2^(q + gamma)
+//
+// The choice of (alpha,gamma) determines the size of the table and the form of
+// the digit generation procedure. Using (alpha,gamma)=(-60,-32) works out well
+// in practice:
+//
+// The idea is to cut the number c * w = f * 2^e into two parts, which can be
+// processed independently: An integral part p1, and a fractional part p2:
+//
+//      f * 2^e = ( (f div 2^-e) * 2^-e + (f mod 2^-e) ) * 2^e
+//              = (f div 2^-e) + (f mod 2^-e) * 2^e
+//              = p1 + p2 * 2^e
+//
+// The conversion of p1 into decimal form requires a series of divisions and
+// modulos by (a power of) 10. These operations are faster for 32-bit than for
+// 64-bit integers, so p1 should ideally fit into a 32-bit integer. This can be
+// achieved by choosing
+//
+//      -e >= 32   or   e <= -32 := gamma
+//
+// In order to convert the fractional part
+//
+//      p2 * 2^e = p2 / 2^-e = d[-1] / 10^1 + d[-2] / 10^2 + ...
+//
+// into decimal form, the fraction is repeatedly multiplied by 10 and the digits
+// d[-i] are extracted in order:
+//
+//      (10 * p2) div 2^-e = d[-1]
+//      (10 * p2) mod 2^-e = d[-2] / 10^1 + ...
+//
+// The multiplication by 10 must not overflow. It is sufficient to choose
+//
+//      10 * p2 < 16 * p2 = 2^4 * p2 <= 2^64.
+//
+// Since p2 = f mod 2^-e < 2^-e,
+//
+//      -e <= 60   or   e >= -60 := alpha
+
+constexpr int kAlpha = -60;
+constexpr int kGamma = -32;
+
+struct cached_power // c = f * 2^e ~= 10^k
+{
+    uint64_t f;
+    int e;
+    int k;
+};
+
+/*!
+For a normalized diyfp w = f * 2^e, this function returns a (normalized) cached
+power-of-ten c = f_c * 2^e_c, such that the exponent of the product w * c
+satisfies (Definition 3.2 from [1])
+
+     alpha <= e_c + e + q <= gamma.
+*/
+inline cached_power get_cached_power_for_binary_exponent(int e)
+{
+    // Now
+    //
+    //      alpha <= e_c + e + q <= gamma                                    (1)
+    //      ==> f_c * 2^alpha <= c * 2^e * 2^q
+    //
+    // and since the c's are normalized, 2^(q-1) <= f_c,
+    //
+    //      ==> 2^(q - 1 + alpha) <= c * 2^(e + q)
+    //      ==> 2^(alpha - e - 1) <= c
+    //
+    // If c were an exakt power of ten, i.e. c = 10^k, one may determine k as
+    //
+    //      k = ceil( log_10( 2^(alpha - e - 1) ) )
+    //        = ceil( (alpha - e - 1) * log_10(2) )
+    //
+    // From the paper:
+    // "In theory the result of the procedure could be wrong since c is rounded,
+    //  and the computation itself is approximated [...]. In practice, however,
+    //  this simple function is sufficient."
+    //
+    // For IEEE double precision floating-point numbers converted into
+    // normalized diyfp's w = f * 2^e, with q = 64,
+    //
+    //      e >= -1022      (min IEEE exponent)
+    //           -52        (p - 1)
+    //           -52        (p - 1, possibly normalize denormal IEEE numbers)
+    //           -11        (normalize the diyfp)
+    //         = -1137
+    //
+    // and
+    //
+    //      e <= +1023      (max IEEE exponent)
+    //           -52        (p - 1)
+    //           -11        (normalize the diyfp)
+    //         = 960
+    //
+    // This binary exponent range [-1137,960] results in a decimal exponent
+    // range [-307,324]. One does not need to store a cached power for each
+    // k in this range. For each such k it suffices to find a cached power
+    // such that the exponent of the product lies in [alpha,gamma].
+    // This implies that the difference of the decimal exponents of adjacent
+    // table entries must be less than or equal to
+    //
+    //      floor( (gamma - alpha) * log_10(2) ) = 8.
+    //
+    // (A smaller distance gamma-alpha would require a larger table.)
+
+    // NB:
+    // Actually this function returns c, such that -60 <= e_c + e + 64 <= -34.
+
+    constexpr int kCachedPowersSize = 79;
+    constexpr int kCachedPowersMinDecExp = -300;
+    constexpr int kCachedPowersDecStep = 8;
+
+    static constexpr cached_power kCachedPowers[] =
+    {
+        { 0xAB70FE17C79AC6CA, -1060, -300 },
+        { 0xFF77B1FCBEBCDC4F, -1034, -292 },
+        { 0xBE5691EF416BD60C, -1007, -284 },
+        { 0x8DD01FAD907FFC3C,  -980, -276 },
+        { 0xD3515C2831559A83,  -954, -268 },
+        { 0x9D71AC8FADA6C9B5,  -927, -260 },
+        { 0xEA9C227723EE8BCB,  -901, -252 },
+        { 0xAECC49914078536D,  -874, -244 },
+        { 0x823C12795DB6CE57,  -847, -236 },
+        { 0xC21094364DFB5637,  -821, -228 },
+        { 0x9096EA6F3848984F,  -794, -220 },
+        { 0xD77485CB25823AC7,  -768, -212 },
+        { 0xA086CFCD97BF97F4,  -741, -204 },
+        { 0xEF340A98172AACE5,  -715, -196 },
+        { 0xB23867FB2A35B28E,  -688, -188 },
+        { 0x84C8D4DFD2C63F3B,  -661, -180 },
+        { 0xC5DD44271AD3CDBA,  -635, -172 },
+        { 0x936B9FCEBB25C996,  -608, -164 },
+        { 0xDBAC6C247D62A584,  -582, -156 },
+        { 0xA3AB66580D5FDAF6,  -555, -148 },
+        { 0xF3E2F893DEC3F126,  -529, -140 },
+        { 0xB5B5ADA8AAFF80B8,  -502, -132 },
+        { 0x87625F056C7C4A8B,  -475, -124 },
+        { 0xC9BCFF6034C13053,  -449, -116 },
+        { 0x964E858C91BA2655,  -422, -108 },
+        { 0xDFF9772470297EBD,  -396, -100 },
+        { 0xA6DFBD9FB8E5B88F,  -369,  -92 },
+        { 0xF8A95FCF88747D94,  -343,  -84 },
+        { 0xB94470938FA89BCF,  -316,  -76 },
+        { 0x8A08F0F8BF0F156B,  -289,  -68 },
+        { 0xCDB02555653131B6,  -263,  -60 },
+        { 0x993FE2C6D07B7FAC,  -236,  -52 },
+        { 0xE45C10C42A2B3B06,  -210,  -44 },
+        { 0xAA242499697392D3,  -183,  -36 },
+        { 0xFD87B5F28300CA0E,  -157,  -28 },
+        { 0xBCE5086492111AEB,  -130,  -20 },
+        { 0x8CBCCC096F5088CC,  -103,  -12 },
+        { 0xD1B71758E219652C,   -77,   -4 },
+        { 0x9C40000000000000,   -50,    4 },
+        { 0xE8D4A51000000000,   -24,   12 },
+        { 0xAD78EBC5AC620000,     3,   20 },
+        { 0x813F3978F8940984,    30,   28 },
+        { 0xC097CE7BC90715B3,    56,   36 },
+        { 0x8F7E32CE7BEA5C70,    83,   44 },
+        { 0xD5D238A4ABE98068,   109,   52 },
+        { 0x9F4F2726179A2245,   136,   60 },
+        { 0xED63A231D4C4FB27,   162,   68 },
+        { 0xB0DE65388CC8ADA8,   189,   76 },
+        { 0x83C7088E1AAB65DB,   216,   84 },
+        { 0xC45D1DF942711D9A,   242,   92 },
+        { 0x924D692CA61BE758,   269,  100 },
+        { 0xDA01EE641A708DEA,   295,  108 },
+        { 0xA26DA3999AEF774A,   322,  116 },
+        { 0xF209787BB47D6B85,   348,  124 },
+        { 0xB454E4A179DD1877,   375,  132 },
+        { 0x865B86925B9BC5C2,   402,  140 },
+        { 0xC83553C5C8965D3D,   428,  148 },
+        { 0x952AB45CFA97A0B3,   455,  156 },
+        { 0xDE469FBD99A05FE3,   481,  164 },
+        { 0xA59BC234DB398C25,   508,  172 },
+        { 0xF6C69A72A3989F5C,   534,  180 },
+        { 0xB7DCBF5354E9BECE,   561,  188 },
+        { 0x88FCF317F22241E2,   588,  196 },
+        { 0xCC20CE9BD35C78A5,   614,  204 },
+        { 0x98165AF37B2153DF,   641,  212 },
+        { 0xE2A0B5DC971F303A,   667,  220 },
+        { 0xA8D9D1535CE3B396,   694,  228 },
+        { 0xFB9B7CD9A4A7443C,   720,  236 },
+        { 0xBB764C4CA7A44410,   747,  244 },
+        { 0x8BAB8EEFB6409C1A,   774,  252 },
+        { 0xD01FEF10A657842C,   800,  260 },
+        { 0x9B10A4E5E9913129,   827,  268 },
+        { 0xE7109BFBA19C0C9D,   853,  276 },
+        { 0xAC2820D9623BF429,   880,  284 },
+        { 0x80444B5E7AA7CF85,   907,  292 },
+        { 0xBF21E44003ACDD2D,   933,  300 },
+        { 0x8E679C2F5E44FF8F,   960,  308 },
+        { 0xD433179D9C8CB841,   986,  316 },
+        { 0x9E19DB92B4E31BA9,  1013,  324 },
+    };
+
+    // This computation gives exactly the same results for k as
+    //      k = ceil((kAlpha - e - 1) * 0.30102999566398114)
+    // for |e| <= 1500, but doesn't require floating-point operations.
+    // NB: log_10(2) ~= 78913 / 2^18
+    assert(e >= -1500);
+    assert(e <=  1500);
+    const int f = kAlpha - e - 1;
+    const int k = (f * 78913) / (1 << 18) + static_cast<int>(f > 0);
+
+    const int index = (-kCachedPowersMinDecExp + k + (kCachedPowersDecStep - 1)) / kCachedPowersDecStep;
+    assert(index >= 0);
+    assert(index < kCachedPowersSize);
+    static_cast<void>(kCachedPowersSize); // Fix warning.
+
+    const cached_power cached = kCachedPowers[index];
+    assert(kAlpha <= cached.e + e + 64);
+    assert(kGamma >= cached.e + e + 64);
+
+    return cached;
+}
+
+/*!
+For n != 0, returns k, such that pow10 := 10^(k-1) <= n < 10^k.
+For n == 0, returns 1 and sets pow10 := 1.
+*/
+inline int find_largest_pow10(const uint32_t n, uint32_t& pow10)
+{
+    // LCOV_EXCL_START
+    if (n >= 1000000000)
+    {
+        pow10 = 1000000000;
+        return 10;
+    }
+    // LCOV_EXCL_STOP
+    else if (n >= 100000000)
+    {
+        pow10 = 100000000;
+        return  9;
+    }
+    else if (n >= 10000000)
+    {
+        pow10 = 10000000;
+        return  8;
+    }
+    else if (n >= 1000000)
+    {
+        pow10 = 1000000;
+        return  7;
+    }
+    else if (n >= 100000)
+    {
+        pow10 = 100000;
+        return  6;
+    }
+    else if (n >= 10000)
+    {
+        pow10 = 10000;
+        return  5;
+    }
+    else if (n >= 1000)
+    {
+        pow10 = 1000;
+        return  4;
+    }
+    else if (n >= 100)
+    {
+        pow10 = 100;
+        return  3;
+    }
+    else if (n >= 10)
+    {
+        pow10 = 10;
+        return  2;
+    }
+    else
+    {
+        pow10 = 1;
+        return 1;
+    }
+}
+
+inline void grisu2_round(char* buf, int len, uint64_t dist, uint64_t delta,
+                         uint64_t rest, uint64_t ten_k)
+{
+    assert(len >= 1);
+    assert(dist <= delta);
+    assert(rest <= delta);
+    assert(ten_k > 0);
+
+    //               <--------------------------- delta ---->
+    //                                  <---- dist --------->
+    // --------------[------------------+-------------------]--------------
+    //               M-                 w                   M+
+    //
+    //                                  ten_k
+    //                                <------>
+    //                                       <---- rest ---->
+    // --------------[------------------+----+--------------]--------------
+    //                                  w    V
+    //                                       = buf * 10^k
+    //
+    // ten_k represents a unit-in-the-last-place in the decimal representation
+    // stored in buf.
+    // Decrement buf by ten_k while this takes buf closer to w.
+
+    // The tests are written in this order to avoid overflow in unsigned
+    // integer arithmetic.
+
+    while (rest < dist
+            and delta - rest >= ten_k
+            and (rest + ten_k < dist or dist - rest > rest + ten_k - dist))
+    {
+        assert(buf[len - 1] != '0');
+        buf[len - 1]--;
+        rest += ten_k;
+    }
+}
+
+/*!
+Generates V = buffer * 10^decimal_exponent, such that M- <= V <= M+.
+M- and M+ must be normalized and share the same exponent -60 <= e <= -32.
+*/
+inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent,
+                             diyfp M_minus, diyfp w, diyfp M_plus)
+{
+    static_assert(kAlpha >= -60, "internal error");
+    static_assert(kGamma <= -32, "internal error");
+
+    // Generates the digits (and the exponent) of a decimal floating-point
+    // number V = buffer * 10^decimal_exponent in the range [M-, M+]. The diyfp's
+    // w, M- and M+ share the same exponent e, which satisfies alpha <= e <= gamma.
+    //
+    //               <--------------------------- delta ---->
+    //                                  <---- dist --------->
+    // --------------[------------------+-------------------]--------------
+    //               M-                 w                   M+
+    //
+    // Grisu2 generates the digits of M+ from left to right and stops as soon as
+    // V is in [M-,M+].
+
+    assert(M_plus.e >= kAlpha);
+    assert(M_plus.e <= kGamma);
+
+    uint64_t delta = diyfp::sub(M_plus, M_minus).f; // (significand of (M+ - M-), implicit exponent is e)
+    uint64_t dist  = diyfp::sub(M_plus, w      ).f; // (significand of (M+ - w ), implicit exponent is e)
+
+    // Split M+ = f * 2^e into two parts p1 and p2 (note: e < 0):
+    //
+    //      M+ = f * 2^e
+    //         = ((f div 2^-e) * 2^-e + (f mod 2^-e)) * 2^e
+    //         = ((p1        ) * 2^-e + (p2        )) * 2^e
+    //         = p1 + p2 * 2^e
+
+    const diyfp one(uint64_t{1} << -M_plus.e, M_plus.e);
+
+    auto p1 = static_cast<uint32_t>(M_plus.f >> -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.)
+    uint64_t p2 = M_plus.f & (one.f - 1);                    // p2 = f mod 2^-e
+
+    // 1)
+    //
+    // Generate the digits of the integral part p1 = d[n-1]...d[1]d[0]
+
+    assert(p1 > 0);
+
+    uint32_t pow10;
+    const int k = find_largest_pow10(p1, pow10);
+
+    //      10^(k-1) <= p1 < 10^k, pow10 = 10^(k-1)
+    //
+    //      p1 = (p1 div 10^(k-1)) * 10^(k-1) + (p1 mod 10^(k-1))
+    //         = (d[k-1]         ) * 10^(k-1) + (p1 mod 10^(k-1))
+    //
+    //      M+ = p1                                             + p2 * 2^e
+    //         = d[k-1] * 10^(k-1) + (p1 mod 10^(k-1))          + p2 * 2^e
+    //         = d[k-1] * 10^(k-1) + ((p1 mod 10^(k-1)) * 2^-e + p2) * 2^e
+    //         = d[k-1] * 10^(k-1) + (                         rest) * 2^e
+    //
+    // Now generate the digits d[n] of p1 from left to right (n = k-1,...,0)
+    //
+    //      p1 = d[k-1]...d[n] * 10^n + d[n-1]...d[0]
+    //
+    // but stop as soon as
+    //
+    //      rest * 2^e = (d[n-1]...d[0] * 2^-e + p2) * 2^e <= delta * 2^e
+
+    int n = k;
+    while (n > 0)
+    {
+        // Invariants:
+        //      M+ = buffer * 10^n + (p1 + p2 * 2^e)    (buffer = 0 for n = k)
+        //      pow10 = 10^(n-1) <= p1 < 10^n
+        //
+        const uint32_t d = p1 / pow10;  // d = p1 div 10^(n-1)
+        const uint32_t r = p1 % pow10;  // r = p1 mod 10^(n-1)
+        //
+        //      M+ = buffer * 10^n + (d * 10^(n-1) + r) + p2 * 2^e
+        //         = (buffer * 10 + d) * 10^(n-1) + (r + p2 * 2^e)
+        //
+        assert(d <= 9);
+        buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d
+        //
+        //      M+ = buffer * 10^(n-1) + (r + p2 * 2^e)
+        //
+        p1 = r;
+        n--;
+        //
+        //      M+ = buffer * 10^n + (p1 + p2 * 2^e)
+        //      pow10 = 10^n
+        //
+
+        // Now check if enough digits have been generated.
+        // Compute
+        //
+        //      p1 + p2 * 2^e = (p1 * 2^-e + p2) * 2^e = rest * 2^e
+        //
+        // Note:
+        // Since rest and delta share the same exponent e, it suffices to
+        // compare the significands.
+        const uint64_t rest = (uint64_t{p1} << -one.e) + p2;
+        if (rest <= delta)
+        {
+            // V = buffer * 10^n, with M- <= V <= M+.
+
+            decimal_exponent += n;
+
+            // We may now just stop. But instead look if the buffer could be
+            // decremented to bring V closer to w.
+            //
+            // pow10 = 10^n is now 1 ulp in the decimal representation V.
+            // The rounding procedure works with diyfp's with an implicit
+            // exponent of e.
+            //
+            //      10^n = (10^n * 2^-e) * 2^e = ulp * 2^e
+            //
+            const uint64_t ten_n = uint64_t{pow10} << -one.e;
+            grisu2_round(buffer, length, dist, delta, rest, ten_n);
+
+            return;
+        }
+
+        pow10 /= 10;
+        //
+        //      pow10 = 10^(n-1) <= p1 < 10^n
+        // Invariants restored.
+    }
+
+    // 2)
+    //
+    // The digits of the integral part have been generated:
+    //
+    //      M+ = d[k-1]...d[1]d[0] + p2 * 2^e
+    //         = buffer            + p2 * 2^e
+    //
+    // Now generate the digits of the fractional part p2 * 2^e.
+    //
+    // Note:
+    // No decimal point is generated: the exponent is adjusted instead.
+    //
+    // p2 actually represents the fraction
+    //
+    //      p2 * 2^e
+    //          = p2 / 2^-e
+    //          = d[-1] / 10^1 + d[-2] / 10^2 + ...
+    //
+    // Now generate the digits d[-m] of p1 from left to right (m = 1,2,...)
+    //
+    //      p2 * 2^e = d[-1]d[-2]...d[-m] * 10^-m
+    //                      + 10^-m * (d[-m-1] / 10^1 + d[-m-2] / 10^2 + ...)
+    //
+    // using
+    //
+    //      10^m * p2 = ((10^m * p2) div 2^-e) * 2^-e + ((10^m * p2) mod 2^-e)
+    //                = (                   d) * 2^-e + (                   r)
+    //
+    // or
+    //      10^m * p2 * 2^e = d + r * 2^e
+    //
+    // i.e.
+    //
+    //      M+ = buffer + p2 * 2^e
+    //         = buffer + 10^-m * (d + r * 2^e)
+    //         = (buffer * 10^m + d) * 10^-m + 10^-m * r * 2^e
+    //
+    // and stop as soon as 10^-m * r * 2^e <= delta * 2^e
+
+    assert(p2 > delta);
+
+    int m = 0;
+    for (;;)
+    {
+        // Invariant:
+        //      M+ = buffer * 10^-m + 10^-m * (d[-m-1] / 10 + d[-m-2] / 10^2 + ...) * 2^e
+        //         = buffer * 10^-m + 10^-m * (p2                                 ) * 2^e
+        //         = buffer * 10^-m + 10^-m * (1/10 * (10 * p2)                   ) * 2^e
+        //         = buffer * 10^-m + 10^-m * (1/10 * ((10*p2 div 2^-e) * 2^-e + (10*p2 mod 2^-e)) * 2^e
+        //
+        assert(p2 <= UINT64_MAX / 10);
+        p2 *= 10;
+        const uint64_t d = p2 >> -one.e;     // d = (10 * p2) div 2^-e
+        const uint64_t r = p2 & (one.f - 1); // r = (10 * p2) mod 2^-e
+        //
+        //      M+ = buffer * 10^-m + 10^-m * (1/10 * (d * 2^-e + r) * 2^e
+        //         = buffer * 10^-m + 10^-m * (1/10 * (d + r * 2^e))
+        //         = (buffer * 10 + d) * 10^(-m-1) + 10^(-m-1) * r * 2^e
+        //
+        assert(d <= 9);
+        buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d
+        //
+        //      M+ = buffer * 10^(-m-1) + 10^(-m-1) * r * 2^e
+        //
+        p2 = r;
+        m++;
+        //
+        //      M+ = buffer * 10^-m + 10^-m * p2 * 2^e
+        // Invariant restored.
+
+        // Check if enough digits have been generated.
+        //
+        //      10^-m * p2 * 2^e <= delta * 2^e
+        //              p2 * 2^e <= 10^m * delta * 2^e
+        //                    p2 <= 10^m * delta
+        delta *= 10;
+        dist  *= 10;
+        if (p2 <= delta)
+        {
+            break;
+        }
+    }
+
+    // V = buffer * 10^-m, with M- <= V <= M+.
+
+    decimal_exponent -= m;
+
+    // 1 ulp in the decimal representation is now 10^-m.
+    // Since delta and dist are now scaled by 10^m, we need to do the
+    // same with ulp in order to keep the units in sync.
+    //
+    //      10^m * 10^-m = 1 = 2^-e * 2^e = ten_m * 2^e
+    //
+    const uint64_t ten_m = one.f;
+    grisu2_round(buffer, length, dist, delta, p2, ten_m);
+
+    // By construction this algorithm generates the shortest possible decimal
+    // number (Loitsch, Theorem 6.2) which rounds back to w.
+    // For an input number of precision p, at least
+    //
+    //      N = 1 + ceil(p * log_10(2))
+    //
+    // decimal digits are sufficient to identify all binary floating-point
+    // numbers (Matula, "In-and-Out conversions").
+    // This implies that the algorithm does not produce more than N decimal
+    // digits.
+    //
+    //      N = 17 for p = 53 (IEEE double precision)
+    //      N = 9  for p = 24 (IEEE single precision)
+}
+
+/*!
+v = buf * 10^decimal_exponent
+len is the length of the buffer (number of decimal digits)
+The buffer must be large enough, i.e. >= max_digits10.
+*/
+inline void grisu2(char* buf, int& len, int& decimal_exponent,
+                   diyfp m_minus, diyfp v, diyfp m_plus)
+{
+    assert(m_plus.e == m_minus.e);
+    assert(m_plus.e == v.e);
+
+    //  --------(-----------------------+-----------------------)--------    (A)
+    //          m-                      v                       m+
+    //
+    //  --------------------(-----------+-----------------------)--------    (B)
+    //                      m-          v                       m+
+    //
+    // First scale v (and m- and m+) such that the exponent is in the range
+    // [alpha, gamma].
+
+    const cached_power cached = get_cached_power_for_binary_exponent(m_plus.e);
+
+    const diyfp c_minus_k(cached.f, cached.e); // = c ~= 10^-k
+
+    // The exponent of the products is = v.e + c_minus_k.e + q and is in the range [alpha,gamma]
+    const diyfp w       = diyfp::mul(v,       c_minus_k);
+    const diyfp w_minus = diyfp::mul(m_minus, c_minus_k);
+    const diyfp w_plus  = diyfp::mul(m_plus,  c_minus_k);
+
+    //  ----(---+---)---------------(---+---)---------------(---+---)----
+    //          w-                      w                       w+
+    //          = c*m-                  = c*v                   = c*m+
+    //
+    // diyfp::mul rounds its result and c_minus_k is approximated too. w, w- and
+    // w+ are now off by a small amount.
+    // In fact:
+    //
+    //      w - v * 10^k < 1 ulp
+    //
+    // To account for this inaccuracy, add resp. subtract 1 ulp.
+    //
+    //  --------+---[---------------(---+---)---------------]---+--------
+    //          w-  M-                  w                   M+  w+
+    //
+    // Now any number in [M-, M+] (bounds included) will round to w when input,
+    // regardless of how the input rounding algorithm breaks ties.
+    //
+    // And digit_gen generates the shortest possible such number in [M-, M+].
+    // Note that this does not mean that Grisu2 always generates the shortest
+    // possible number in the interval (m-, m+).
+    const diyfp M_minus(w_minus.f + 1, w_minus.e);
+    const diyfp M_plus (w_plus.f  - 1, w_plus.e );
+
+    decimal_exponent = -cached.k; // = -(-k) = k
+
+    grisu2_digit_gen(buf, len, decimal_exponent, M_minus, w, M_plus);
+}
+
+/*!
+v = buf * 10^decimal_exponent
+len is the length of the buffer (number of decimal digits)
+The buffer must be large enough, i.e. >= max_digits10.
+*/
+template <typename FloatType>
+void grisu2(char* buf, int& len, int& decimal_exponent, FloatType value)
+{
+    static_assert(diyfp::kPrecision >= std::numeric_limits<FloatType>::digits + 3,
+                  "internal error: not enough precision");
+
+    assert(std::isfinite(value));
+    assert(value > 0);
+
+    // If the neighbors (and boundaries) of 'value' are always computed for double-precision
+    // numbers, all float's can be recovered using strtod (and strtof). However, the resulting
+    // decimal representations are not exactly "short".
+    //
+    // The documentation for 'std::to_chars' (https://en.cppreference.com/w/cpp/utility/to_chars)
+    // says "value is converted to a string as if by std::sprintf in the default ("C") locale"
+    // and since sprintf promotes float's to double's, I think this is exactly what 'std::to_chars'
+    // does.
+    // On the other hand, the documentation for 'std::to_chars' requires that "parsing the
+    // representation using the corresponding std::from_chars function recovers value exactly". That
+    // indicates that single precision floating-point numbers should be recovered using
+    // 'std::strtof'.
+    //
+    // NB: If the neighbors are computed for single-precision numbers, there is a single float
+    //     (7.0385307e-26f) which can't be recovered using strtod. The resulting double precision
+    //     value is off by 1 ulp.
+#if 0
+    const boundaries w = compute_boundaries(static_cast<double>(value));
+#else
+    const boundaries w = compute_boundaries(value);
+#endif
+
+    grisu2(buf, len, decimal_exponent, w.minus, w.w, w.plus);
+}
+
+/*!
+@brief appends a decimal representation of e to buf
+@return a pointer to the element following the exponent.
+@pre -1000 < e < 1000
+*/
+inline char* append_exponent(char* buf, int e)
+{
+    assert(e > -1000);
+    assert(e <  1000);
+
+    if (e < 0)
+    {
+        e = -e;
+        *buf++ = '-';
+    }
+    else
+    {
+        *buf++ = '+';
+    }
+
+    auto k = static_cast<uint32_t>(e);
+    if (k < 10)
+    {
+        // Always print at least two digits in the exponent.
+        // This is for compatibility with printf("%g").
+        *buf++ = '0';
+        *buf++ = static_cast<char>('0' + k);
+    }
+    else if (k < 100)
+    {
+        *buf++ = static_cast<char>('0' + k / 10);
+        k %= 10;
+        *buf++ = static_cast<char>('0' + k);
+    }
+    else
+    {
+        *buf++ = static_cast<char>('0' + k / 100);
+        k %= 100;
+        *buf++ = static_cast<char>('0' + k / 10);
+        k %= 10;
+        *buf++ = static_cast<char>('0' + k);
+    }
+
+    return buf;
+}
+
+/*!
+@brief prettify v = buf * 10^decimal_exponent
+
+If v is in the range [10^min_exp, 10^max_exp) it will be printed in fixed-point
+notation. Otherwise it will be printed in exponential notation.
+
+@pre min_exp < 0
+@pre max_exp > 0
+*/
+inline char* format_buffer(char* buf, int len, int decimal_exponent,
+                           int min_exp, int max_exp)
+{
+    assert(min_exp < 0);
+    assert(max_exp > 0);
+
+    const int k = len;
+    const int n = len + decimal_exponent;
+
+    // v = buf * 10^(n-k)
+    // k is the length of the buffer (number of decimal digits)
+    // n is the position of the decimal point relative to the start of the buffer.
+
+    if (k <= n and n <= max_exp)
+    {
+        // digits[000]
+        // len <= max_exp + 2
+
+        std::memset(buf + k, '0', static_cast<size_t>(n - k));
+        // Make it look like a floating-point number (#362, #378)
+        buf[n + 0] = '.';
+        buf[n + 1] = '0';
+        return buf + (n + 2);
+    }
+
+    if (0 < n and n <= max_exp)
+    {
+        // dig.its
+        // len <= max_digits10 + 1
+
+        assert(k > n);
+
+        std::memmove(buf + (n + 1), buf + n, static_cast<size_t>(k - n));
+        buf[n] = '.';
+        return buf + (k + 1);
+    }
+
+    if (min_exp < n and n <= 0)
+    {
+        // 0.[000]digits
+        // len <= 2 + (-min_exp - 1) + max_digits10
+
+        std::memmove(buf + (2 + -n), buf, static_cast<size_t>(k));
+        buf[0] = '0';
+        buf[1] = '.';
+        std::memset(buf + 2, '0', static_cast<size_t>(-n));
+        return buf + (2 + (-n) + k);
+    }
+
+    if (k == 1)
+    {
+        // dE+123
+        // len <= 1 + 5
+
+        buf += 1;
+    }
+    else
+    {
+        // d.igitsE+123
+        // len <= max_digits10 + 1 + 5
+
+        std::memmove(buf + 2, buf + 1, static_cast<size_t>(k - 1));
+        buf[1] = '.';
+        buf += 1 + k;
+    }
+
+    *buf++ = 'e';
+    return append_exponent(buf, n - 1);
+}
+
+} // namespace dtoa_impl
+
+/*!
+@brief generates a decimal representation of the floating-point number value in [first, last).
+
+The format of the resulting decimal representation is similar to printf's %g
+format. Returns an iterator pointing past-the-end of the decimal representation.
+
+@note The input number must be finite, i.e. NaN's and Inf's are not supported.
+@note The buffer must be large enough.
+@note The result is NOT null-terminated.
+*/
+template <typename FloatType>
+char* to_chars(char* first, const char* last, FloatType value)
+{
+    static_cast<void>(last); // maybe unused - fix warning
+    assert(std::isfinite(value));
+
+    // Use signbit(value) instead of (value < 0) since signbit works for -0.
+    if (std::signbit(value))
+    {
+        value = -value;
+        *first++ = '-';
+    }
+
+    if (value == 0) // +-0
+    {
+        *first++ = '0';
+        // Make it look like a floating-point number (#362, #378)
+        *first++ = '.';
+        *first++ = '0';
+        return first;
+    }
+
+    assert(last - first >= std::numeric_limits<FloatType>::max_digits10);
+
+    // Compute v = buffer * 10^decimal_exponent.
+    // The decimal digits are stored in the buffer, which needs to be interpreted
+    // as an unsigned decimal integer.
+    // len is the length of the buffer, i.e. the number of decimal digits.
+    int len = 0;
+    int decimal_exponent = 0;
+    dtoa_impl::grisu2(first, len, decimal_exponent, value);
+
+    assert(len <= std::numeric_limits<FloatType>::max_digits10);
+
+    // Format the buffer like printf("%.*g", prec, value)
+    constexpr int kMinExp = -4;
+    // Use digits10 here to increase compatibility with version 2.
+    constexpr int kMaxExp = std::numeric_limits<FloatType>::digits10;
+
+    assert(last - first >= kMaxExp + 2);
+    assert(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits<FloatType>::max_digits10);
+    assert(last - first >= std::numeric_limits<FloatType>::max_digits10 + 6);
+
+    return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp);
+}
+
+} // namespace detail
+} // namespace nlohmann
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+// #include <nlohmann/detail/output/binary_writer.hpp>
+
+// #include <nlohmann/detail/output/output_adapters.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
 ///////////////////
 // serialization //
 ///////////////////
 
+/// how to treat decoding errors
+enum class error_handler_t
+{
+    strict,  ///< throw a type_error exception in case of invalid UTF-8
+    replace, ///< replace invalid UTF-8 sequences with U+FFFD
+    ignore   ///< ignore invalid UTF-8 sequences
+};
+
 template<typename BasicJsonType>
 class serializer
 {
@@ -6077,20 +10927,32 @@ class serializer
     using number_float_t = typename BasicJsonType::number_float_t;
     using number_integer_t = typename BasicJsonType::number_integer_t;
     using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    static constexpr uint8_t UTF8_ACCEPT = 0;
+    static constexpr uint8_t UTF8_REJECT = 1;
+
   public:
     /*!
     @param[in] s  output stream to serialize to
     @param[in] ichar  indentation character to use
+    @param[in] error_handler_  how to react on decoding errors
     */
-    serializer(output_adapter_t<char> s, const char ichar)
-        : o(std::move(s)), loc(std::localeconv()),
-          thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep)),
-          decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point)),
-          indent_char(ichar), indent_string(512, indent_char) {}
+    serializer(output_adapter_t<char> s, const char ichar,
+               error_handler_t error_handler_ = error_handler_t::strict)
+        : o(std::move(s))
+        , loc(std::localeconv())
+        , thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep))
+        , decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point))
+        , indent_char(ichar)
+        , indent_string(512, indent_char)
+        , error_handler(error_handler_)
+    {}
 
     // delete because of pointer members
     serializer(const serializer&) = delete;
     serializer& operator=(const serializer&) = delete;
+    serializer(serializer&&) = delete;
+    serializer& operator=(serializer&&) = delete;
+    ~serializer() = default;
 
     /*!
     @brief internal implementation of the serialization function
@@ -6303,323 +11165,249 @@ class serializer
 
   private:
     /*!
-    @brief returns the number of expected bytes following in UTF-8 string
-
-    @param[in]  u  the first byte of a UTF-8 string
-    @return  the number of expected bytes following
-    */
-    static constexpr std::size_t bytes_following(const uint8_t u)
-    {
-        return ((u <= 127) ? 0
-                : ((192 <= u and u <= 223) ? 1
-                   : ((224 <= u and u <= 239) ? 2
-                      : ((240 <= u and u <= 247) ? 3 : std::string::npos))));
-    }
+    @brief dump escaped string
 
-    /*!
-    @brief calculates the extra space to escape a JSON string
+    Escape a string by replacing certain special characters by a sequence of an
+    escape character (backslash) and another character and other control
+    characters by a sequence of "\u" followed by a four-digit hex
+    representation. The escaped string is written to output stream @a o.
 
     @param[in] s  the string to escape
     @param[in] ensure_ascii  whether to escape non-ASCII characters with
                              \uXXXX sequences
-    @return the number of characters required to escape string @a s
 
     @complexity Linear in the length of string @a s.
     */
-    static std::size_t extra_space(const string_t& s,
-                                   const bool ensure_ascii) noexcept
+    void dump_escaped(const string_t& s, const bool ensure_ascii)
     {
-        std::size_t res = 0;
+        uint32_t codepoint;
+        uint8_t state = UTF8_ACCEPT;
+        std::size_t bytes = 0;  // number of bytes written to string_buffer
+
+        // number of bytes written at the point of the last valid byte
+        std::size_t bytes_after_last_accept = 0;
+        std::size_t undumped_chars = 0;
 
         for (std::size_t i = 0; i < s.size(); ++i)
         {
-            switch (s[i])
-            {
-                // control characters that can be escaped with a backslash
-                case '"':
-                case '\\':
-                case '\b':
-                case '\f':
-                case '\n':
-                case '\r':
-                case '\t':
-                {
-                    // from c (1 byte) to \x (2 bytes)
-                    res += 1;
-                    break;
-                }
+            const auto byte = static_cast<uint8_t>(s[i]);
 
-                // control characters that need \uxxxx escaping
-                case 0x00:
-                case 0x01:
-                case 0x02:
-                case 0x03:
-                case 0x04:
-                case 0x05:
-                case 0x06:
-                case 0x07:
-                case 0x0B:
-                case 0x0E:
-                case 0x0F:
-                case 0x10:
-                case 0x11:
-                case 0x12:
-                case 0x13:
-                case 0x14:
-                case 0x15:
-                case 0x16:
-                case 0x17:
-                case 0x18:
-                case 0x19:
-                case 0x1A:
-                case 0x1B:
-                case 0x1C:
-                case 0x1D:
-                case 0x1E:
-                case 0x1F:
-                {
-                    // from c (1 byte) to \uxxxx (6 bytes)
-                    res += 5;
-                    break;
-                }
-
-                default:
+            switch (decode(state, codepoint, byte))
+            {
+                case UTF8_ACCEPT:  // decode found a new code point
                 {
-                    if (ensure_ascii and (s[i] & 0x80 or s[i] == 0x7F))
+                    switch (codepoint)
                     {
-                        const auto bytes = bytes_following(static_cast<uint8_t>(s[i]));
-                        // invalid characters will be detected by throw_if_invalid_utf8
-                        assert (bytes != std::string::npos);
-
-                        if (bytes == 3)
+                        case 0x08: // backspace
                         {
-                            // codepoints that need 4 bytes (i.e., 3 additional
-                            // bytes) in UTF-8 need a surrogate pair when \u
-                            // escaping is used: from 4 bytes to \uxxxx\uxxxx
-                            // (12 bytes)
-                            res += (12 - bytes - 1);
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = 'b';
+                            break;
                         }
-                        else
+
+                        case 0x09: // horizontal tab
                         {
-                            // from x bytes to \uxxxx (6 bytes)
-                            res += (6 - bytes - 1);
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = 't';
+                            break;
                         }
 
-                        // skip the additional bytes
-                        i += bytes;
-                    }
-                    break;
-                }
-            }
-        }
-
-        return res;
-    }
-
-    static void escape_codepoint(int codepoint, string_t& result, std::size_t& pos)
-    {
-        // expecting a proper codepoint
-        assert(0x00 <= codepoint and codepoint <= 0x10FFFF);
-
-        // the last written character was the backslash before the 'u'
-        assert(result[pos] == '\\');
-
-        // write the 'u'
-        result[++pos] = 'u';
-
-        // convert a number 0..15 to its hex representation (0..f)
-        static const std::array<char, 16> hexify =
-        {
-            {
-                '0', '1', '2', '3', '4', '5', '6', '7',
-                '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'
-            }
-        };
-
-        if (codepoint < 0x10000)
-        {
-            // codepoints U+0000..U+FFFF can be represented as \uxxxx.
-            result[++pos] = hexify[(codepoint >> 12) & 0x0F];
-            result[++pos] = hexify[(codepoint >> 8) & 0x0F];
-            result[++pos] = hexify[(codepoint >> 4) & 0x0F];
-            result[++pos] = hexify[codepoint & 0x0F];
-        }
-        else
-        {
-            // codepoints U+10000..U+10FFFF need a surrogate pair to be
-            // represented as \uxxxx\uxxxx.
-            // http://www.unicode.org/faq/utf_bom.html#utf16-4
-            codepoint -= 0x10000;
-            const int high_surrogate = 0xD800 | ((codepoint >> 10) & 0x3FF);
-            const int low_surrogate = 0xDC00 | (codepoint & 0x3FF);
-            result[++pos] = hexify[(high_surrogate >> 12) & 0x0F];
-            result[++pos] = hexify[(high_surrogate >> 8) & 0x0F];
-            result[++pos] = hexify[(high_surrogate >> 4) & 0x0F];
-            result[++pos] = hexify[high_surrogate & 0x0F];
-            ++pos;  // backslash is already in output
-            result[++pos] = 'u';
-            result[++pos] = hexify[(low_surrogate >> 12) & 0x0F];
-            result[++pos] = hexify[(low_surrogate >> 8) & 0x0F];
-            result[++pos] = hexify[(low_surrogate >> 4) & 0x0F];
-            result[++pos] = hexify[low_surrogate & 0x0F];
-        }
-
-        ++pos;
-    }
+                        case 0x0A: // newline
+                        {
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = 'n';
+                            break;
+                        }
 
-    /*!
-    @brief dump escaped string
+                        case 0x0C: // formfeed
+                        {
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = 'f';
+                            break;
+                        }
 
-    Escape a string by replacing certain special characters by a sequence of an
-    escape character (backslash) and another character and other control
-    characters by a sequence of "\u" followed by a four-digit hex
-    representation. The escaped string is written to output stream @a o.
+                        case 0x0D: // carriage return
+                        {
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = 'r';
+                            break;
+                        }
 
-    @param[in] s  the string to escape
-    @param[in] ensure_ascii  whether to escape non-ASCII characters with
-                             \uXXXX sequences
+                        case 0x22: // quotation mark
+                        {
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = '\"';
+                            break;
+                        }
 
-    @complexity Linear in the length of string @a s.
-    */
-    void dump_escaped(const string_t& s, const bool ensure_ascii) const
-    {
-        throw_if_invalid_utf8(s);
+                        case 0x5C: // reverse solidus
+                        {
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = '\\';
+                            break;
+                        }
 
-        const auto space = extra_space(s, ensure_ascii);
-        if (space == 0)
-        {
-            o->write_characters(s.c_str(), s.size());
-            return;
-        }
+                        default:
+                        {
+                            // escape control characters (0x00..0x1F) or, if
+                            // ensure_ascii parameter is used, non-ASCII characters
+                            if ((codepoint <= 0x1F) or (ensure_ascii and (codepoint >= 0x7F)))
+                            {
+                                if (codepoint <= 0xFFFF)
+                                {
+                                    (std::snprintf)(string_buffer.data() + bytes, 7, "\\u%04x",
+                                                    static_cast<uint16_t>(codepoint));
+                                    bytes += 6;
+                                }
+                                else
+                                {
+                                    (std::snprintf)(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x",
+                                                    static_cast<uint16_t>(0xD7C0 + (codepoint >> 10)),
+                                                    static_cast<uint16_t>(0xDC00 + (codepoint & 0x3FF)));
+                                    bytes += 12;
+                                }
+                            }
+                            else
+                            {
+                                // copy byte to buffer (all previous bytes
+                                // been copied have in default case above)
+                                string_buffer[bytes++] = s[i];
+                            }
+                            break;
+                        }
+                    }
 
-        // create a result string of necessary size
-        string_t result(s.size() + space, '\\');
-        std::size_t pos = 0;
+                    // write buffer and reset index; there must be 13 bytes
+                    // left, as this is the maximal number of bytes to be
+                    // written ("\uxxxx\uxxxx\0") for one code point
+                    if (string_buffer.size() - bytes < 13)
+                    {
+                        o->write_characters(string_buffer.data(), bytes);
+                        bytes = 0;
+                    }
 
-        for (std::size_t i = 0; i < s.size(); ++i)
-        {
-            switch (s[i])
-            {
-                case '"': // quotation mark (0x22)
-                {
-                    result[pos + 1] = '"';
-                    pos += 2;
+                    // remember the byte position of this accept
+                    bytes_after_last_accept = bytes;
+                    undumped_chars = 0;
                     break;
                 }
 
-                case '\\': // reverse solidus (0x5C)
+                case UTF8_REJECT:  // decode found invalid UTF-8 byte
                 {
-                    // nothing to change
-                    pos += 2;
-                    break;
-                }
+                    switch (error_handler)
+                    {
+                        case error_handler_t::strict:
+                        {
+                            std::string sn(3, '\0');
+                            (std::snprintf)(&sn[0], sn.size(), "%.2X", byte);
+                            JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + sn));
+                        }
 
-                case '\b': // backspace (0x08)
-                {
-                    result[pos + 1] = 'b';
-                    pos += 2;
-                    break;
-                }
+                        case error_handler_t::ignore:
+                        case error_handler_t::replace:
+                        {
+                            // in case we saw this character the first time, we
+                            // would like to read it again, because the byte
+                            // may be OK for itself, but just not OK for the
+                            // previous sequence
+                            if (undumped_chars > 0)
+                            {
+                                --i;
+                            }
 
-                case '\f': // formfeed (0x0C)
-                {
-                    result[pos + 1] = 'f';
-                    pos += 2;
+                            // reset length buffer to the last accepted index;
+                            // thus removing/ignoring the invalid characters
+                            bytes = bytes_after_last_accept;
+
+                            if (error_handler == error_handler_t::replace)
+                            {
+                                // add a replacement character
+                                if (ensure_ascii)
+                                {
+                                    string_buffer[bytes++] = '\\';
+                                    string_buffer[bytes++] = 'u';
+                                    string_buffer[bytes++] = 'f';
+                                    string_buffer[bytes++] = 'f';
+                                    string_buffer[bytes++] = 'f';
+                                    string_buffer[bytes++] = 'd';
+                                }
+                                else
+                                {
+                                    string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xEF');
+                                    string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBF');
+                                    string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBD');
+                                }
+                                bytes_after_last_accept = bytes;
+                            }
+
+                            undumped_chars = 0;
+
+                            // continue processing the string
+                            state = UTF8_ACCEPT;
+                            break;
+                        }
+                    }
                     break;
                 }
 
-                case '\n': // newline (0x0A)
+                default:  // decode found yet incomplete multi-byte code point
                 {
-                    result[pos + 1] = 'n';
-                    pos += 2;
+                    if (not ensure_ascii)
+                    {
+                        // code point will not be escaped - copy byte to buffer
+                        string_buffer[bytes++] = s[i];
+                    }
+                    ++undumped_chars;
                     break;
                 }
+            }
+        }
 
-                case '\r': // carriage return (0x0D)
+        // we finished processing the string
+        if (JSON_LIKELY(state == UTF8_ACCEPT))
+        {
+            // write buffer
+            if (bytes > 0)
+            {
+                o->write_characters(string_buffer.data(), bytes);
+            }
+        }
+        else
+        {
+            // we finish reading, but do not accept: string was incomplete
+            switch (error_handler)
+            {
+                case error_handler_t::strict:
                 {
-                    result[pos + 1] = 'r';
-                    pos += 2;
-                    break;
+                    std::string sn(3, '\0');
+                    (std::snprintf)(&sn[0], sn.size(), "%.2X", static_cast<uint8_t>(s.back()));
+                    JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + sn));
                 }
 
-                case '\t': // horizontal tab (0x09)
+                case error_handler_t::ignore:
                 {
-                    result[pos + 1] = 't';
-                    pos += 2;
+                    // write all accepted bytes
+                    o->write_characters(string_buffer.data(), bytes_after_last_accept);
                     break;
                 }
 
-                default:
+                case error_handler_t::replace:
                 {
-                    // escape control characters (0x00..0x1F) or, if
-                    // ensure_ascii parameter is used, non-ASCII characters
-                    if ((0x00 <= s[i] and s[i] <= 0x1F) or
-                            (ensure_ascii and (s[i] & 0x80 or s[i] == 0x7F)))
+                    // write all accepted bytes
+                    o->write_characters(string_buffer.data(), bytes_after_last_accept);
+                    // add a replacement character
+                    if (ensure_ascii)
                     {
-                        const auto bytes = bytes_following(static_cast<uint8_t>(s[i]));
-                        // invalid characters will be detected by throw_if_invalid_utf8
-                        assert (bytes != std::string::npos);
-
-                        // check that the additional bytes are present
-                        assert(i + bytes < s.size());
-
-                        // to use \uxxxx escaping, we first need to calculate
-                        // the codepoint from the UTF-8 bytes
-                        int codepoint = 0;
-
-                        // bytes is unsigned type:
-                        assert(bytes <= 3);
-                        switch (bytes)
-                        {
-                            case 0:
-                            {
-                                codepoint = s[i] & 0xFF;
-                                break;
-                            }
-
-                            case 1:
-                            {
-                                codepoint = ((s[i] & 0x3F) << 6)
-                                            + (s[i + 1] & 0x7F);
-                                break;
-                            }
-
-                            case 2:
-                            {
-                                codepoint = ((s[i] & 0x1F) << 12)
-                                            + ((s[i + 1] & 0x7F) << 6)
-                                            + (s[i + 2] & 0x7F);
-                                break;
-                            }
-
-                            case 3:
-                            {
-                                codepoint = ((s[i] & 0xF) << 18)
-                                            + ((s[i + 1] & 0x7F) << 12)
-                                            + ((s[i + 2] & 0x7F) << 6)
-                                            + (s[i + 3] & 0x7F);
-                                break;
-                            }
-
-                            default:
-                                break;  // LCOV_EXCL_LINE
-                        }
-
-                        escape_codepoint(codepoint, result, pos);
-                        i += bytes;
+                        o->write_characters("\\ufffd", 6);
                     }
                     else
                     {
-                        // all other characters are added as-is
-                        result[pos++] = s[i];
+                        o->write_characters("\xEF\xBF\xBD", 3);
                     }
                     break;
                 }
             }
         }
-
-        assert(pos == result.size());
-        o->write_characters(result.c_str(), result.size());
     }
 
     /*!
@@ -6644,7 +11432,7 @@ class serializer
             return;
         }
 
-        const bool is_negative = (x <= 0) and (x != 0);  // see issue #755
+        const bool is_negative = std::is_same<NumberType, number_integer_t>::value and not (x >= 0);  // see issue #755
         std::size_t i = 0;
 
         while (x != 0)
@@ -6679,17 +11467,39 @@ class serializer
     void dump_float(number_float_t x)
     {
         // NaN / inf
-        if (not std::isfinite(x) or std::isnan(x))
+        if (not std::isfinite(x))
         {
             o->write_characters("null", 4);
             return;
         }
 
-        // get number of digits for a text -> float -> text round-trip
-        static constexpr auto d = std::numeric_limits<number_float_t>::digits10;
+        // If number_float_t is an IEEE-754 single or double precision number,
+        // use the Grisu2 algorithm to produce short numbers which are
+        // guaranteed to round-trip, using strtof and strtod, resp.
+        //
+        // NB: The test below works if <long double> == <double>.
+        static constexpr bool is_ieee_single_or_double
+            = (std::numeric_limits<number_float_t>::is_iec559 and std::numeric_limits<number_float_t>::digits == 24 and std::numeric_limits<number_float_t>::max_exponent == 128) or
+              (std::numeric_limits<number_float_t>::is_iec559 and std::numeric_limits<number_float_t>::digits == 53 and std::numeric_limits<number_float_t>::max_exponent == 1024);
+
+        dump_float(x, std::integral_constant<bool, is_ieee_single_or_double>());
+    }
+
+    void dump_float(number_float_t x, std::true_type /*is_ieee_single_or_double*/)
+    {
+        char* begin = number_buffer.data();
+        char* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x);
+
+        o->write_characters(begin, static_cast<size_t>(end - begin));
+    }
+
+    void dump_float(number_float_t x, std::false_type /*is_ieee_single_or_double*/)
+    {
+        // get number of digits for a float -> text -> float round-trip
+        static constexpr auto d = std::numeric_limits<number_float_t>::max_digits10;
 
         // the actual conversion
-        std::ptrdiff_t len = snprintf(number_buffer.data(), number_buffer.size(), "%.*g", d, x);
+        std::ptrdiff_t len = (std::snprintf)(number_buffer.data(), number_buffer.size(), "%.*g", d, x);
 
         // negative value indicates an error
         assert(len > 0);
@@ -6744,15 +11554,16 @@ class serializer
     followed.
 
     @param[in,out] state  the state of the decoding
+    @param[in,out] codep  codepoint (valid only if resulting state is UTF8_ACCEPT)
     @param[in] byte       next byte to decode
+    @return               new state
 
-    @note The function has been edited: a std::array is used and the code
-          point is not calculated.
+    @note The function has been edited: a std::array is used.
 
     @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
     @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/
     */
-    static void decode(uint8_t& state, const uint8_t byte)
+    static uint8_t decode(uint8_t& state, uint32_t& codep, const uint8_t byte) noexcept
     {
         static const std::array<uint8_t, 400> utf8d =
         {
@@ -6775,42 +11586,13 @@ class serializer
         };
 
         const uint8_t type = utf8d[byte];
-        state = utf8d[256u + state * 16u + type];
-    }
 
-    /*!
-    @brief throw an exception if a string is not UTF-8 encoded
-
-    @param[in] str  UTF-8 string to check
-    @throw type_error.316 if passed string is not UTF-8 encoded
+        codep = (state != UTF8_ACCEPT)
+                ? (byte & 0x3fu) | (codep << 6)
+                : static_cast<uint32_t>(0xff >> type) & (byte);
 
-    @since version 3.0.0
-    */
-    static void throw_if_invalid_utf8(const std::string& str)
-    {
-        // start with state 0 (= accept)
-        uint8_t state = 0;
-
-        for (size_t i = 0; i < str.size(); ++i)
-        {
-            const auto byte = static_cast<uint8_t>(str[i]);
-            decode(state, byte);
-            if (state == 1)
-            {
-                // state 1 means reject
-                std::stringstream ss;
-                ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << static_cast<int>(byte);
-                JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + ss.str()));
-            }
-        }
-
-        if (state != 0)
-        {
-            // we finish reading, but do not accept: string was incomplete
-            std::stringstream ss;
-            ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << static_cast<int>(static_cast<uint8_t>(str.back()));
-            JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + ss.str()));
-        }
+        state = utf8d[256u + state * 16u + type];
+        return state;
     }
 
   private:
@@ -6827,13 +11609,33 @@ class serializer
     /// the locale's decimal point character
     const char decimal_point = '\0';
 
+    /// string buffer
+    std::array<char, 512> string_buffer{{}};
+
     /// the indentation character
     const char indent_char;
-
     /// the indentation string
     string_t indent_string;
+
+    /// error_handler how to react on decoding errors
+    const error_handler_t error_handler;
 };
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/json_ref.hpp>
 
+
+#include <initializer_list>
+#include <utility>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
 template<typename BasicJsonType>
 class json_ref
 {
@@ -6852,15 +11654,19 @@ class json_ref
         : owned_value(init), value_ref(&owned_value), is_rvalue(true)
     {}
 
-    template<class... Args>
-    json_ref(Args&& ... args)
-        : owned_value(std::forward<Args>(args)...), value_ref(&owned_value), is_rvalue(true)
-    {}
+    template <
+        class... Args,
+        enable_if_t<std::is_constructible<value_type, Args...>::value, int> = 0 >
+    json_ref(Args && ... args)
+        : owned_value(std::forward<Args>(args)...), value_ref(&owned_value),
+          is_rvalue(true) {}
 
     // class should be movable only
     json_ref(json_ref&&) = default;
     json_ref(const json_ref&) = delete;
     json_ref& operator=(const json_ref&) = delete;
+    json_ref& operator=(json_ref&&) = delete;
+    ~json_ref() = default;
 
     value_type moved_or_copied() const
     {
@@ -6886,74 +11692,30 @@ class json_ref
     value_type* value_ref = nullptr;
     const bool is_rvalue;
 };
+}  // namespace detail
+}  // namespace nlohmann
 
-} // namespace detail
-
-/// namespace to hold default `to_json` / `from_json` functions
-namespace
-{
-constexpr const auto& to_json = detail::static_const<detail::to_json_fn>::value;
-constexpr const auto& from_json = detail::static_const<detail::from_json_fn>::value;
-}
-
-
-/*!
-@brief default JSONSerializer template argument
-
-This serializer ignores the template arguments and uses ADL
-([argument-dependent lookup](http://en.cppreference.com/w/cpp/language/adl))
-for serialization.
-*/
-template<typename, typename>
-struct adl_serializer
-{
-    /*!
-    @brief convert a JSON value to any value type
-
-    This function is usually called by the `get()` function of the
-    @ref basic_json class (either explicit or via conversion operators).
-
-    @param[in] j         JSON value to read from
-    @param[in,out] val  value to write to
-    */
-    template<typename BasicJsonType, typename ValueType>
-    static void from_json(BasicJsonType&& j, ValueType& val) noexcept(
-        noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), val)))
-    {
-        ::nlohmann::from_json(std::forward<BasicJsonType>(j), val);
-    }
+// #include <nlohmann/detail/json_pointer.hpp>
 
-    /*!
-    @brief convert any value type to a JSON value
 
-    This function is usually called by the constructors of the @ref basic_json
-    class.
+#include <cassert> // assert
+#include <numeric> // accumulate
+#include <string> // string
+#include <vector> // vector
 
-    @param[in,out] j  JSON value to write to
-    @param[in] val     value to read from
-    */
-    template<typename BasicJsonType, typename ValueType>
-    static void to_json(BasicJsonType& j, ValueType&& val) noexcept(
-        noexcept(::nlohmann::to_json(j, std::forward<ValueType>(val))))
-    {
-        ::nlohmann::to_json(j, std::forward<ValueType>(val));
-    }
-};
+// #include <nlohmann/detail/macro_scope.hpp>
 
-/*!
-@brief JSON Pointer
+// #include <nlohmann/detail/exceptions.hpp>
 
-A JSON pointer defines a string syntax for identifying a specific value
-within a JSON document. It can be used with functions `at` and
-`operator[]`. Furthermore, JSON pointers are the base for JSON patches.
+// #include <nlohmann/detail/value_t.hpp>
 
-@sa [RFC 6901](https://tools.ietf.org/html/rfc6901)
 
-@since version 2.0.0
-*/
+namespace nlohmann
+{
+template<typename BasicJsonType>
 class json_pointer
 {
-    /// allow basic_json to access private members
+    // allow basic_json to access private members
     NLOHMANN_BASIC_JSON_TPL_DECLARATION
     friend class basic_json;
 
@@ -6967,19 +11729,21 @@ class json_pointer
     @param[in] s  string representing the JSON pointer; if omitted, the empty
                   string is assumed which references the whole JSON value
 
-    @throw parse_error.107 if the given JSON pointer @a s is nonempty and
-    does not begin with a slash (`/`); see example below
+    @throw parse_error.107 if the given JSON pointer @a s is nonempty and does
+                           not begin with a slash (`/`); see example below
 
-    @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s
-    is not followed by `0` (representing `~`) or `1` (representing `/`);
-    see example below
+    @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s is
+    not followed by `0` (representing `~`) or `1` (representing `/`); see
+    example below
 
-    @liveexample{The example shows the construction several valid JSON
-    pointers as well as the exceptional behavior.,json_pointer}
+    @liveexample{The example shows the construction several valid JSON pointers
+    as well as the exceptional behavior.,json_pointer}
 
     @since version 2.0.0
     */
-    explicit json_pointer(const std::string& s = "") : reference_tokens(split(s)) {}
+    explicit json_pointer(const std::string& s = "")
+        : reference_tokens(split(s))
+    {}
 
     /*!
     @brief return a string representation of the JSON pointer
@@ -6996,7 +11760,7 @@ class json_pointer
 
     @since version 2.0.0
     */
-    std::string to_string() const noexcept
+    std::string to_string() const
     {
         return std::accumulate(reference_tokens.begin(), reference_tokens.end(),
                                std::string{},
@@ -7021,7 +11785,7 @@ class json_pointer
     */
     static int array_index(const std::string& s)
     {
-        size_t processed_chars = 0;
+        std::size_t processed_chars = 0;
         const int res = std::stoi(s, &processed_chars);
 
         // check if the string was completely read
@@ -7051,7 +11815,7 @@ class json_pointer
     }
 
     /// return whether pointer points to the root document
-    bool is_root() const
+    bool is_root() const noexcept
     {
         return reference_tokens.empty();
     }
@@ -7076,8 +11840,66 @@ class json_pointer
     @throw parse_error.109 if array index is not a number
     @throw type_error.313 if value cannot be unflattened
     */
-    NLOHMANN_BASIC_JSON_TPL_DECLARATION
-    NLOHMANN_BASIC_JSON_TPL& get_and_create(NLOHMANN_BASIC_JSON_TPL& j) const;
+    BasicJsonType& get_and_create(BasicJsonType& j) const
+    {
+        using size_type = typename BasicJsonType::size_type;
+        auto result = &j;
+
+        // in case no reference tokens exist, return a reference to the JSON value
+        // j which will be overwritten by a primitive value
+        for (const auto& reference_token : reference_tokens)
+        {
+            switch (result->m_type)
+            {
+                case detail::value_t::null:
+                {
+                    if (reference_token == "0")
+                    {
+                        // start a new array if reference token is 0
+                        result = &result->operator[](0);
+                    }
+                    else
+                    {
+                        // start a new object otherwise
+                        result = &result->operator[](reference_token);
+                    }
+                    break;
+                }
+
+                case detail::value_t::object:
+                {
+                    // create an entry in the object
+                    result = &result->operator[](reference_token);
+                    break;
+                }
+
+                case detail::value_t::array:
+                {
+                    // create an entry in the array
+                    JSON_TRY
+                    {
+                        result = &result->operator[](static_cast<size_type>(array_index(reference_token)));
+                    }
+                    JSON_CATCH(std::invalid_argument&)
+                    {
+                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                    }
+                    break;
+                }
+
+                /*
+                The following code is only reached if there exists a reference
+                token _and_ the current value is primitive. In this case, we have
+                an error situation, because primitive values may only occur as
+                single value; that is, with an empty list of reference tokens.
+                */
+                default:
+                    JSON_THROW(detail::type_error::create(313, "invalid value to unflatten"));
+            }
+        }
+
+        return *result;
+    }
 
     /*!
     @brief return a reference to the pointed to value
@@ -7098,8 +11920,75 @@ class json_pointer
     @throw parse_error.109   if an array index was not a number
     @throw out_of_range.404  if the JSON pointer can not be resolved
     */
-    NLOHMANN_BASIC_JSON_TPL_DECLARATION
-    NLOHMANN_BASIC_JSON_TPL& get_unchecked(NLOHMANN_BASIC_JSON_TPL* ptr) const;
+    BasicJsonType& get_unchecked(BasicJsonType* ptr) const
+    {
+        using size_type = typename BasicJsonType::size_type;
+        for (const auto& reference_token : reference_tokens)
+        {
+            // convert null values to arrays or objects before continuing
+            if (ptr->m_type == detail::value_t::null)
+            {
+                // check if reference token is a number
+                const bool nums =
+                    std::all_of(reference_token.begin(), reference_token.end(),
+                                [](const char x)
+                {
+                    return (x >= '0' and x <= '9');
+                });
+
+                // change value to array for numbers or "-" or to object otherwise
+                *ptr = (nums or reference_token == "-")
+                       ? detail::value_t::array
+                       : detail::value_t::object;
+            }
+
+            switch (ptr->m_type)
+            {
+                case detail::value_t::object:
+                {
+                    // use unchecked object access
+                    ptr = &ptr->operator[](reference_token);
+                    break;
+                }
+
+                case detail::value_t::array:
+                {
+                    // error condition (cf. RFC 6901, Sect. 4)
+                    if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                    {
+                        JSON_THROW(detail::parse_error::create(106, 0,
+                                                               "array index '" + reference_token +
+                                                               "' must not begin with '0'"));
+                    }
+
+                    if (reference_token == "-")
+                    {
+                        // explicitly treat "-" as index beyond the end
+                        ptr = &ptr->operator[](ptr->m_value.array->size());
+                    }
+                    else
+                    {
+                        // convert array index to number; unchecked access
+                        JSON_TRY
+                        {
+                            ptr = &ptr->operator[](
+                                static_cast<size_type>(array_index(reference_token)));
+                        }
+                        JSON_CATCH(std::invalid_argument&)
+                        {
+                            JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                        }
+                    }
+                    break;
+                }
+
+                default:
+                    JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+            }
+        }
+
+        return *ptr;
+    }
 
     /*!
     @throw parse_error.106   if an array index begins with '0'
@@ -7107,8 +11996,57 @@ class json_pointer
     @throw out_of_range.402  if the array index '-' is used
     @throw out_of_range.404  if the JSON pointer can not be resolved
     */
-    NLOHMANN_BASIC_JSON_TPL_DECLARATION
-    NLOHMANN_BASIC_JSON_TPL& get_checked(NLOHMANN_BASIC_JSON_TPL* ptr) const;
+    BasicJsonType& get_checked(BasicJsonType* ptr) const
+    {
+        using size_type = typename BasicJsonType::size_type;
+        for (const auto& reference_token : reference_tokens)
+        {
+            switch (ptr->m_type)
+            {
+                case detail::value_t::object:
+                {
+                    // note: at performs range check
+                    ptr = &ptr->at(reference_token);
+                    break;
+                }
+
+                case detail::value_t::array:
+                {
+                    if (JSON_UNLIKELY(reference_token == "-"))
+                    {
+                        // "-" always fails the range check
+                        JSON_THROW(detail::out_of_range::create(402,
+                                                                "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
+                                                                ") is out of range"));
+                    }
+
+                    // error condition (cf. RFC 6901, Sect. 4)
+                    if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                    {
+                        JSON_THROW(detail::parse_error::create(106, 0,
+                                                               "array index '" + reference_token +
+                                                               "' must not begin with '0'"));
+                    }
+
+                    // note: at performs range check
+                    JSON_TRY
+                    {
+                        ptr = &ptr->at(static_cast<size_type>(array_index(reference_token)));
+                    }
+                    JSON_CATCH(std::invalid_argument&)
+                    {
+                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                    }
+                    break;
+                }
+
+                default:
+                    JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+            }
+        }
+
+        return *ptr;
+    }
 
     /*!
     @brief return a const reference to the pointed to value
@@ -7123,8 +12061,58 @@ class json_pointer
     @throw out_of_range.402  if the array index '-' is used
     @throw out_of_range.404  if the JSON pointer can not be resolved
     */
-    NLOHMANN_BASIC_JSON_TPL_DECLARATION
-    const NLOHMANN_BASIC_JSON_TPL& get_unchecked(const NLOHMANN_BASIC_JSON_TPL* ptr) const;
+    const BasicJsonType& get_unchecked(const BasicJsonType* ptr) const
+    {
+        using size_type = typename BasicJsonType::size_type;
+        for (const auto& reference_token : reference_tokens)
+        {
+            switch (ptr->m_type)
+            {
+                case detail::value_t::object:
+                {
+                    // use unchecked object access
+                    ptr = &ptr->operator[](reference_token);
+                    break;
+                }
+
+                case detail::value_t::array:
+                {
+                    if (JSON_UNLIKELY(reference_token == "-"))
+                    {
+                        // "-" cannot be used for const access
+                        JSON_THROW(detail::out_of_range::create(402,
+                                                                "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
+                                                                ") is out of range"));
+                    }
+
+                    // error condition (cf. RFC 6901, Sect. 4)
+                    if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                    {
+                        JSON_THROW(detail::parse_error::create(106, 0,
+                                                               "array index '" + reference_token +
+                                                               "' must not begin with '0'"));
+                    }
+
+                    // use unchecked array access
+                    JSON_TRY
+                    {
+                        ptr = &ptr->operator[](
+                            static_cast<size_type>(array_index(reference_token)));
+                    }
+                    JSON_CATCH(std::invalid_argument&)
+                    {
+                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                    }
+                    break;
+                }
+
+                default:
+                    JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+            }
+        }
+
+        return *ptr;
+    }
 
     /*!
     @throw parse_error.106   if an array index begins with '0'
@@ -7132,8 +12120,57 @@ class json_pointer
     @throw out_of_range.402  if the array index '-' is used
     @throw out_of_range.404  if the JSON pointer can not be resolved
     */
-    NLOHMANN_BASIC_JSON_TPL_DECLARATION
-    const NLOHMANN_BASIC_JSON_TPL& get_checked(const NLOHMANN_BASIC_JSON_TPL* ptr) const;
+    const BasicJsonType& get_checked(const BasicJsonType* ptr) const
+    {
+        using size_type = typename BasicJsonType::size_type;
+        for (const auto& reference_token : reference_tokens)
+        {
+            switch (ptr->m_type)
+            {
+                case detail::value_t::object:
+                {
+                    // note: at performs range check
+                    ptr = &ptr->at(reference_token);
+                    break;
+                }
+
+                case detail::value_t::array:
+                {
+                    if (JSON_UNLIKELY(reference_token == "-"))
+                    {
+                        // "-" always fails the range check
+                        JSON_THROW(detail::out_of_range::create(402,
+                                                                "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
+                                                                ") is out of range"));
+                    }
+
+                    // error condition (cf. RFC 6901, Sect. 4)
+                    if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                    {
+                        JSON_THROW(detail::parse_error::create(106, 0,
+                                                               "array index '" + reference_token +
+                                                               "' must not begin with '0'"));
+                    }
+
+                    // note: at performs range check
+                    JSON_TRY
+                    {
+                        ptr = &ptr->at(static_cast<size_type>(array_index(reference_token)));
+                    }
+                    JSON_CATCH(std::invalid_argument&)
+                    {
+                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                    }
+                    break;
+                }
+
+                default:
+                    JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+            }
+        }
+
+        return *ptr;
+    }
 
     /*!
     @brief split the string input to reference tokens
@@ -7170,11 +12207,11 @@ class json_pointer
             std::size_t slash = reference_string.find_first_of('/', 1),
             // set the beginning of the first reference token
             start = 1;
-            // we can stop if start == string::npos+1 = 0
+            // we can stop if start == 0 (if slash == std::string::npos)
             start != 0;
             // set the beginning of the next reference token
             // (will eventually be 0 if slash == std::string::npos)
-            start = slash + 1,
+            start = (slash == std::string::npos) ? 0 : slash + 1,
             // find next slash
             slash = reference_string.find_first_of('/', start))
         {
@@ -7230,7 +12267,7 @@ class json_pointer
         {}
     }
 
-    /// escape "~"" to "~0" and "/" to "~1"
+    /// escape "~" to "~0" and "/" to "~1"
     static std::string escape(std::string s)
     {
         replace_substring(s, "~", "~0");
@@ -7252,10 +12289,57 @@ class json_pointer
 
     @note Empty objects or arrays are flattened to `null`.
     */
-    NLOHMANN_BASIC_JSON_TPL_DECLARATION
     static void flatten(const std::string& reference_string,
-                        const NLOHMANN_BASIC_JSON_TPL& value,
-                        NLOHMANN_BASIC_JSON_TPL& result);
+                        const BasicJsonType& value,
+                        BasicJsonType& result)
+    {
+        switch (value.m_type)
+        {
+            case detail::value_t::array:
+            {
+                if (value.m_value.array->empty())
+                {
+                    // flatten empty array as null
+                    result[reference_string] = nullptr;
+                }
+                else
+                {
+                    // iterate array and use index as reference string
+                    for (std::size_t i = 0; i < value.m_value.array->size(); ++i)
+                    {
+                        flatten(reference_string + "/" + std::to_string(i),
+                                value.m_value.array->operator[](i), result);
+                    }
+                }
+                break;
+            }
+
+            case detail::value_t::object:
+            {
+                if (value.m_value.object->empty())
+                {
+                    // flatten empty object as null
+                    result[reference_string] = nullptr;
+                }
+                else
+                {
+                    // iterate object and use keys as reference string
+                    for (const auto& element : *value.m_value.object)
+                    {
+                        flatten(reference_string + "/" + escape(element.first), element.second, result);
+                    }
+                }
+                break;
+            }
+
+            default:
+            {
+                // add primitive value with its reference string
+                result[reference_string] = value;
+                break;
+            }
+        }
+    }
 
     /*!
     @param[in] value  flattened JSON
@@ -7267,19 +12351,112 @@ class json_pointer
     @throw type_error.315  if object values are not primitive
     @throw type_error.313  if value cannot be unflattened
     */
-    NLOHMANN_BASIC_JSON_TPL_DECLARATION
-    static NLOHMANN_BASIC_JSON_TPL
-    unflatten(const NLOHMANN_BASIC_JSON_TPL& value);
+    static BasicJsonType
+    unflatten(const BasicJsonType& value)
+    {
+        if (JSON_UNLIKELY(not value.is_object()))
+        {
+            JSON_THROW(detail::type_error::create(314, "only objects can be unflattened"));
+        }
+
+        BasicJsonType result;
+
+        // iterate the JSON object values
+        for (const auto& element : *value.m_value.object)
+        {
+            if (JSON_UNLIKELY(not element.second.is_primitive()))
+            {
+                JSON_THROW(detail::type_error::create(315, "values in object must be primitive"));
+            }
+
+            // assign value to reference pointed to by JSON pointer; Note that if
+            // the JSON pointer is "" (i.e., points to the whole value), function
+            // get_and_create returns a reference to result itself. An assignment
+            // will then create a primitive value.
+            json_pointer(element.first).get_and_create(result) = element.second;
+        }
+
+        return result;
+    }
 
     friend bool operator==(json_pointer const& lhs,
-                           json_pointer const& rhs) noexcept;
+                           json_pointer const& rhs) noexcept
+    {
+        return (lhs.reference_tokens == rhs.reference_tokens);
+    }
 
     friend bool operator!=(json_pointer const& lhs,
-                           json_pointer const& rhs) noexcept;
+                           json_pointer const& rhs) noexcept
+    {
+        return not (lhs == rhs);
+    }
 
     /// the reference tokens
     std::vector<std::string> reference_tokens;
 };
+}  // namespace nlohmann
+
+// #include <nlohmann/adl_serializer.hpp>
+
+
+#include <utility>
+
+// #include <nlohmann/detail/conversions/from_json.hpp>
+
+// #include <nlohmann/detail/conversions/to_json.hpp>
+
+
+namespace nlohmann
+{
+
+template<typename, typename>
+struct adl_serializer
+{
+    /*!
+    @brief convert a JSON value to any value type
+
+    This function is usually called by the `get()` function of the
+    @ref basic_json class (either explicit or via conversion operators).
+
+    @param[in] j        JSON value to read from
+    @param[in,out] val  value to write to
+    */
+    template<typename BasicJsonType, typename ValueType>
+    static auto from_json(BasicJsonType&& j, ValueType& val) noexcept(
+        noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), val)))
+    -> decltype(::nlohmann::from_json(std::forward<BasicJsonType>(j), val), void())
+    {
+        ::nlohmann::from_json(std::forward<BasicJsonType>(j), val);
+    }
+
+    /*!
+    @brief convert any value type to a JSON value
+
+    This function is usually called by the constructors of the @ref basic_json
+    class.
+
+    @param[in,out] j  JSON value to write to
+    @param[in] val    value to read from
+    */
+    template <typename BasicJsonType, typename ValueType>
+    static auto to_json(BasicJsonType& j, ValueType&& val) noexcept(
+        noexcept(::nlohmann::to_json(j, std::forward<ValueType>(val))))
+    -> decltype(::nlohmann::to_json(j, std::forward<ValueType>(val)), void())
+    {
+        ::nlohmann::to_json(j, std::forward<ValueType>(val));
+    }
+};
+
+}  // namespace nlohmann
+
+
+/*!
+@brief namespace for Niels Lohmann
+@see https://github.com/nlohmann
+@since version 1.0.0
+*/
+namespace nlohmann
+{
 
 /*!
 @brief a class to store JSON values
@@ -7305,42 +12482,42 @@ and `from_json()` (@ref adl_serializer by default)
 
 @requirement The class satisfies the following concept requirements:
 - Basic
- - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible):
+ - [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible):
    JSON values can be default constructed. The result will be a JSON null
    value.
- - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible):
+ - [MoveConstructible](https://en.cppreference.com/w/cpp/named_req/MoveConstructible):
    A JSON value can be constructed from an rvalue argument.
- - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible):
+ - [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible):
    A JSON value can be copy-constructed from an lvalue expression.
- - [MoveAssignable](http://en.cppreference.com/w/cpp/concept/MoveAssignable):
+ - [MoveAssignable](https://en.cppreference.com/w/cpp/named_req/MoveAssignable):
    A JSON value van be assigned from an rvalue argument.
- - [CopyAssignable](http://en.cppreference.com/w/cpp/concept/CopyAssignable):
+ - [CopyAssignable](https://en.cppreference.com/w/cpp/named_req/CopyAssignable):
    A JSON value can be copy-assigned from an lvalue expression.
- - [Destructible](http://en.cppreference.com/w/cpp/concept/Destructible):
+ - [Destructible](https://en.cppreference.com/w/cpp/named_req/Destructible):
    JSON values can be destructed.
 - Layout
- - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType):
+ - [StandardLayoutType](https://en.cppreference.com/w/cpp/named_req/StandardLayoutType):
    JSON values have
-   [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout):
+   [standard layout](https://en.cppreference.com/w/cpp/language/data_members#Standard_layout):
    All non-static data members are private and standard layout types, the
    class has no virtual functions or (virtual) base classes.
 - Library-wide
- - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable):
+ - [EqualityComparable](https://en.cppreference.com/w/cpp/named_req/EqualityComparable):
    JSON values can be compared with `==`, see @ref
    operator==(const_reference,const_reference).
- - [LessThanComparable](http://en.cppreference.com/w/cpp/concept/LessThanComparable):
+ - [LessThanComparable](https://en.cppreference.com/w/cpp/named_req/LessThanComparable):
    JSON values can be compared with `<`, see @ref
    operator<(const_reference,const_reference).
- - [Swappable](http://en.cppreference.com/w/cpp/concept/Swappable):
+ - [Swappable](https://en.cppreference.com/w/cpp/named_req/Swappable):
    Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of
    other compatible types, using unqualified function call @ref swap().
- - [NullablePointer](http://en.cppreference.com/w/cpp/concept/NullablePointer):
+ - [NullablePointer](https://en.cppreference.com/w/cpp/named_req/NullablePointer):
    JSON values can be compared against `std::nullptr_t` objects which are used
    to model the `null` value.
 - Container
- - [Container](http://en.cppreference.com/w/cpp/concept/Container):
+ - [Container](https://en.cppreference.com/w/cpp/named_req/Container):
    JSON values can be used like STL containers and provide iterator access.
- - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer);
+ - [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer);
    JSON values can be used like STL containers and provide reverse iterator
    access.
 
@@ -7367,15 +12544,19 @@ class basic_json
 {
   private:
     template<detail::value_t> friend struct detail::external_constructor;
-    friend ::nlohmann::json_pointer;
+    friend ::nlohmann::json_pointer<basic_json>;
     friend ::nlohmann::detail::parser<basic_json>;
     friend ::nlohmann::detail::serializer<basic_json>;
     template<typename BasicJsonType>
     friend class ::nlohmann::detail::iter_impl;
     template<typename BasicJsonType, typename CharType>
     friend class ::nlohmann::detail::binary_writer;
-    template<typename BasicJsonType>
+    template<typename BasicJsonType, typename SAX>
     friend class ::nlohmann::detail::binary_reader;
+    template<typename BasicJsonType>
+    friend class ::nlohmann::detail::json_sax_dom_parser;
+    template<typename BasicJsonType>
+    friend class ::nlohmann::detail::json_sax_dom_callback_parser;
 
     /// workaround type for MSVC
     using basic_json_t = NLOHMANN_BASIC_JSON_TPL;
@@ -7403,13 +12584,19 @@ class basic_json
 
   public:
     using value_t = detail::value_t;
-    /// @copydoc nlohmann::json_pointer
-    using json_pointer = ::nlohmann::json_pointer;
+    /// JSON Pointer, see @ref nlohmann::json_pointer
+    using json_pointer = ::nlohmann::json_pointer<basic_json>;
     template<typename T, typename SFINAE>
     using json_serializer = JSONSerializer<T, SFINAE>;
+    /// how to treat decoding errors
+    using error_handler_t = detail::error_handler_t;
     /// helper type for initializer lists of basic_json values
     using initializer_list_t = std::initializer_list<detail::json_ref<basic_json>>;
 
+    using input_format_t = detail::input_format_t;
+    /// SAX interface type, see @ref nlohmann::json_sax
+    using json_sax_t = json_sax<basic_json>;
+
     ////////////////
     // exceptions //
     ////////////////
@@ -7517,10 +12704,13 @@ class basic_json
         result["copyright"] = "(C) 2013-2017 Niels Lohmann";
         result["name"] = "JSON for Modern C++";
         result["url"] = "https://github.com/nlohmann/json";
-        result["version"] =
-        {
-            {"string", "3.0.1"}, {"major", 3}, {"minor", 0}, {"patch", 1}
-        };
+        result["version"]["string"] =
+            std::to_string(NLOHMANN_JSON_VERSION_MAJOR) + "." +
+            std::to_string(NLOHMANN_JSON_VERSION_MINOR) + "." +
+            std::to_string(NLOHMANN_JSON_VERSION_PATCH);
+        result["version"]["major"] = NLOHMANN_JSON_VERSION_MAJOR;
+        result["version"]["minor"] = NLOHMANN_JSON_VERSION_MINOR;
+        result["version"]["patch"] = NLOHMANN_JSON_VERSION_PATCH;
 
 #ifdef _WIN32
         result["platform"] = "win32";
@@ -7622,10 +12812,10 @@ class basic_json
     - When all names are unique, objects will be interoperable in the sense
       that all software implementations receiving that object will agree on
       the name-value mappings.
-    - When the names within an object are not unique, later stored name/value
-      pairs overwrite previously stored name/value pairs, leaving the used
-      names unique. For instance, `{"key": 1}` and `{"key": 2, "key": 1}` will
-      be treated as equal and both stored as `{"key": 1}`.
+    - When the names within an object are not unique, it is unspecified which
+      one of the values for a given key will be chosen. For instance,
+      `{"key": 2, "key": 1}` could be equal to either `{"key": 1}` or
+      `{"key": 2}`.
     - Internally, name/value pairs are stored in lexicographical order of the
       names. Objects will also be serialized (see @ref dump) in this order.
       For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored
@@ -8139,7 +13329,7 @@ class basic_json
                     object = nullptr;  // silence warning, see #821
                     if (JSON_UNLIKELY(t == value_t::null))
                     {
-                        JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.0.1")); // LCOV_EXCL_LINE
+                        JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.5.0")); // LCOV_EXCL_LINE
                     }
                     break;
                 }
@@ -8182,7 +13372,7 @@ class basic_json
             array = create<array_t>(std::move(value));
         }
 
-        void destroy(value_t t)
+        void destroy(value_t t) noexcept
         {
             switch (t)
             {
@@ -8227,7 +13417,7 @@ class basic_json
     value is changed, because the invariant expresses a relationship between
     @a m_type and @a m_value.
     */
-    void assert_invariant() const
+    void assert_invariant() const noexcept
     {
         assert(m_type != value_t::object or m_value.object != nullptr);
         assert(m_type != value_t::array or m_value.array != nullptr);
@@ -8307,7 +13497,6 @@ class basic_json
     */
     using parser_callback_t = typename parser::parser_callback_t;
 
-
     //////////////////
     // constructors //
     //////////////////
@@ -8409,6 +13598,7 @@ class basic_json
     - @a CompatibleType is not derived from `std::istream`,
     - @a CompatibleType is not @ref basic_json (to avoid hijacking copy/move
          constructors),
+    - @a CompatibleType is not a different @ref basic_json type (i.e. with different template arguments)
     - @a CompatibleType is not a @ref basic_json nested type (e.g.,
          @ref json_pointer, @ref iterator, etc ...)
     - @ref @ref json_serializer<U> has a
@@ -8432,21 +13622,91 @@ class basic_json
 
     @since version 2.1.0
     */
-    template<typename CompatibleType, typename U = detail::uncvref_t<CompatibleType>,
-             detail::enable_if_t<not std::is_base_of<std::istream, U>::value and
-                                 not std::is_same<U, basic_json_t>::value and
-                                 not detail::is_basic_json_nested_type<
-                                     basic_json_t, U>::value and
-                                 detail::has_to_json<basic_json, U>::value,
-                                 int> = 0>
-    basic_json(CompatibleType && val) noexcept(noexcept(JSONSerializer<U>::to_json(
-                std::declval<basic_json_t&>(), std::forward<CompatibleType>(val))))
+    template <typename CompatibleType,
+              typename U = detail::uncvref_t<CompatibleType>,
+              detail::enable_if_t<
+                  not detail::is_basic_json<U>::value and detail::is_compatible_type<basic_json_t, U>::value, int> = 0>
+    basic_json(CompatibleType && val) noexcept(noexcept(
+                JSONSerializer<U>::to_json(std::declval<basic_json_t&>(),
+                                           std::forward<CompatibleType>(val))))
     {
         JSONSerializer<U>::to_json(*this, std::forward<CompatibleType>(val));
         assert_invariant();
     }
 
     /*!
+    @brief create a JSON value from an existing one
+
+    This is a constructor for existing @ref basic_json types.
+    It does not hijack copy/move constructors, since the parameter has different
+    template arguments than the current ones.
+
+    The constructor tries to convert the internal @ref m_value of the parameter.
+
+    @tparam BasicJsonType a type such that:
+    - @a BasicJsonType is a @ref basic_json type.
+    - @a BasicJsonType has different template arguments than @ref basic_json_t.
+
+    @param[in] val the @ref basic_json value to be converted.
+
+    @complexity Usually linear in the size of the passed @a val, also
+                depending on the implementation of the called `to_json()`
+                method.
+
+    @exceptionsafety Depends on the called constructor. For types directly
+    supported by the library (i.e., all types for which no `to_json()` function
+    was provided), strong guarantee holds: if an exception is thrown, there are
+    no changes to any JSON value.
+
+    @since version 3.2.0
+    */
+    template <typename BasicJsonType,
+              detail::enable_if_t<
+                  detail::is_basic_json<BasicJsonType>::value and not std::is_same<basic_json, BasicJsonType>::value, int> = 0>
+    basic_json(const BasicJsonType& val)
+    {
+        using other_boolean_t = typename BasicJsonType::boolean_t;
+        using other_number_float_t = typename BasicJsonType::number_float_t;
+        using other_number_integer_t = typename BasicJsonType::number_integer_t;
+        using other_number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+        using other_string_t = typename BasicJsonType::string_t;
+        using other_object_t = typename BasicJsonType::object_t;
+        using other_array_t = typename BasicJsonType::array_t;
+
+        switch (val.type())
+        {
+            case value_t::boolean:
+                JSONSerializer<other_boolean_t>::to_json(*this, val.template get<other_boolean_t>());
+                break;
+            case value_t::number_float:
+                JSONSerializer<other_number_float_t>::to_json(*this, val.template get<other_number_float_t>());
+                break;
+            case value_t::number_integer:
+                JSONSerializer<other_number_integer_t>::to_json(*this, val.template get<other_number_integer_t>());
+                break;
+            case value_t::number_unsigned:
+                JSONSerializer<other_number_unsigned_t>::to_json(*this, val.template get<other_number_unsigned_t>());
+                break;
+            case value_t::string:
+                JSONSerializer<other_string_t>::to_json(*this, val.template get_ref<const other_string_t&>());
+                break;
+            case value_t::object:
+                JSONSerializer<other_object_t>::to_json(*this, val.template get_ref<const other_object_t&>());
+                break;
+            case value_t::array:
+                JSONSerializer<other_array_t>::to_json(*this, val.template get_ref<const other_array_t&>());
+                break;
+            case value_t::null:
+                *this = nullptr;
+                break;
+            case value_t::discarded:
+                m_type = value_t::discarded;
+                break;
+        }
+        assert_invariant();
+    }
+
+    /*!
     @brief create a container (array or object) from an initializer list
 
     Creates a JSON value of type array or object from the passed initializer
@@ -8718,7 +13978,7 @@ class basic_json
     @warning A precondition is enforced with a runtime assertion that will
              result in calling `std::abort` if this precondition is not met.
              Assertions can be disabled by defining `NDEBUG` at compile time.
-             See http://en.cppreference.com/w/cpp/error/assert for more
+             See https://en.cppreference.com/w/cpp/error/assert for more
              information.
 
     @throw invalid_iterator.201 if iterators @a first and @a last are not
@@ -8858,7 +14118,7 @@ class basic_json
     changes to any JSON value.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is linear.
     - As postcondition, it holds: `other == basic_json(other)`.
@@ -8943,7 +14203,7 @@ class basic_json
     exceptions.
 
     @requirement This function helps `basic_json` satisfying the
-    [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible)
+    [MoveConstructible](https://en.cppreference.com/w/cpp/named_req/MoveConstructible)
     requirements.
 
     @liveexample{The code below shows the move constructor explicitly called
@@ -8977,7 +14237,7 @@ class basic_json
     @complexity Linear.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is linear.
 
@@ -8988,7 +14248,7 @@ class basic_json
 
     @since version 1.0.0
     */
-    reference& operator=(basic_json other) noexcept (
+    basic_json& operator=(basic_json other) noexcept (
         std::is_nothrow_move_constructible<value_t>::value and
         std::is_nothrow_move_assignable<value_t>::value and
         std::is_nothrow_move_constructible<json_value>::value and
@@ -9014,14 +14274,14 @@ class basic_json
     @complexity Linear.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is linear.
     - All stored elements are destroyed and all memory is freed.
 
     @since version 1.0.0
     */
-    ~basic_json()
+    ~basic_json() noexcept
     {
         assert_invariant();
         m_value.destroy(m_type);
@@ -9054,6 +14314,10 @@ class basic_json
     @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters
     in the output are escaped with `\uXXXX` sequences, and the result consists
     of ASCII characters only.
+    @param[in] error_handler  how to react on decoding errors; there are three
+    possible values: `strict` (throws and exception in case a decoding error
+    occurs; default), `replace` (replace invalid UTF-8 sequences with U+FFFD),
+    and `ignore` (ignore invalid UTF-8 sequences during serialization).
 
     @return string containing the serialization of the JSON value
 
@@ -9072,13 +14336,16 @@ class basic_json
     @see https://docs.python.org/2/library/json.html#json.dump
 
     @since version 1.0.0; indentation character @a indent_char, option
-           @a ensure_ascii and exceptions added in version 3.0.0
+           @a ensure_ascii and exceptions added in version 3.0.0; error
+           handlers added in version 3.4.0.
     */
-    string_t dump(const int indent = -1, const char indent_char = ' ',
-                  const bool ensure_ascii = false) const
+    string_t dump(const int indent = -1,
+                  const char indent_char = ' ',
+                  const bool ensure_ascii = false,
+                  const error_handler_t error_handler = error_handler_t::strict) const
     {
         string_t result;
-        serializer s(detail::output_adapter<char>(result), indent_char);
+        serializer s(detail::output_adapter<char, string_t>(result), indent_char, error_handler);
 
         if (indent >= 0)
         {
@@ -9619,11 +14886,34 @@ class basic_json
     }
 
     /*!
+    @brief get special-case overload
+
+    This overloads converts the current @ref basic_json in a different
+    @ref basic_json type
+
+    @tparam BasicJsonType == @ref basic_json
+
+    @return a copy of *this, converted into @tparam BasicJsonType
+
+    @complexity Depending on the implementation of the called `from_json()`
+                method.
+
+    @since version 3.2.0
+    */
+    template<typename BasicJsonType, detail::enable_if_t<
+                 not std::is_same<BasicJsonType, basic_json>::value and
+                 detail::is_basic_json<BasicJsonType>::value, int> = 0>
+    BasicJsonType get() const
+    {
+        return *this;
+    }
+
+    /*!
     @brief get a value (explicit)
 
     Explicit type conversion between the JSON value and a compatible value
-    which is [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible)
-    and [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible).
+    which is [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible)
+    and [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible).
     The value is converted by calling the @ref json_serializer<ValueType>
     `from_json()` method.
 
@@ -9659,7 +14949,7 @@ class basic_json
     */
     template<typename ValueTypeCV, typename ValueType = detail::uncvref_t<ValueTypeCV>,
              detail::enable_if_t <
-                 not std::is_same<basic_json_t, ValueType>::value and
+                 not detail::is_basic_json<ValueType>::value and
                  detail::has_from_json<basic_json_t, ValueType>::value and
                  not detail::has_non_default_from_json<basic_json_t, ValueType>::value,
                  int> = 0>
@@ -9683,8 +14973,8 @@ class basic_json
     @brief get a value (explicit); special case
 
     Explicit type conversion between the JSON value and a compatible value
-    which is **not** [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible)
-    and **not** [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible).
+    which is **not** [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible)
+    and **not** [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible).
     The value is converted by calling the @ref json_serializer<ValueType>
     `from_json()` method.
 
@@ -9723,17 +15013,64 @@ class basic_json
     }
 
     /*!
-    @brief get a pointer value (explicit)
+    @brief get a value (explicit)
 
-    Explicit pointer access to the internally stored JSON value. No copies are
+    Explicit type conversion between the JSON value and a compatible value.
+    The value is filled into the input parameter by calling the @ref json_serializer<ValueType>
+    `from_json()` method.
+
+    The function is equivalent to executing
+    @code {.cpp}
+    ValueType v;
+    JSONSerializer<ValueType>::from_json(*this, v);
+    @endcode
+
+    This overloads is chosen if:
+    - @a ValueType is not @ref basic_json,
+    - @ref json_serializer<ValueType> has a `from_json()` method of the form
+      `void from_json(const basic_json&, ValueType&)`, and
+
+    @tparam ValueType the input parameter type.
+
+    @return the input parameter, allowing chaining calls.
+
+    @throw what @ref json_serializer<ValueType> `from_json()` method throws
+
+    @liveexample{The example below shows several conversions from JSON values
+    to other types. There a few things to note: (1) Floating-point numbers can
+    be converted to integers\, (2) A JSON array can be converted to a standard
+    `std::vector<short>`\, (3) A JSON object can be converted to C++
+    associative containers such as `std::unordered_map<std::string\,
+    json>`.,get_to}
+
+    @since version 3.3.0
+    */
+    template<typename ValueType,
+             detail::enable_if_t <
+                 not detail::is_basic_json<ValueType>::value and
+                 detail::has_from_json<basic_json_t, ValueType>::value,
+                 int> = 0>
+    ValueType & get_to(ValueType& v) const noexcept(noexcept(
+                JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>(), v)))
+    {
+        JSONSerializer<ValueType>::from_json(*this, v);
+        return v;
+    }
+
+
+    /*!
+    @brief get a pointer value (implicit)
+
+    Implicit pointer access to the internally stored JSON value. No copies are
     made.
 
-    @warning The pointer becomes invalid if the underlying JSON object
-    changes.
+    @warning Writing data to the pointee of the result yields an undefined
+    state.
 
     @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref
     object_t, @ref string_t, @ref boolean_t, @ref number_integer_t,
-    @ref number_unsigned_t, or @ref number_float_t.
+    @ref number_unsigned_t, or @ref number_float_t. Enforced by a static
+    assertion.
 
     @return pointer to the internally stored JSON value if the requested
     pointer type @a PointerType fits to the JSON value; `nullptr` otherwise
@@ -9743,45 +15080,43 @@ class basic_json
     @liveexample{The example below shows how pointers to internal values of a
     JSON value can be requested. Note that no type conversions are made and a
     `nullptr` is returned if the value and the requested pointer type does not
-    match.,get__PointerType}
-
-    @sa @ref get_ptr() for explicit pointer-member access
+    match.,get_ptr}
 
     @since version 1.0.0
     */
     template<typename PointerType, typename std::enable_if<
                  std::is_pointer<PointerType>::value, int>::type = 0>
-    PointerType get() noexcept
+    auto get_ptr() noexcept -> decltype(std::declval<basic_json_t&>().get_impl_ptr(std::declval<PointerType>()))
     {
-        // delegate the call to get_ptr
-        return get_ptr<PointerType>();
+        // delegate the call to get_impl_ptr<>()
+        return get_impl_ptr(static_cast<PointerType>(nullptr));
     }
 
     /*!
-    @brief get a pointer value (explicit)
-    @copydoc get()
+    @brief get a pointer value (implicit)
+    @copydoc get_ptr()
     */
     template<typename PointerType, typename std::enable_if<
-                 std::is_pointer<PointerType>::value, int>::type = 0>
-    constexpr const PointerType get() const noexcept
+                 std::is_pointer<PointerType>::value and
+                 std::is_const<typename std::remove_pointer<PointerType>::type>::value, int>::type = 0>
+    constexpr auto get_ptr() const noexcept -> decltype(std::declval<const basic_json_t&>().get_impl_ptr(std::declval<PointerType>()))
     {
-        // delegate the call to get_ptr
-        return get_ptr<PointerType>();
+        // delegate the call to get_impl_ptr<>() const
+        return get_impl_ptr(static_cast<PointerType>(nullptr));
     }
 
     /*!
-    @brief get a pointer value (implicit)
+    @brief get a pointer value (explicit)
 
-    Implicit pointer access to the internally stored JSON value. No copies are
+    Explicit pointer access to the internally stored JSON value. No copies are
     made.
 
-    @warning Writing data to the pointee of the result yields an undefined
-    state.
+    @warning The pointer becomes invalid if the underlying JSON object
+    changes.
 
     @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref
     object_t, @ref string_t, @ref boolean_t, @ref number_integer_t,
-    @ref number_unsigned_t, or @ref number_float_t. Enforced by a static
-    assertion.
+    @ref number_unsigned_t, or @ref number_float_t.
 
     @return pointer to the internally stored JSON value if the requested
     pointer type @a PointerType fits to the JSON value; `nullptr` otherwise
@@ -9791,59 +15126,30 @@ class basic_json
     @liveexample{The example below shows how pointers to internal values of a
     JSON value can be requested. Note that no type conversions are made and a
     `nullptr` is returned if the value and the requested pointer type does not
-    match.,get_ptr}
+    match.,get__PointerType}
+
+    @sa @ref get_ptr() for explicit pointer-member access
 
     @since version 1.0.0
     */
     template<typename PointerType, typename std::enable_if<
                  std::is_pointer<PointerType>::value, int>::type = 0>
-    PointerType get_ptr() noexcept
+    auto get() noexcept -> decltype(std::declval<basic_json_t&>().template get_ptr<PointerType>())
     {
-        // get the type of the PointerType (remove pointer and const)
-        using pointee_t = typename std::remove_const<typename
-                          std::remove_pointer<typename
-                          std::remove_const<PointerType>::type>::type>::type;
-        // make sure the type matches the allowed types
-        static_assert(
-            std::is_same<object_t, pointee_t>::value
-            or std::is_same<array_t, pointee_t>::value
-            or std::is_same<string_t, pointee_t>::value
-            or std::is_same<boolean_t, pointee_t>::value
-            or std::is_same<number_integer_t, pointee_t>::value
-            or std::is_same<number_unsigned_t, pointee_t>::value
-            or std::is_same<number_float_t, pointee_t>::value
-            , "incompatible pointer type");
-
-        // delegate the call to get_impl_ptr<>()
-        return get_impl_ptr(static_cast<PointerType>(nullptr));
+        // delegate the call to get_ptr
+        return get_ptr<PointerType>();
     }
 
     /*!
-    @brief get a pointer value (implicit)
-    @copydoc get_ptr()
+    @brief get a pointer value (explicit)
+    @copydoc get()
     */
     template<typename PointerType, typename std::enable_if<
-                 std::is_pointer<PointerType>::value and
-                 std::is_const<typename std::remove_pointer<PointerType>::type>::value, int>::type = 0>
-    constexpr const PointerType get_ptr() const noexcept
+                 std::is_pointer<PointerType>::value, int>::type = 0>
+    constexpr auto get() const noexcept -> decltype(std::declval<const basic_json_t&>().template get_ptr<PointerType>())
     {
-        // get the type of the PointerType (remove pointer and const)
-        using pointee_t = typename std::remove_const<typename
-                          std::remove_pointer<typename
-                          std::remove_const<PointerType>::type>::type>::type;
-        // make sure the type matches the allowed types
-        static_assert(
-            std::is_same<object_t, pointee_t>::value
-            or std::is_same<array_t, pointee_t>::value
-            or std::is_same<string_t, pointee_t>::value
-            or std::is_same<boolean_t, pointee_t>::value
-            or std::is_same<number_integer_t, pointee_t>::value
-            or std::is_same<number_unsigned_t, pointee_t>::value
-            or std::is_same<number_float_t, pointee_t>::value
-            , "incompatible pointer type");
-
-        // delegate the call to get_impl_ptr<>() const
-        return get_impl_ptr(static_cast<PointerType>(nullptr));
+        // delegate the call to get_ptr
+        return get_ptr<PointerType>();
     }
 
     /*!
@@ -9925,13 +15231,16 @@ class basic_json
     template < typename ValueType, typename std::enable_if <
                    not std::is_pointer<ValueType>::value and
                    not std::is_same<ValueType, detail::json_ref<basic_json>>::value and
-                   not std::is_same<ValueType, typename string_t::value_type>::value
+                   not std::is_same<ValueType, typename string_t::value_type>::value and
+                   not detail::is_basic_json<ValueType>::value
+
 #ifndef _MSC_VER  // fix for issue #167 operator<< ambiguity under VS2015
                    and not std::is_same<ValueType, std::initializer_list<typename string_t::value_type>>::value
-#endif
-#if defined(JSON_HAS_CPP_17)
+#if defined(JSON_HAS_CPP_17) && defined(_MSC_VER) and _MSC_VER <= 1914
                    and not std::is_same<ValueType, typename std::string_view>::value
 #endif
+#endif
+                   and detail::is_detected<detail::get_template_function, const basic_json_t&, ValueType>::value
                    , int >::type = 0 >
     operator ValueType() const
     {
@@ -10195,7 +15504,7 @@ class basic_json
             return m_value.array->operator[](idx);
         }
 
-        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name())));
     }
 
     /*!
@@ -10225,7 +15534,7 @@ class basic_json
             return m_value.array->operator[](idx);
         }
 
-        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name())));
     }
 
     /*!
@@ -10271,7 +15580,7 @@ class basic_json
             return m_value.object->operator[](key);
         }
 
-        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name())));
     }
 
     /*!
@@ -10313,7 +15622,7 @@ class basic_json
             return m_value.object->find(key)->second;
         }
 
-        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name())));
     }
 
     /*!
@@ -10360,7 +15669,7 @@ class basic_json
             return m_value.object->operator[](key);
         }
 
-        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name())));
     }
 
     /*!
@@ -10403,7 +15712,7 @@ class basic_json
             return m_value.object->find(key)->second;
         }
 
-        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name())));
     }
 
     /*!
@@ -10476,7 +15785,7 @@ class basic_json
 
     /*!
     @brief overload for a default value of type const char*
-    @copydoc basic_json::value(const typename object_t::key_type&, ValueType) const
+    @copydoc basic_json::value(const typename object_t::key_type&, const ValueType&) const
     */
     string_t value(const typename object_t::key_type& key, const char* default_value) const
     {
@@ -10512,7 +15821,7 @@ class basic_json
     @return copy of the element at key @a key or @a default_value if @a key
     is not found
 
-    @throw type_error.306 if the JSON value is not an objec; in that case,
+    @throw type_error.306 if the JSON value is not an object; in that case,
     using `value()` with a key makes no sense.
 
     @complexity Logarithmic in the size of the container.
@@ -10536,7 +15845,7 @@ class basic_json
             {
                 return ptr.get_checked(this);
             }
-            JSON_CATCH (out_of_range&)
+            JSON_INTERNAL_CATCH (out_of_range&)
             {
                 return default_value;
             }
@@ -11047,7 +16356,7 @@ class basic_json
     @complexity Constant.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is constant.
 
@@ -11086,7 +16395,7 @@ class basic_json
     @complexity Constant.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is constant.
     - Has the semantics of `const_cast<const basic_json&>(*this).begin()`.
@@ -11118,7 +16427,7 @@ class basic_json
     @complexity Constant.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is constant.
 
@@ -11157,7 +16466,7 @@ class basic_json
     @complexity Constant.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is constant.
     - Has the semantics of `const_cast<const basic_json&>(*this).end()`.
@@ -11187,7 +16496,7 @@ class basic_json
     @complexity Constant.
 
     @requirement This function helps `basic_json` satisfying the
-    [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer)
+    [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer)
     requirements:
     - The complexity is constant.
     - Has the semantics of `reverse_iterator(end())`.
@@ -11224,7 +16533,7 @@ class basic_json
     @complexity Constant.
 
     @requirement This function helps `basic_json` satisfying the
-    [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer)
+    [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer)
     requirements:
     - The complexity is constant.
     - Has the semantics of `reverse_iterator(begin())`.
@@ -11261,7 +16570,7 @@ class basic_json
     @complexity Constant.
 
     @requirement This function helps `basic_json` satisfying the
-    [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer)
+    [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer)
     requirements:
     - The complexity is constant.
     - Has the semantics of `const_cast<const basic_json&>(*this).rbegin()`.
@@ -11290,7 +16599,7 @@ class basic_json
     @complexity Constant.
 
     @requirement This function helps `basic_json` satisfying the
-    [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer)
+    [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer)
     requirements:
     - The complexity is constant.
     - Has the semantics of `const_cast<const basic_json&>(*this).rend()`.
@@ -11361,18 +16670,100 @@ class basic_json
 
     @note The name of this function is not yet final and may change in the
     future.
+
+    @deprecated This stream operator is deprecated and will be removed in
+                future 4.0.0 of the library. Please use @ref items() instead;
+                that is, replace `json::iterator_wrapper(j)` with `j.items()`.
     */
-    static iteration_proxy<iterator> iterator_wrapper(reference ref)
+    JSON_DEPRECATED
+    static iteration_proxy<iterator> iterator_wrapper(reference ref) noexcept
     {
-        return iteration_proxy<iterator>(ref);
+        return ref.items();
     }
 
     /*!
     @copydoc iterator_wrapper(reference)
     */
-    static iteration_proxy<const_iterator> iterator_wrapper(const_reference ref)
+    JSON_DEPRECATED
+    static iteration_proxy<const_iterator> iterator_wrapper(const_reference ref) noexcept
     {
-        return iteration_proxy<const_iterator>(ref);
+        return ref.items();
+    }
+
+    /*!
+    @brief helper to access iterator member functions in range-based for
+
+    This function allows to access @ref iterator::key() and @ref
+    iterator::value() during range-based for loops. In these loops, a
+    reference to the JSON values is returned, so there is no access to the
+    underlying iterator.
+
+    For loop without `items()` function:
+
+    @code{cpp}
+    for (auto it = j_object.begin(); it != j_object.end(); ++it)
+    {
+        std::cout << "key: " << it.key() << ", value:" << it.value() << '\n';
+    }
+    @endcode
+
+    Range-based for loop without `items()` function:
+
+    @code{cpp}
+    for (auto it : j_object)
+    {
+        // "it" is of type json::reference and has no key() member
+        std::cout << "value: " << it << '\n';
+    }
+    @endcode
+
+    Range-based for loop with `items()` function:
+
+    @code{cpp}
+    for (auto& el : j_object.items())
+    {
+        std::cout << "key: " << el.key() << ", value:" << el.value() << '\n';
+    }
+    @endcode
+
+    The `items()` function also allows to use
+    [structured bindings](https://en.cppreference.com/w/cpp/language/structured_binding)
+    (C++17):
+
+    @code{cpp}
+    for (auto& [key, val] : j_object.items())
+    {
+        std::cout << "key: " << key << ", value:" << val << '\n';
+    }
+    @endcode
+
+    @note When iterating over an array, `key()` will return the index of the
+          element as string (see example). For primitive types (e.g., numbers),
+          `key()` returns an empty string.
+
+    @return iteration proxy object wrapping @a ref with an interface to use in
+            range-based for loops
+
+    @liveexample{The following code shows how the function is used.,items}
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
+    @complexity Constant.
+
+    @since version 3.1.0, structured bindings support since 3.5.0.
+    */
+    iteration_proxy<iterator> items() noexcept
+    {
+        return iteration_proxy<iterator>(*this);
+    }
+
+    /*!
+    @copydoc items()
+    */
+    iteration_proxy<const_iterator> items() const noexcept
+    {
+        return iteration_proxy<const_iterator>(*this);
     }
 
     /// @}
@@ -11417,7 +16808,7 @@ class basic_json
     false in the case of a string.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is constant.
     - Has the semantics of `begin() == end()`.
@@ -11488,7 +16879,7 @@ class basic_json
     the case of a string.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is constant.
     - Has the semantics of `std::distance(begin(), end())`.
@@ -11558,7 +16949,7 @@ class basic_json
     @exceptionsafety No-throw guarantee: this function never throws exceptions.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is constant.
     - Has the semantics of returning `b.size()` where `b` is the largest
@@ -11970,6 +17361,26 @@ class basic_json
         return {it, res.second};
     }
 
+    /// Helper for insertion of an iterator
+    /// @note: This uses std::distance to support GCC 4.8,
+    ///        see https://github.com/nlohmann/json/pull/1257
+    template<typename... Args>
+    iterator insert_iterator(const_iterator pos, Args&& ... args)
+    {
+        iterator result(this);
+        assert(m_value.array != nullptr);
+
+        auto insert_pos = std::distance(m_value.array->begin(), pos.m_it.array_iterator);
+        m_value.array->insert(pos.m_it.array_iterator, std::forward<Args>(args)...);
+        result.m_it.array_iterator = m_value.array->begin() + insert_pos;
+
+        // This could have been written as:
+        // result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val);
+        // but the return value of insert is missing in GCC 4.8, so it is written this way instead.
+
+        return result;
+    }
+
     /*!
     @brief inserts element
 
@@ -12004,9 +17415,7 @@ class basic_json
             }
 
             // insert to array and return iterator
-            iterator result(this);
-            result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, val);
-            return result;
+            return insert_iterator(pos, val);
         }
 
         JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
@@ -12057,9 +17466,7 @@ class basic_json
             }
 
             // insert to array and return iterator
-            iterator result(this);
-            result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val);
-            return result;
+            return insert_iterator(pos, cnt, val);
         }
 
         JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
@@ -12121,12 +17528,7 @@ class basic_json
         }
 
         // insert to array and return iterator
-        iterator result(this);
-        result.m_it.array_iterator = m_value.array->insert(
-                                         pos.m_it.array_iterator,
-                                         first.m_it.array_iterator,
-                                         last.m_it.array_iterator);
-        return result;
+        return insert_iterator(pos, first.m_it.array_iterator, last.m_it.array_iterator);
     }
 
     /*!
@@ -12168,9 +17570,7 @@ class basic_json
         }
 
         // insert to array and return iterator
-        iterator result(this);
-        result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, ilist.begin(), ilist.end());
-        return result;
+        return insert_iterator(pos, ilist.begin(), ilist.end());
     }
 
     /*!
@@ -12312,7 +17712,7 @@ class basic_json
 
         // passed iterators must belong to objects
         if (JSON_UNLIKELY(not first.m_object->is_object()
-                          or not first.m_object->is_object()))
+                          or not last.m_object->is_object()))
         {
             JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects"));
         }
@@ -12943,8 +18343,8 @@ class basic_json
 
     /*!
     @brief serialize to stream
-    @deprecated This stream operator is deprecated and will be removed in a
-                future version of the library. Please use
+    @deprecated This stream operator is deprecated and will be removed in
+                future 4.0.0 of the library. Please use
                 @ref operator<<(std::ostream&, const basic_json&)
                 instead; that is, replace calls like `j >> o;` with `o << j;`.
     @since version 1.0.0; deprecated since version 3.0.0
@@ -12999,6 +18399,8 @@ class basic_json
     @param[in] cb  a parser callback function of type @ref parser_callback_t
     which is used to control the deserialization by filtering unwanted values
     (optional)
+    @param[in] allow_exceptions  whether to throw exceptions in case of a
+    parse error (optional, true by default)
 
     @return result of the deserialization
 
@@ -13027,19 +18429,7 @@ class basic_json
 
     @since version 2.0.3 (contiguous containers)
     */
-    static basic_json parse(detail::input_adapter i,
-                            const parser_callback_t cb = nullptr,
-                            const bool allow_exceptions = true)
-    {
-        basic_json result;
-        parser(i, cb, allow_exceptions).parse(true, result);
-        return result;
-    }
-
-    /*!
-    @copydoc basic_json parse(detail::input_adapter, const parser_callback_t)
-    */
-    static basic_json parse(detail::input_adapter& i,
+    static basic_json parse(detail::input_adapter&& i,
                             const parser_callback_t cb = nullptr,
                             const bool allow_exceptions = true)
     {
@@ -13048,14 +18438,80 @@ class basic_json
         return result;
     }
 
-    static bool accept(detail::input_adapter i)
+    static bool accept(detail::input_adapter&& i)
     {
         return parser(i).accept(true);
     }
 
-    static bool accept(detail::input_adapter& i)
+    /*!
+    @brief generate SAX events
+
+    The SAX event lister must follow the interface of @ref json_sax.
+
+    This function reads from a compatible input. Examples are:
+    - an array of 1-byte values
+    - strings with character/literal type with size of 1 byte
+    - input streams
+    - container with contiguous storage of 1-byte values. Compatible container
+      types include `std::vector`, `std::string`, `std::array`,
+      `std::valarray`, and `std::initializer_list`. Furthermore, C-style
+      arrays can be used with `std::begin()`/`std::end()`. User-defined
+      containers can be used as long as they implement random-access iterators
+      and a contiguous storage.
+
+    @pre Each element of the container has a size of 1 byte. Violating this
+    precondition yields undefined behavior. **This precondition is enforced
+    with a static assertion.**
+
+    @pre The container storage is contiguous. Violating this precondition
+    yields undefined behavior. **This precondition is enforced with an
+    assertion.**
+    @pre Each element of the container has a size of 1 byte. Violating this
+    precondition yields undefined behavior. **This precondition is enforced
+    with a static assertion.**
+
+    @warning There is no way to enforce all preconditions at compile-time. If
+             the function is called with a noncompliant container and with
+             assertions switched off, the behavior is undefined and will most
+             likely yield segmentation violation.
+
+    @param[in] i  input to read from
+    @param[in,out] sax  SAX event listener
+    @param[in] format  the format to parse (JSON, CBOR, MessagePack, or UBJSON)
+    @param[in] strict  whether the input has to be consumed completely
+
+    @return return value of the last processed SAX event
+
+    @throw parse_error.101 if a parse error occurs; example: `""unexpected end
+    of input; expected string literal""`
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+
+    @complexity Linear in the length of the input. The parser is a predictive
+    LL(1) parser. The complexity can be higher if the SAX consumer @a sax has
+    a super-linear complexity.
+
+    @note A UTF-8 byte order mark is silently ignored.
+
+    @liveexample{The example below demonstrates the `sax_parse()` function
+    reading from string and processing the events with a user-defined SAX
+    event consumer.,sax_parse}
+
+    @since version 3.2.0
+    */
+    template <typename SAX>
+    static bool sax_parse(detail::input_adapter&& i, SAX* sax,
+                          input_format_t format = input_format_t::json,
+                          const bool strict = true)
     {
-        return parser(i).accept(true);
+        assert(sax);
+        switch (format)
+        {
+            case input_format_t::json:
+                return parser(std::move(i)).sax_parse(sax, strict);
+            default:
+                return detail::binary_reader<basic_json, SAX>(std::move(i)).sax_parse(format, sax, strict);
+        }
     }
 
     /*!
@@ -13127,10 +18583,19 @@ class basic_json
         return parser(detail::input_adapter(first, last)).accept(true);
     }
 
+    template<class IteratorType, class SAX, typename std::enable_if<
+                 std::is_base_of<
+                     std::random_access_iterator_tag,
+                     typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0>
+    static bool sax_parse(IteratorType first, IteratorType last, SAX* sax)
+    {
+        return parser(detail::input_adapter(first, last)).sax_parse(sax);
+    }
+
     /*!
     @brief deserialize from stream
-    @deprecated This stream operator is deprecated and will be removed in a
-                future version of the library. Please use
+    @deprecated This stream operator is deprecated and will be removed in
+                version 4.0.0 of the library. Please use
                 @ref operator>>(std::istream&, basic_json&)
                 instead; that is, replace calls like `j << i;` with `i >> j;`.
     @since version 1.0.0; deprecated since version 3.0.0
@@ -13331,9 +18796,11 @@ class basic_json
     vector in CBOR format.,to_cbor}
 
     @sa http://cbor.io
-    @sa @ref from_cbor(const std::vector<uint8_t>&, const size_t) for the
+    @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the
         analogous deserialization
     @sa @ref to_msgpack(const basic_json&) for the related MessagePack format
+    @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the
+             related UBJSON format
 
     @since version 2.0.9
     */
@@ -13426,9 +18893,10 @@ class basic_json
     vector in MessagePack format.,to_msgpack}
 
     @sa http://msgpack.org
-    @sa @ref from_msgpack(const std::vector<uint8_t>&, const size_t) for the
-        analogous deserialization
+    @sa @ref from_msgpack for the analogous deserialization
     @sa @ref to_cbor(const basic_json& for the related CBOR format
+    @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the
+             related UBJSON format
 
     @since version 2.0.9
     */
@@ -13450,6 +18918,192 @@ class basic_json
     }
 
     /*!
+    @brief create a UBJSON serialization of a given JSON value
+
+    Serializes a given JSON value @a j to a byte vector using the UBJSON
+    (Universal Binary JSON) serialization format. UBJSON aims to be more compact
+    than JSON itself, yet more efficient to parse.
+
+    The library uses the following mapping from JSON values types to
+    UBJSON types according to the UBJSON specification:
+
+    JSON value type | value/range                       | UBJSON type | marker
+    --------------- | --------------------------------- | ----------- | ------
+    null            | `null`                            | null        | `Z`
+    boolean         | `true`                            | true        | `T`
+    boolean         | `false`                           | false       | `F`
+    number_integer  | -9223372036854775808..-2147483649 | int64       | `L`
+    number_integer  | -2147483648..-32769               | int32       | `l`
+    number_integer  | -32768..-129                      | int16       | `I`
+    number_integer  | -128..127                         | int8        | `i`
+    number_integer  | 128..255                          | uint8       | `U`
+    number_integer  | 256..32767                        | int16       | `I`
+    number_integer  | 32768..2147483647                 | int32       | `l`
+    number_integer  | 2147483648..9223372036854775807   | int64       | `L`
+    number_unsigned | 0..127                            | int8        | `i`
+    number_unsigned | 128..255                          | uint8       | `U`
+    number_unsigned | 256..32767                        | int16       | `I`
+    number_unsigned | 32768..2147483647                 | int32       | `l`
+    number_unsigned | 2147483648..9223372036854775807   | int64       | `L`
+    number_float    | *any value*                       | float64     | `D`
+    string          | *with shortest length indicator*  | string      | `S`
+    array           | *see notes on optimized format*   | array       | `[`
+    object          | *see notes on optimized format*   | map         | `{`
+
+    @note The mapping is **complete** in the sense that any JSON value type
+          can be converted to a UBJSON value.
+
+    @note The following values can **not** be converted to a UBJSON value:
+          - strings with more than 9223372036854775807 bytes (theoretical)
+          - unsigned integer numbers above 9223372036854775807
+
+    @note The following markers are not used in the conversion:
+          - `Z`: no-op values are not created.
+          - `C`: single-byte strings are serialized with `S` markers.
+
+    @note Any UBJSON output created @ref to_ubjson can be successfully parsed
+          by @ref from_ubjson.
+
+    @note If NaN or Infinity are stored inside a JSON number, they are
+          serialized properly. This behavior differs from the @ref dump()
+          function which serializes NaN or Infinity to `null`.
+
+    @note The optimized formats for containers are supported: Parameter
+          @a use_size adds size information to the beginning of a container and
+          removes the closing marker. Parameter @a use_type further checks
+          whether all elements of a container have the same type and adds the
+          type marker to the beginning of the container. The @a use_type
+          parameter must only be used together with @a use_size = true. Note
+          that @a use_size = true alone may result in larger representations -
+          the benefit of this parameter is that the receiving side is
+          immediately informed on the number of elements of the container.
+
+    @param[in] j  JSON value to serialize
+    @param[in] use_size  whether to add size annotations to container types
+    @param[in] use_type  whether to add type annotations to container types
+                         (must be combined with @a use_size = true)
+    @return UBJSON serialization as byte vector
+
+    @complexity Linear in the size of the JSON value @a j.
+
+    @liveexample{The example shows the serialization of a JSON value to a byte
+    vector in UBJSON format.,to_ubjson}
+
+    @sa http://ubjson.org
+    @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the
+        analogous deserialization
+    @sa @ref to_cbor(const basic_json& for the related CBOR format
+    @sa @ref to_msgpack(const basic_json&) for the related MessagePack format
+
+    @since version 3.1.0
+    */
+    static std::vector<uint8_t> to_ubjson(const basic_json& j,
+                                          const bool use_size = false,
+                                          const bool use_type = false)
+    {
+        std::vector<uint8_t> result;
+        to_ubjson(j, result, use_size, use_type);
+        return result;
+    }
+
+    static void to_ubjson(const basic_json& j, detail::output_adapter<uint8_t> o,
+                          const bool use_size = false, const bool use_type = false)
+    {
+        binary_writer<uint8_t>(o).write_ubjson(j, use_size, use_type);
+    }
+
+    static void to_ubjson(const basic_json& j, detail::output_adapter<char> o,
+                          const bool use_size = false, const bool use_type = false)
+    {
+        binary_writer<char>(o).write_ubjson(j, use_size, use_type);
+    }
+
+
+    /*!
+    @brief Serializes the given JSON object `j` to BSON and returns a vector
+           containing the corresponding BSON-representation.
+
+    BSON (Binary JSON) is a binary format in which zero or more ordered key/value pairs are
+    stored as a single entity (a so-called document).
+
+    The library uses the following mapping from JSON values types to BSON types:
+
+    JSON value type | value/range                       | BSON type   | marker
+    --------------- | --------------------------------- | ----------- | ------
+    null            | `null`                            | null        | 0x0A
+    boolean         | `true`, `false`                   | boolean     | 0x08
+    number_integer  | -9223372036854775808..-2147483649 | int64       | 0x12
+    number_integer  | -2147483648..2147483647           | int32       | 0x10
+    number_integer  | 2147483648..9223372036854775807   | int64       | 0x12
+    number_unsigned | 0..2147483647                     | int32       | 0x10
+    number_unsigned | 2147483648..9223372036854775807   | int64       | 0x12
+    number_unsigned | 9223372036854775808..18446744073709551615| --   | --
+    number_float    | *any value*                       | double      | 0x01
+    string          | *any value*                       | string      | 0x02
+    array           | *any value*                       | document    | 0x04
+    object          | *any value*                       | document    | 0x03
+
+    @warning The mapping is **incomplete**, since only JSON-objects (and things
+    contained therein) can be serialized to BSON.
+    Also, integers larger than 9223372036854775807 cannot be serialized to BSON,
+    and the keys may not contain U+0000, since they are serialized a
+    zero-terminated c-strings.
+
+    @throw out_of_range.407  if `j.is_number_unsigned() && j.get<std::uint64_t>() > 9223372036854775807`
+    @throw out_of_range.409  if a key in `j` contains a NULL (U+0000)
+    @throw type_error.317    if `!j.is_object()`
+
+    @pre The input `j` is required to be an object: `j.is_object() == true`.
+
+    @note Any BSON output created via @ref to_bson can be successfully parsed
+          by @ref from_bson.
+
+    @param[in] j  JSON value to serialize
+    @return BSON serialization as byte vector
+
+    @complexity Linear in the size of the JSON value @a j.
+
+    @liveexample{The example shows the serialization of a JSON value to a byte
+    vector in BSON format.,to_bson}
+
+    @sa http://bsonspec.org/spec.html
+    @sa @ref from_bson(detail::input_adapter&&, const bool strict) for the
+        analogous deserialization
+    @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the
+             related UBJSON format
+    @sa @ref to_cbor(const basic_json&) for the related CBOR format
+    @sa @ref to_msgpack(const basic_json&) for the related MessagePack format
+    */
+    static std::vector<uint8_t> to_bson(const basic_json& j)
+    {
+        std::vector<uint8_t> result;
+        to_bson(j, result);
+        return result;
+    }
+
+    /*!
+    @brief Serializes the given JSON object `j` to BSON and forwards the
+           corresponding BSON-representation to the given output_adapter `o`.
+    @param j The JSON object to convert to BSON.
+    @param o The output adapter that receives the binary BSON representation.
+    @pre The input `j` shall be an object: `j.is_object() == true`
+    @sa @ref to_bson(const basic_json&)
+    */
+    static void to_bson(const basic_json& j, detail::output_adapter<uint8_t> o)
+    {
+        binary_writer<uint8_t>(o).write_bson(j);
+    }
+
+    /*!
+    @copydoc to_bson(const basic_json&, detail::output_adapter<uint8_t>)
+    */
+    static void to_bson(const basic_json& j, detail::output_adapter<char> o)
+    {
+        binary_writer<char>(o).write_bson(j);
+    }
+
+
+    /*!
     @brief create a JSON value from an input in CBOR format
 
     Deserializes a given input @a i to a JSON value using the CBOR (Concise
@@ -13490,7 +19144,7 @@ class basic_json
     map                    | object          | 0xBF
     False                  | `false`         | 0xF4
     True                   | `true`          | 0xF5
-    Nill                   | `null`          | 0xF6
+    Null                   | `null`          | 0xF6
     Half-Precision Float   | number_float    | 0xF9
     Single-Precision Float | number_float    | 0xFA
     Double-Precision Float | number_float    | 0xFB
@@ -13518,6 +19172,9 @@ class basic_json
     @param[in] i  an input in CBOR format convertible to an input adapter
     @param[in] strict  whether to expect the input to be consumed until EOF
                        (true by default)
+    @param[in] allow_exceptions  whether to throw exceptions in case of a
+    parse error (optional, true by default)
+
     @return deserialized JSON value
 
     @throw parse_error.110 if the given input ends prematurely or the end of
@@ -13533,27 +19190,39 @@ class basic_json
 
     @sa http://cbor.io
     @sa @ref to_cbor(const basic_json&) for the analogous serialization
-    @sa @ref from_msgpack(detail::input_adapter, const bool) for the
+    @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for the
         related MessagePack format
+    @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the
+        related UBJSON format
 
     @since version 2.0.9; parameter @a start_index since 2.1.1; changed to
            consume input adapters, removed start_index parameter, and added
-           @a strict parameter since 3.0.0
+           @a strict parameter since 3.0.0; added @a allow_exceptions parameter
+           since 3.2.0
     */
-    static basic_json from_cbor(detail::input_adapter i,
-                                const bool strict = true)
+    static basic_json from_cbor(detail::input_adapter&& i,
+                                const bool strict = true,
+                                const bool allow_exceptions = true)
     {
-        return binary_reader(i).parse_cbor(strict);
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::cbor, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
     }
 
     /*!
-    @copydoc from_cbor(detail::input_adapter, const bool)
+    @copydoc from_cbor(detail::input_adapter&&, const bool, const bool)
     */
     template<typename A1, typename A2,
              detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
-    static basic_json from_cbor(A1 && a1, A2 && a2, const bool strict = true)
+    static basic_json from_cbor(A1 && a1, A2 && a2,
+                                const bool strict = true,
+                                const bool allow_exceptions = true)
     {
-        return binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).parse_cbor(strict);
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::cbor, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
     }
 
     /*!
@@ -13606,6 +19275,10 @@ class basic_json
                   adapter
     @param[in] strict  whether to expect the input to be consumed until EOF
                        (true by default)
+    @param[in] allow_exceptions  whether to throw exceptions in case of a
+    parse error (optional, true by default)
+
+    @return deserialized JSON value
 
     @throw parse_error.110 if the given input ends prematurely or the end of
     file was not reached when @a strict was set to true
@@ -13620,29 +19293,212 @@ class basic_json
 
     @sa http://msgpack.org
     @sa @ref to_msgpack(const basic_json&) for the analogous serialization
-    @sa @ref from_cbor(detail::input_adapter, const bool) for the related CBOR
-        format
+    @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the
+        related CBOR format
+    @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for
+        the related UBJSON format
+    @sa @ref from_bson(detail::input_adapter&&, const bool, const bool) for
+        the related BSON format
 
     @since version 2.0.9; parameter @a start_index since 2.1.1; changed to
            consume input adapters, removed start_index parameter, and added
-           @a strict parameter since 3.0.0
+           @a strict parameter since 3.0.0; added @a allow_exceptions parameter
+           since 3.2.0
+    */
+    static basic_json from_msgpack(detail::input_adapter&& i,
+                                   const bool strict = true,
+                                   const bool allow_exceptions = true)
+    {
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::msgpack, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
+    }
+
+    /*!
+    @copydoc from_msgpack(detail::input_adapter&&, const bool, const bool)
+    */
+    template<typename A1, typename A2,
+             detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
+    static basic_json from_msgpack(A1 && a1, A2 && a2,
+                                   const bool strict = true,
+                                   const bool allow_exceptions = true)
+    {
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::msgpack, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
+    }
+
+    /*!
+    @brief create a JSON value from an input in UBJSON format
+
+    Deserializes a given input @a i to a JSON value using the UBJSON (Universal
+    Binary JSON) serialization format.
+
+    The library maps UBJSON types to JSON value types as follows:
+
+    UBJSON type | JSON value type                         | marker
+    ----------- | --------------------------------------- | ------
+    no-op       | *no value, next value is read*          | `N`
+    null        | `null`                                  | `Z`
+    false       | `false`                                 | `F`
+    true        | `true`                                  | `T`
+    float32     | number_float                            | `d`
+    float64     | number_float                            | `D`
+    uint8       | number_unsigned                         | `U`
+    int8        | number_integer                          | `i`
+    int16       | number_integer                          | `I`
+    int32       | number_integer                          | `l`
+    int64       | number_integer                          | `L`
+    string      | string                                  | `S`
+    char        | string                                  | `C`
+    array       | array (optimized values are supported)  | `[`
+    object      | object (optimized values are supported) | `{`
+
+    @note The mapping is **complete** in the sense that any UBJSON value can
+          be converted to a JSON value.
+
+    @param[in] i  an input in UBJSON format convertible to an input adapter
+    @param[in] strict  whether to expect the input to be consumed until EOF
+                       (true by default)
+    @param[in] allow_exceptions  whether to throw exceptions in case of a
+    parse error (optional, true by default)
+
+    @return deserialized JSON value
+
+    @throw parse_error.110 if the given input ends prematurely or the end of
+    file was not reached when @a strict was set to true
+    @throw parse_error.112 if a parse error occurs
+    @throw parse_error.113 if a string could not be parsed successfully
+
+    @complexity Linear in the size of the input @a i.
+
+    @liveexample{The example shows the deserialization of a byte vector in
+    UBJSON format to a JSON value.,from_ubjson}
+
+    @sa http://ubjson.org
+    @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the
+             analogous serialization
+    @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the
+        related CBOR format
+    @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for
+        the related MessagePack format
+    @sa @ref from_bson(detail::input_adapter&&, const bool, const bool) for
+        the related BSON format
+
+    @since version 3.1.0; added @a allow_exceptions parameter since 3.2.0
     */
-    static basic_json from_msgpack(detail::input_adapter i,
-                                   const bool strict = true)
+    static basic_json from_ubjson(detail::input_adapter&& i,
+                                  const bool strict = true,
+                                  const bool allow_exceptions = true)
     {
-        return binary_reader(i).parse_msgpack(strict);
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::ubjson, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
     }
 
     /*!
-    @copydoc from_msgpack(detail::input_adapter, const bool)
+    @copydoc from_ubjson(detail::input_adapter&&, const bool, const bool)
     */
     template<typename A1, typename A2,
              detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
-    static basic_json from_msgpack(A1 && a1, A2 && a2, const bool strict = true)
+    static basic_json from_ubjson(A1 && a1, A2 && a2,
+                                  const bool strict = true,
+                                  const bool allow_exceptions = true)
     {
-        return binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).parse_msgpack(strict);
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::ubjson, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
     }
 
+    /*!
+    @brief Create a JSON value from an input in BSON format
+
+    Deserializes a given input @a i to a JSON value using the BSON (Binary JSON)
+    serialization format.
+
+    The library maps BSON record types to JSON value types as follows:
+
+    BSON type       | BSON marker byte | JSON value type
+    --------------- | ---------------- | ---------------------------
+    double          | 0x01             | number_float
+    string          | 0x02             | string
+    document        | 0x03             | object
+    array           | 0x04             | array
+    binary          | 0x05             | still unsupported
+    undefined       | 0x06             | still unsupported
+    ObjectId        | 0x07             | still unsupported
+    boolean         | 0x08             | boolean
+    UTC Date-Time   | 0x09             | still unsupported
+    null            | 0x0A             | null
+    Regular Expr.   | 0x0B             | still unsupported
+    DB Pointer      | 0x0C             | still unsupported
+    JavaScript Code | 0x0D             | still unsupported
+    Symbol          | 0x0E             | still unsupported
+    JavaScript Code | 0x0F             | still unsupported
+    int32           | 0x10             | number_integer
+    Timestamp       | 0x11             | still unsupported
+    128-bit decimal float | 0x13       | still unsupported
+    Max Key         | 0x7F             | still unsupported
+    Min Key         | 0xFF             | still unsupported
+
+    @warning The mapping is **incomplete**. The unsupported mappings
+             are indicated in the table above.
+
+    @param[in] i  an input in BSON format convertible to an input adapter
+    @param[in] strict  whether to expect the input to be consumed until EOF
+                       (true by default)
+    @param[in] allow_exceptions  whether to throw exceptions in case of a
+    parse error (optional, true by default)
+
+    @return deserialized JSON value
+
+    @throw parse_error.114 if an unsupported BSON record type is encountered
+
+    @complexity Linear in the size of the input @a i.
+
+    @liveexample{The example shows the deserialization of a byte vector in
+    BSON format to a JSON value.,from_bson}
+
+    @sa http://bsonspec.org/spec.html
+    @sa @ref to_bson(const basic_json&) for the analogous serialization
+    @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the
+        related CBOR format
+    @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for
+        the related MessagePack format
+    @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the
+        related UBJSON format
+    */
+    static basic_json from_bson(detail::input_adapter&& i,
+                                const bool strict = true,
+                                const bool allow_exceptions = true)
+    {
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::bson, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
+    }
+
+    /*!
+    @copydoc from_bson(detail::input_adapter&&, const bool, const bool)
+    */
+    template<typename A1, typename A2,
+             detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
+    static basic_json from_bson(A1 && a1, A2 && a2,
+                                const bool strict = true,
+                                const bool allow_exceptions = true)
+    {
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::bson, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
+    }
+
+
+
     /// @}
 
     //////////////////////////
@@ -14008,20 +19864,20 @@ class basic_json
                                 // avoid undefined behavior
                                 JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
                             }
-                            else
-                            {
-                                // default case: insert add offset
-                                parent.insert(parent.begin() + static_cast<difference_type>(idx), val);
-                            }
+
+                            // default case: insert add offset
+                            parent.insert(parent.begin() + static_cast<difference_type>(idx), val);
                         }
                         break;
                     }
 
+                    // LCOV_EXCL_START
                     default:
                     {
                         // if there exists a parent it cannot be primitive
-                        assert(false);  // LCOV_EXCL_LINE
+                        assert(false);
                     }
+                        // LCOV_EXCL_STOP
                 }
             }
         };
@@ -14066,7 +19922,7 @@ class basic_json
             // wrapper to get a value for an operation
             const auto get_value = [&val](const std::string & op,
                                           const std::string & member,
-                                          bool string_type) -> basic_json&
+                                          bool string_type) -> basic_json &
             {
                 // find value
                 auto it = val.m_value.object->find(member);
@@ -14163,7 +20019,7 @@ class basic_json
                         // the "path" location must exist - use at()
                         success = (result.at(ptr) == get_value("test", "value", false));
                     }
-                    JSON_CATCH (out_of_range&)
+                    JSON_INTERNAL_CATCH (out_of_range&)
                     {
                         // ignore out of range errors: success remains false
                     }
@@ -14216,6 +20072,7 @@ class basic_json
     diff for two JSON values.,diff}
 
     @sa @ref patch -- apply a JSON patch
+    @sa @ref merge_patch -- apply a JSON Merge Patch
 
     @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902)
 
@@ -14347,418 +20204,86 @@ class basic_json
     }
 
     /// @}
-};
 
-/////////////
-// presets //
-/////////////
+    ////////////////////////////////
+    // JSON Merge Patch functions //
+    ////////////////////////////////
 
-/*!
-@brief default JSON class
-
-This type is the default specialization of the @ref basic_json class which
-uses the standard template types.
-
-@since version 1.0.0
-*/
-using json = basic_json<>;
-
-//////////////////
-// json_pointer //
-//////////////////
-
-NLOHMANN_BASIC_JSON_TPL_DECLARATION
-NLOHMANN_BASIC_JSON_TPL&
-json_pointer::get_and_create(NLOHMANN_BASIC_JSON_TPL& j) const
-{
-    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
-    auto result = &j;
-
-    // in case no reference tokens exist, return a reference to the JSON value
-    // j which will be overwritten by a primitive value
-    for (const auto& reference_token : reference_tokens)
-    {
-        switch (result->m_type)
-        {
-            case detail::value_t::null:
-            {
-                if (reference_token == "0")
-                {
-                    // start a new array if reference token is 0
-                    result = &result->operator[](0);
-                }
-                else
-                {
-                    // start a new object otherwise
-                    result = &result->operator[](reference_token);
-                }
-                break;
-            }
-
-            case detail::value_t::object:
-            {
-                // create an entry in the object
-                result = &result->operator[](reference_token);
-                break;
-            }
-
-            case detail::value_t::array:
-            {
-                // create an entry in the array
-                JSON_TRY
-                {
-                    result = &result->operator[](static_cast<size_type>(array_index(reference_token)));
-                }
-                JSON_CATCH(std::invalid_argument&)
-                {
-                    JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                }
-                break;
-            }
+    /// @name JSON Merge Patch functions
+    /// @{
 
-            /*
-            The following code is only reached if there exists a reference
-            token _and_ the current value is primitive. In this case, we have
-            an error situation, because primitive values may only occur as
-            single value; that is, with an empty list of reference tokens.
-            */
-            default:
-                JSON_THROW(detail::type_error::create(313, "invalid value to unflatten"));
-        }
-    }
+    /*!
+    @brief applies a JSON Merge Patch
+
+    The merge patch format is primarily intended for use with the HTTP PATCH
+    method as a means of describing a set of modifications to a target
+    resource's content. This function applies a merge patch to the current
+    JSON value.
+
+    The function implements the following algorithm from Section 2 of
+    [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396):
+
+    ```
+    define MergePatch(Target, Patch):
+      if Patch is an Object:
+        if Target is not an Object:
+          Target = {} // Ignore the contents and set it to an empty Object
+        for each Name/Value pair in Patch:
+          if Value is null:
+            if Name exists in Target:
+              remove the Name/Value pair from Target
+          else:
+            Target[Name] = MergePatch(Target[Name], Value)
+        return Target
+      else:
+        return Patch
+    ```
+
+    Thereby, `Target` is the current object; that is, the patch is applied to
+    the current value.
+
+    @param[in] apply_patch  the patch to apply
+
+    @complexity Linear in the lengths of @a patch.
+
+    @liveexample{The following code shows how a JSON Merge Patch is applied to
+    a JSON document.,merge_patch}
 
-    return *result;
-}
+    @sa @ref patch -- apply a JSON patch
+    @sa [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396)
 
-NLOHMANN_BASIC_JSON_TPL_DECLARATION
-NLOHMANN_BASIC_JSON_TPL&
-json_pointer::get_unchecked(NLOHMANN_BASIC_JSON_TPL* ptr) const
-{
-    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
-    for (const auto& reference_token : reference_tokens)
+    @since version 3.0.0
+    */
+    void merge_patch(const basic_json& apply_patch)
     {
-        // convert null values to arrays or objects before continuing
-        if (ptr->m_type == detail::value_t::null)
-        {
-            // check if reference token is a number
-            const bool nums =
-                std::all_of(reference_token.begin(), reference_token.end(),
-                            [](const char x)
-            {
-                return (x >= '0' and x <= '9');
-            });
-
-            // change value to array for numbers or "-" or to object otherwise
-            *ptr = (nums or reference_token == "-")
-                   ? detail::value_t::array
-                   : detail::value_t::object;
-        }
-
-        switch (ptr->m_type)
+        if (apply_patch.is_object())
         {
-            case detail::value_t::object:
+            if (not is_object())
             {
-                // use unchecked object access
-                ptr = &ptr->operator[](reference_token);
-                break;
+                *this = object();
             }
-
-            case detail::value_t::array:
+            for (auto it = apply_patch.begin(); it != apply_patch.end(); ++it)
             {
-                // error condition (cf. RFC 6901, Sect. 4)
-                if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
-                {
-                    JSON_THROW(detail::parse_error::create(106, 0,
-                                                           "array index '" + reference_token +
-                                                           "' must not begin with '0'"));
-                }
-
-                if (reference_token == "-")
+                if (it.value().is_null())
                 {
-                    // explicitly treat "-" as index beyond the end
-                    ptr = &ptr->operator[](ptr->m_value.array->size());
+                    erase(it.key());
                 }
                 else
                 {
-                    // convert array index to number; unchecked access
-                    JSON_TRY
-                    {
-                        ptr = &ptr->operator[](
-                            static_cast<size_type>(array_index(reference_token)));
-                    }
-                    JSON_CATCH(std::invalid_argument&)
-                    {
-                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                    }
-                }
-                break;
-            }
-
-            default:
-                JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
-        }
-    }
-
-    return *ptr;
-}
-
-NLOHMANN_BASIC_JSON_TPL_DECLARATION
-NLOHMANN_BASIC_JSON_TPL&
-json_pointer::get_checked(NLOHMANN_BASIC_JSON_TPL* ptr) const
-{
-    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
-    for (const auto& reference_token : reference_tokens)
-    {
-        switch (ptr->m_type)
-        {
-            case detail::value_t::object:
-            {
-                // note: at performs range check
-                ptr = &ptr->at(reference_token);
-                break;
-            }
-
-            case detail::value_t::array:
-            {
-                if (JSON_UNLIKELY(reference_token == "-"))
-                {
-                    // "-" always fails the range check
-                    JSON_THROW(detail::out_of_range::create(402,
-                                                            "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
-                                                            ") is out of range"));
-                }
-
-                // error condition (cf. RFC 6901, Sect. 4)
-                if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
-                {
-                    JSON_THROW(detail::parse_error::create(106, 0,
-                                                           "array index '" + reference_token +
-                                                           "' must not begin with '0'"));
-                }
-
-                // note: at performs range check
-                JSON_TRY
-                {
-                    ptr = &ptr->at(static_cast<size_type>(array_index(reference_token)));
+                    operator[](it.key()).merge_patch(it.value());
                 }
-                JSON_CATCH(std::invalid_argument&)
-                {
-                    JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                }
-                break;
             }
-
-            default:
-                JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
         }
-    }
-
-    return *ptr;
-}
-
-NLOHMANN_BASIC_JSON_TPL_DECLARATION
-const NLOHMANN_BASIC_JSON_TPL&
-json_pointer::get_unchecked(const NLOHMANN_BASIC_JSON_TPL* ptr) const
-{
-    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
-    for (const auto& reference_token : reference_tokens)
-    {
-        switch (ptr->m_type)
-        {
-            case detail::value_t::object:
-            {
-                // use unchecked object access
-                ptr = &ptr->operator[](reference_token);
-                break;
-            }
-
-            case detail::value_t::array:
-            {
-                if (JSON_UNLIKELY(reference_token == "-"))
-                {
-                    // "-" cannot be used for const access
-                    JSON_THROW(detail::out_of_range::create(402,
-                                                            "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
-                                                            ") is out of range"));
-                }
-
-                // error condition (cf. RFC 6901, Sect. 4)
-                if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
-                {
-                    JSON_THROW(detail::parse_error::create(106, 0,
-                                                           "array index '" + reference_token +
-                                                           "' must not begin with '0'"));
-                }
-
-                // use unchecked array access
-                JSON_TRY
-                {
-                    ptr = &ptr->operator[](
-                        static_cast<size_type>(array_index(reference_token)));
-                }
-                JSON_CATCH(std::invalid_argument&)
-                {
-                    JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                }
-                break;
-            }
-
-            default:
-                JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
-        }
-    }
-
-    return *ptr;
-}
-
-NLOHMANN_BASIC_JSON_TPL_DECLARATION
-const NLOHMANN_BASIC_JSON_TPL&
-json_pointer::get_checked(const NLOHMANN_BASIC_JSON_TPL* ptr) const
-{
-    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
-    for (const auto& reference_token : reference_tokens)
-    {
-        switch (ptr->m_type)
-        {
-            case detail::value_t::object:
-            {
-                // note: at performs range check
-                ptr = &ptr->at(reference_token);
-                break;
-            }
-
-            case detail::value_t::array:
-            {
-                if (JSON_UNLIKELY(reference_token == "-"))
-                {
-                    // "-" always fails the range check
-                    JSON_THROW(detail::out_of_range::create(402,
-                                                            "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
-                                                            ") is out of range"));
-                }
-
-                // error condition (cf. RFC 6901, Sect. 4)
-                if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
-                {
-                    JSON_THROW(detail::parse_error::create(106, 0,
-                                                           "array index '" + reference_token +
-                                                           "' must not begin with '0'"));
-                }
-
-                // note: at performs range check
-                JSON_TRY
-                {
-                    ptr = &ptr->at(static_cast<size_type>(array_index(reference_token)));
-                }
-                JSON_CATCH(std::invalid_argument&)
-                {
-                    JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                }
-                break;
-            }
-
-            default:
-                JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
-        }
-    }
-
-    return *ptr;
-}
-
-NLOHMANN_BASIC_JSON_TPL_DECLARATION
-void json_pointer::flatten(const std::string& reference_string,
-                           const NLOHMANN_BASIC_JSON_TPL& value,
-                           NLOHMANN_BASIC_JSON_TPL& result)
-{
-    switch (value.m_type)
-    {
-        case detail::value_t::array:
-        {
-            if (value.m_value.array->empty())
-            {
-                // flatten empty array as null
-                result[reference_string] = nullptr;
-            }
-            else
-            {
-                // iterate array and use index as reference string
-                for (std::size_t i = 0; i < value.m_value.array->size(); ++i)
-                {
-                    flatten(reference_string + "/" + std::to_string(i),
-                            value.m_value.array->operator[](i), result);
-                }
-            }
-            break;
-        }
-
-        case detail::value_t::object:
-        {
-            if (value.m_value.object->empty())
-            {
-                // flatten empty object as null
-                result[reference_string] = nullptr;
-            }
-            else
-            {
-                // iterate object and use keys as reference string
-                for (const auto& element : *value.m_value.object)
-                {
-                    flatten(reference_string + "/" + escape(element.first), element.second, result);
-                }
-            }
-            break;
-        }
-
-        default:
-        {
-            // add primitive value with its reference string
-            result[reference_string] = value;
-            break;
-        }
-    }
-}
-
-NLOHMANN_BASIC_JSON_TPL_DECLARATION
-NLOHMANN_BASIC_JSON_TPL
-json_pointer::unflatten(const NLOHMANN_BASIC_JSON_TPL& value)
-{
-    if (JSON_UNLIKELY(not value.is_object()))
-    {
-        JSON_THROW(detail::type_error::create(314, "only objects can be unflattened"));
-    }
-
-    NLOHMANN_BASIC_JSON_TPL result;
-
-    // iterate the JSON object values
-    for (const auto& element : *value.m_value.object)
-    {
-        if (JSON_UNLIKELY(not element.second.is_primitive()))
+        else
         {
-            JSON_THROW(detail::type_error::create(315, "values in object must be primitive"));
+            *this = apply_patch;
         }
-
-        // assign value to reference pointed to by JSON pointer; Note that if
-        // the JSON pointer is "" (i.e., points to the whole value), function
-        // get_and_create returns a reference to result itself. An assignment
-        // will then create a primitive value.
-        json_pointer(element.first).get_and_create(result) = element.second;
     }
 
-    return result;
-}
-
-inline bool operator==(json_pointer const& lhs, json_pointer const& rhs) noexcept
-{
-    return (lhs.reference_tokens == rhs.reference_tokens);
-}
-
-inline bool operator!=(json_pointer const& lhs, json_pointer const& rhs) noexcept
-{
-    return not (lhs == rhs);
-}
+    /// @}
+};
 } // namespace nlohmann
 
-
 ///////////////////////
 // nonmember support //
 ///////////////////////
@@ -14766,20 +20291,6 @@ inline bool operator!=(json_pointer const& lhs, json_pointer const& rhs) noexcep
 // specialization of std::swap, and std::hash
 namespace std
 {
-/*!
-@brief exchanges the values of two JSON objects
-
-@since version 1.0.0
-*/
-template<>
-inline void swap(nlohmann::json& j1,
-                 nlohmann::json& j2) noexcept(
-                     is_nothrow_move_constructible<nlohmann::json>::value and
-                     is_nothrow_move_assignable<nlohmann::json>::value
-                 )
-{
-    j1.swap(j2);
-}
 
 /// hash value for JSON objects
 template<>
@@ -14815,6 +20326,20 @@ struct less< ::nlohmann::detail::value_t>
     }
 };
 
+/*!
+@brief exchanges the values of two JSON objects
+
+@since version 1.0.0
+*/
+template<>
+inline void swap<nlohmann::json>(nlohmann::json& j1, nlohmann::json& j2) noexcept(
+    is_nothrow_move_constructible<nlohmann::json>::value and
+    is_nothrow_move_assignable<nlohmann::json>::value
+)
+{
+    j1.swap(j2);
+}
+
 } // namespace std
 
 /*!
@@ -14853,6 +20378,9 @@ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std
     return nlohmann::json::json_pointer(std::string(s, n));
 }
 
+// #include <nlohmann/detail/macro_unscope.hpp>
+
+
 // restore GCC/clang diagnostic settings
 #if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__)
     #pragma GCC diagnostic pop
@@ -14862,13 +20390,17 @@ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std
 #endif
 
 // clean up
+#undef JSON_INTERNAL_CATCH
 #undef JSON_CATCH
 #undef JSON_THROW
 #undef JSON_TRY
 #undef JSON_LIKELY
 #undef JSON_UNLIKELY
 #undef JSON_DEPRECATED
+#undef JSON_HAS_CPP_14
+#undef JSON_HAS_CPP_17
 #undef NLOHMANN_BASIC_JSON_TPL_DECLARATION
 #undef NLOHMANN_BASIC_JSON_TPL
 
+
 #endif
diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk
index 8792a4a252fa..f9db16268be0 100644
--- a/src/resolve-system-dependencies/local.mk
+++ b/src/resolve-system-dependencies/local.mk
@@ -6,6 +6,6 @@ resolve-system-dependencies_DIR := $(d)
 
 resolve-system-dependencies_INSTALL_DIR := $(libexecdir)/nix
 
-resolve-system-dependencies_LIBS := libstore libmain libutil libformat
+resolve-system-dependencies_LIBS := libstore libmain libutil
 
 resolve-system-dependencies_SOURCES := $(d)/resolve-system-dependencies.cc