about summary refs log tree commit diff
path: root/src/libutil
diff options
context:
space:
mode:
Diffstat (limited to 'src/libutil')
-rw-r--r--src/libutil/archive.hh7
-rw-r--r--src/libutil/compression.cc63
-rw-r--r--src/libutil/compression.hh2
-rw-r--r--src/libutil/config.cc112
-rw-r--r--src/libutil/config.hh151
-rw-r--r--src/libutil/hash.cc4
-rw-r--r--src/libutil/istringstream_nocopy.hh92
-rw-r--r--src/libutil/local.mk2
-rw-r--r--src/libutil/logging.cc9
-rw-r--r--src/libutil/logging.hh10
-rw-r--r--src/libutil/lru-cache.hh8
-rw-r--r--src/libutil/pool.hh8
-rw-r--r--src/libutil/serialise.cc49
-rw-r--r--src/libutil/serialise.hh63
-rw-r--r--src/libutil/sync.hh1
-rw-r--r--src/libutil/types.hh2
-rw-r--r--src/libutil/util.cc50
-rw-r--r--src/libutil/util.hh8
18 files changed, 552 insertions, 89 deletions
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index d58b91df0461..607ebf8b28f9 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -70,6 +70,13 @@ struct ParseSink
     virtual void createSymlink(const Path & path, const string & target) { };
 };
 
+struct TeeSink : ParseSink
+{
+    TeeSource source;
+
+    TeeSink(Source & source) : source(source) { }
+};
+
 void parseDump(ParseSink & sink, Source & source);
 
 void restorePath(const Path & path, Source & source);
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index a3bbb5170d9f..b0b1d709fa44 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -18,7 +18,7 @@ static ref<std::string> decompressXZ(const std::string & in)
     lzma_ret ret = lzma_stream_decoder(
         &strm, UINT64_MAX, LZMA_CONCATENATED);
     if (ret != LZMA_OK)
-        throw Error("unable to initialise lzma decoder");
+        throw CompressionError("unable to initialise lzma decoder");
 
     Finally free([&]() { lzma_end(&strm); });
 
@@ -48,7 +48,7 @@ static ref<std::string> decompressXZ(const std::string & in)
             return res;
 
         if (ret != LZMA_OK)
-            throw Error("error while decompressing xz file");
+            throw CompressionError("error %d while decompressing xz file", ret);
     }
 }
 
@@ -59,7 +59,7 @@ static ref<std::string> decompressBzip2(const std::string & in)
 
     int ret = BZ2_bzDecompressInit(&strm, 0, 0);
     if (ret != BZ_OK)
-        throw Error("unable to initialise bzip2 decoder");
+        throw CompressionError("unable to initialise bzip2 decoder");
 
     Finally free([&]() { BZ2_bzDecompressEnd(&strm); });
 
@@ -85,10 +85,19 @@ static ref<std::string> decompressBzip2(const std::string & in)
             return res;
 
         if (ret != BZ_OK)
-            throw Error("error while decompressing bzip2 file");
+            throw CompressionError("error while decompressing bzip2 file");
+
+        if (strm.avail_in == 0)
+            throw CompressionError("bzip2 data ends prematurely");
     }
 }
 
+static ref<std::string> decompressBrotli(const std::string & in)
+{
+    // FIXME: use libbrotli
+    return make_ref<std::string>(runProgram(BRO, true, {"-d"}, {in}));
+}
+
 ref<std::string> compress(const std::string & method, const std::string & in)
 {
     StringSink ssink;
@@ -106,6 +115,8 @@ ref<std::string> decompress(const std::string & method, const std::string & in)
         return decompressXZ(in);
     else if (method == "bzip2")
         return decompressBzip2(in);
+    else if (method == "br")
+        return decompressBrotli(in);
     else
         throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method);
 }
@@ -130,7 +141,7 @@ struct XzSink : CompressionSink
         lzma_ret ret = lzma_easy_encoder(
             &strm, 6, LZMA_CHECK_CRC64);
         if (ret != LZMA_OK)
-            throw Error("unable to initialise lzma encoder");
+            throw CompressionError("unable to initialise lzma encoder");
         // FIXME: apply the x86 BCJ filter?
 
         strm.next_out = outbuf;
@@ -139,7 +150,6 @@ struct XzSink : CompressionSink
 
     ~XzSink()
     {
-        assert(finished);
         lzma_end(&strm);
     }
 
@@ -155,7 +165,7 @@ struct XzSink : CompressionSink
 
             lzma_ret ret = lzma_code(&strm, LZMA_FINISH);
             if (ret != LZMA_OK && ret != LZMA_STREAM_END)
-                throw Error("error while flushing xz file");
+                throw CompressionError("error while flushing xz file");
 
             if (strm.avail_out == 0 || ret == LZMA_STREAM_END) {
                 nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
@@ -179,7 +189,7 @@ struct XzSink : CompressionSink
 
             lzma_ret ret = lzma_code(&strm, LZMA_RUN);
             if (ret != LZMA_OK)
-                throw Error("error while compressing xz file");
+                throw CompressionError("error while compressing xz file");
 
             if (strm.avail_out == 0) {
                 nextSink(outbuf, sizeof(outbuf));
@@ -202,7 +212,7 @@ struct BzipSink : CompressionSink
         memset(&strm, 0, sizeof(strm));
         int ret = BZ2_bzCompressInit(&strm, 9, 0, 30);
         if (ret != BZ_OK)
-            throw Error("unable to initialise bzip2 encoder");
+            throw CompressionError("unable to initialise bzip2 encoder");
 
         strm.next_out = outbuf;
         strm.avail_out = sizeof(outbuf);
@@ -210,7 +220,6 @@ struct BzipSink : CompressionSink
 
     ~BzipSink()
     {
-        assert(finished);
         BZ2_bzCompressEnd(&strm);
     }
 
@@ -226,7 +235,7 @@ struct BzipSink : CompressionSink
 
             int ret = BZ2_bzCompress(&strm, BZ_FINISH);
             if (ret != BZ_FINISH_OK && ret != BZ_STREAM_END)
-                throw Error("error while flushing bzip2 file");
+                throw CompressionError("error while flushing bzip2 file");
 
             if (strm.avail_out == 0 || ret == BZ_STREAM_END) {
                 nextSink((unsigned char *) outbuf, sizeof(outbuf) - strm.avail_out);
@@ -250,7 +259,7 @@ struct BzipSink : CompressionSink
 
             int ret = BZ2_bzCompress(&strm, BZ_RUN);
             if (ret != BZ_OK)
-                Error("error while compressing bzip2 file");
+                CompressionError("error while compressing bzip2 file");
 
             if (strm.avail_out == 0) {
                 nextSink((unsigned char *) outbuf, sizeof(outbuf));
@@ -261,6 +270,34 @@ struct BzipSink : CompressionSink
     }
 };
 
+struct BrotliSink : CompressionSink
+{
+    Sink & nextSink;
+    std::string data;
+
+    BrotliSink(Sink & nextSink) : nextSink(nextSink)
+    {
+    }
+
+    ~BrotliSink()
+    {
+    }
+
+    // FIXME: use libbrotli
+
+    void finish() override
+    {
+        flush();
+        nextSink(runProgram(BRO, true, {}, data));
+    }
+
+    void write(const unsigned char * data, size_t len) override
+    {
+        checkInterrupt();
+        this->data.append((const char *) data, len);
+    }
+};
+
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink)
 {
     if (method == "none")
@@ -269,6 +306,8 @@ ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & next
         return make_ref<XzSink>(nextSink);
     else if (method == "bzip2")
         return make_ref<BzipSink>(nextSink);
+    else if (method == "br")
+        return make_ref<BrotliSink>(nextSink);
     else
         throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method);
 }
diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh
index eacf559d65e9..e3e6f5a99303 100644
--- a/src/libutil/compression.hh
+++ b/src/libutil/compression.hh
@@ -21,4 +21,6 @@ ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & next
 
 MakeError(UnknownCompressionMethod, Error);
 
+MakeError(CompressionError, Error);
+
 }
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
new file mode 100644
index 000000000000..2f9f988607ea
--- /dev/null
+++ b/src/libutil/config.cc
@@ -0,0 +1,112 @@
+#include "config.hh"
+#include "args.hh"
+
+namespace nix {
+
+void Config::set(const std::string & name, const std::string & value)
+{
+    auto i = _settings.find(name);
+    if (i == _settings.end())
+        throw UsageError("unknown setting '%s'", name);
+    i->second.setting->set(value);
+}
+
+void Config::add(AbstractSetting * setting)
+{
+    _settings.emplace(setting->name, Config::SettingData{false, setting});
+    for (auto & alias : setting->aliases)
+        _settings.emplace(alias, Config::SettingData{true, setting});
+
+    bool set = false;
+
+    auto i = initials.find(setting->name);
+    if (i != initials.end()) {
+        setting->set(i->second);
+        initials.erase(i);
+        set = true;
+    }
+
+    for (auto & alias : setting->aliases) {
+        auto i = initials.find(alias);
+        if (i != initials.end()) {
+            if (set)
+                warn("setting '%s' is set, but it's an alias of '%s' which is also set",
+                    alias, setting->name);
+            else {
+                setting->set(i->second);
+                initials.erase(i);
+                set = true;
+            }
+        }
+    }
+}
+
+void Config::warnUnused()
+{
+    for (auto & i : initials)
+        warn("unknown setting '%s'", i.first);
+}
+
+std::string Config::dump()
+{
+    std::string res;
+    for (auto & opt : _settings)
+        if (!opt.second.isAlias)
+            res += opt.first + " = " + opt.second.setting->to_string() + "\n";
+    return res;
+}
+
+AbstractSetting::AbstractSetting(
+    const std::string & name,
+    const std::string & description,
+    const std::set<std::string> & aliases)
+    : name(name), description(description), aliases(aliases)
+{
+}
+
+template<> void Setting<std::string>::set(const std::string & str)
+{
+    value = str;
+}
+
+template<> std::string Setting<std::string>::to_string()
+{
+    return value;
+}
+
+template<> void Setting<int>::set(const std::string & str)
+{
+    try {
+        value = std::stoi(str);
+    } catch (...) {
+        throw UsageError("setting '%s' has invalid value '%s'", name, str);
+    }
+}
+
+template<> std::string Setting<int>::to_string()
+{
+    return std::to_string(value);
+}
+
+template<> void Setting<bool>::set(const std::string & str)
+{
+    value = str == "true" || str == "1";
+}
+
+template<> std::string Setting<bool>::to_string()
+{
+    return value ? "true" : "false";
+}
+
+void PathSetting::set(const std::string & str)
+{
+    if (str == "") {
+        if (allowEmpty)
+            value = "";
+        else
+            throw UsageError("setting '%s' cannot be empty", name);
+    } else
+        value = canonPath(str);
+}
+
+}
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
new file mode 100644
index 000000000000..fb2d48e9c834
--- /dev/null
+++ b/src/libutil/config.hh
@@ -0,0 +1,151 @@
+#include <map>
+#include <set>
+
+#include "types.hh"
+
+#pragma once
+
+namespace nix {
+
+class Args;
+class AbstractSetting;
+
+/* A class to simplify providing configuration settings. The typical
+   use is to inherit Config and add Setting<T> members:
+
+   class MyClass : private Config
+   {
+     Setting<int> foo{this, 123, "foo", "the number of foos to use"};
+     Setting<std::string> bar{this, "blabla", "bar", "the name of the bar"};
+
+     MyClass() : Config(readConfigFile("/etc/my-app.conf"))
+     {
+       std::cout << foo << "\n"; // will print 123 unless overriden
+     }
+   };
+*/
+
+class Config
+{
+    friend class AbstractSetting;
+
+    struct SettingData
+    {
+        bool isAlias = false;
+        AbstractSetting * setting;
+    };
+
+    std::map<std::string, SettingData> _settings;
+
+    StringMap initials;
+
+public:
+
+    Config(const StringMap & initials)
+        : initials(initials)
+    { }
+
+    void set(const std::string & name, const std::string & value);
+
+    void add(AbstractSetting * setting);
+
+    void warnUnused();
+
+    std::string dump();
+};
+
+class AbstractSetting
+{
+    friend class Config;
+
+public:
+
+    const std::string name;
+    const std::string description;
+    const std::set<std::string> aliases;
+
+    int created = 123;
+
+protected:
+
+    AbstractSetting(
+        const std::string & name,
+        const std::string & description,
+        const std::set<std::string> & aliases);
+
+    virtual ~AbstractSetting()
+    {
+        // Check against a gcc miscompilation causing our constructor
+        // not to run.
+        assert(created == 123);
+    }
+
+    virtual void set(const std::string & value) = 0;
+
+    virtual std::string to_string() = 0;
+};
+
+/* A setting of type T. */
+template<typename T>
+class Setting : public AbstractSetting
+{
+protected:
+
+    T value;
+
+public:
+
+    Setting(Config * options,
+        const T & def,
+        const std::string & name,
+        const std::string & description,
+        const std::set<std::string> & aliases = {})
+        : AbstractSetting(name, description, aliases)
+        , value(def)
+    {
+        options->add(this);
+    }
+
+    operator const T &() const { return value; }
+    bool operator ==(const T & v2) const { return value == v2; }
+    bool operator !=(const T & v2) const { return value != v2; }
+    void operator =(const T & v) { value = v; }
+
+    void set(const std::string & str) override;
+
+    std::string to_string() override;
+};
+
+template<typename T>
+std::ostream & operator <<(std::ostream & str, const Setting<T> & opt)
+{
+    str << (const T &) opt;
+    return str;
+}
+
+/* A special setting for Paths. These are automatically canonicalised
+   (e.g. "/foo//bar/" becomes "/foo/bar"). */
+class PathSetting : public Setting<Path>
+{
+    bool allowEmpty;
+
+public:
+
+    PathSetting(Config * options,
+        bool allowEmpty,
+        const Path & def,
+        const std::string & name,
+        const std::string & description,
+        const std::set<std::string> & aliases = {})
+        : Setting<Path>(options, def, name, description, aliases)
+        , allowEmpty(allowEmpty)
+    {
+        set(value);
+    }
+
+    void set(const std::string & str) override;
+
+    Path operator +(const char * p) const { return value + p; }
+};
+
+}
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index f447c80c5d81..9f4afd93c2fc 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -7,12 +7,12 @@
 #include "hash.hh"
 #include "archive.hh"
 #include "util.hh"
+#include "istringstream_nocopy.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
 
-
 namespace nix {
 
 
@@ -104,7 +104,7 @@ Hash parseHash(HashType ht, const string & s)
         string s2(s, i * 2, 2);
         if (!isxdigit(s2[0]) || !isxdigit(s2[1]))
             throw BadHash(format("invalid hash ‘%1%’") % s);
-        std::istringstream str(s2);
+        istringstream_nocopy str(s2);
         int n;
         str >> std::hex >> n;
         hash.hash[i] = n;
diff --git a/src/libutil/istringstream_nocopy.hh b/src/libutil/istringstream_nocopy.hh
new file mode 100644
index 000000000000..f7beac578e39
--- /dev/null
+++ b/src/libutil/istringstream_nocopy.hh
@@ -0,0 +1,92 @@
+/* This file provides a variant of std::istringstream that doesn't
+   copy its string argument. This is useful for large strings. The
+   caller must ensure that the string object is not destroyed while
+   it's referenced by this object. */
+
+#pragma once
+
+#include <string>
+#include <iostream>
+
+template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
+class basic_istringbuf_nocopy : public std::basic_streambuf<CharT, Traits>
+{
+public:
+    typedef std::basic_string<CharT, Traits, Allocator> string_type;
+
+    typedef typename std::basic_streambuf<CharT, Traits>::off_type off_type;
+
+    typedef typename std::basic_streambuf<CharT, Traits>::pos_type pos_type;
+
+    typedef typename std::basic_streambuf<CharT, Traits>::int_type int_type;
+
+    typedef typename std::basic_streambuf<CharT, Traits>::traits_type traits_type;
+
+private:
+    const string_type & s;
+
+    off_type off;
+
+public:
+    basic_istringbuf_nocopy(const string_type & s) : s{s}, off{0}
+    {
+    }
+
+private:
+    pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which)
+    {
+        if (which & std::ios_base::in) {
+            this->off = dir == std::ios_base::beg
+                ? off
+                : (dir == std::ios_base::end
+                    ? s.size() + off
+                    : this->off + off);
+        }
+        return pos_type(this->off);
+    }
+
+    pos_type seekpos(pos_type pos, std::ios_base::openmode which)
+    {
+        return seekoff(pos, std::ios_base::beg, which);
+    }
+
+    std::streamsize showmanyc()
+    {
+        return s.size() - off;
+    }
+
+    int_type underflow()
+    {
+        if (typename string_type::size_type(off) == s.size())
+            return traits_type::eof();
+        return traits_type::to_int_type(s[off]);
+    }
+
+    int_type uflow()
+    {
+        if (typename string_type::size_type(off) == s.size())
+            return traits_type::eof();
+        return traits_type::to_int_type(s[off++]);
+    }
+
+    int_type pbackfail(int_type ch)
+    {
+        if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1]))
+            return traits_type::eof();
+
+        return traits_type::to_int_type(s[--off]);
+    }
+
+};
+
+template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
+class basic_istringstream_nocopy : public std::basic_iostream<CharT, Traits>
+{
+    typedef basic_istringbuf_nocopy<CharT, Traits, Allocator> buf_type;
+    buf_type buf;
+public:
+    basic_istringstream_nocopy(const typename buf_type::string_type & s) :
+        std::basic_iostream<CharT, Traits>(&buf), buf(s) {};
+};
+
+typedef basic_istringstream_nocopy<char> istringstream_nocopy;
diff --git a/src/libutil/local.mk b/src/libutil/local.mk
index cac5c8795db7..0721b21c2089 100644
--- a/src/libutil/local.mk
+++ b/src/libutil/local.mk
@@ -9,3 +9,5 @@ libutil_SOURCES := $(wildcard $(d)/*.cc)
 libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS)
 
 libutil_LIBS = libformat
+
+libutil_CXXFLAGS = -DBRO=\"$(bro)\"
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index d9e8d22d7685..afcc2ec58543 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -3,7 +3,12 @@
 
 namespace nix {
 
-Logger * logger = 0;
+Logger * logger = makeDefaultLogger();
+
+void Logger::warn(const std::string & msg)
+{
+    log(lvlInfo, ANSI_RED "warning:" ANSI_NORMAL " " + msg);
+}
 
 class SimpleLogger : public Logger
 {
@@ -52,7 +57,7 @@ Verbosity verbosity = lvlInfo;
 void warnOnce(bool & haveWarned, const FormatOrString & fs)
 {
     if (!haveWarned) {
-        printError(format("warning: %1%") % fs.s);
+        warn(fs.s);
         haveWarned = true;
     }
 }
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 3f83664794f7..81aebccdca45 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -30,6 +30,8 @@ public:
         log(lvlInfo, fs);
     }
 
+    virtual void warn(const std::string & msg);
+
     virtual void setExpected(const std::string & label, uint64_t value = 1) { }
     virtual void setProgress(const std::string & label, uint64_t value = 1) { }
     virtual void incExpected(const std::string & label, uint64_t value = 1) { }
@@ -82,6 +84,14 @@ extern Verbosity verbosity; /* suppress msgs > this */
 #define debug(args...) printMsg(lvlDebug, args)
 #define vomit(args...) printMsg(lvlVomit, args)
 
+template<typename... Args>
+inline void warn(const std::string & fs, Args... args)
+{
+    boost::format f(fs);
+    formatHelper(f, args...);
+    logger->warn(f.str());
+}
+
 void warnOnce(bool & haveWarned, const FormatOrString & fs);
 
 void writeToStderr(const string & s);
diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh
index 35983aa2c918..3cb5d50889d9 100644
--- a/src/libutil/lru-cache.hh
+++ b/src/libutil/lru-cache.hh
@@ -11,7 +11,7 @@ class LRUCache
 {
 private:
 
-    size_t maxSize;
+    size_t capacity;
 
     // Stupid wrapper to get around circular dependency between Data
     // and LRU.
@@ -27,14 +27,16 @@ private:
 
 public:
 
-    LRUCache(size_t maxSize) : maxSize(maxSize) { }
+    LRUCache(size_t capacity) : capacity(capacity) { }
 
     /* Insert or upsert an item in the cache. */
     void upsert(const Key & key, const Value & value)
     {
+        if (capacity == 0) return;
+
         erase(key);
 
-        if (data.size() >= maxSize) {
+        if (data.size() >= capacity) {
             /* Retire the oldest item. */
             auto oldest = lru.begin();
             data.erase(*oldest);
diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh
index f291cd578388..20df21948849 100644
--- a/src/libutil/pool.hh
+++ b/src/libutil/pool.hh
@@ -137,15 +137,21 @@ public:
         } catch (...) {
             auto state_(state.lock());
             state_->inUse--;
+            wakeup.notify_one();
             throw;
         }
     }
 
-    unsigned int count()
+    size_t count()
     {
         auto state_(state.lock());
         return state_->idle.size() + state_->inUse;
     }
+
+    size_t capacity()
+    {
+        return state.lock()->max;
+    }
 };
 
 }
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index a68f7a0fa8ee..950e6362a245 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -194,39 +194,9 @@ void readPadding(size_t len, Source & source)
 }
 
 
-unsigned int readInt(Source & source)
-{
-    unsigned char buf[8];
-    source(buf, sizeof(buf));
-    if (buf[4] || buf[5] || buf[6] || buf[7])
-        throw SerialisationError("implementation cannot deal with > 32-bit integers");
-    return
-        buf[0] |
-        (buf[1] << 8) |
-        (buf[2] << 16) |
-        (buf[3] << 24);
-}
-
-
-unsigned long long readLongLong(Source & source)
-{
-    unsigned char buf[8];
-    source(buf, sizeof(buf));
-    return
-        ((unsigned long long) buf[0]) |
-        ((unsigned long long) buf[1] << 8) |
-        ((unsigned long long) buf[2] << 16) |
-        ((unsigned long long) buf[3] << 24) |
-        ((unsigned long long) buf[4] << 32) |
-        ((unsigned long long) buf[5] << 40) |
-        ((unsigned long long) buf[6] << 48) |
-        ((unsigned long long) buf[7] << 56);
-}
-
-
 size_t readString(unsigned char * buf, size_t max, Source & source)
 {
-    size_t len = readInt(source);
+    auto len = readNum<size_t>(source);
     if (len > max) throw Error("string is too long");
     source(buf, len);
     readPadding(len, source);
@@ -236,11 +206,11 @@ size_t readString(unsigned char * buf, size_t max, Source & source)
 
 string readString(Source & source)
 {
-    size_t len = readInt(source);
-    auto buf = std::make_unique<unsigned char[]>(len);
-    source(buf.get(), len);
+    auto len = readNum<size_t>(source);
+    std::string res(len, 0);
+    source((unsigned char*) res.data(), len);
     readPadding(len, source);
-    return string((char *) buf.get(), len);
+    return res;
 }
 
 Source & operator >> (Source & in, string & s)
@@ -250,16 +220,9 @@ Source & operator >> (Source & in, string & s)
 }
 
 
-Source & operator >> (Source & in, unsigned int & n)
-{
-    n = readInt(in);
-    return in;
-}
-
-
 template<class T> T readStrings(Source & source)
 {
-    unsigned int count = readInt(source);
+    auto count = readNum<size_t>(source);
     T ss;
     while (count--)
         ss.insert(ss.end(), readString(source));
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 5646d08c1314..2bdee70807be 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -140,15 +140,16 @@ struct StringSource : Source
 
 
 /* Adapter class of a Source that saves all data read to `s'. */
-struct SavingSourceAdapter : Source
+struct TeeSource : Source
 {
     Source & orig;
-    string s;
-    SavingSourceAdapter(Source & orig) : orig(orig) { }
+    ref<std::string> data;
+    TeeSource(Source & orig)
+        : orig(orig), data(make_ref<std::string>()) { }
     size_t read(unsigned char * data, size_t len)
     {
         size_t n = orig.read(data, len);
-        s.append((const char *) data, n);
+        this->data->append((const char *) data, n);
         return n;
     }
 };
@@ -177,18 +178,64 @@ Sink & operator << (Sink & sink, const Strings & s);
 Sink & operator << (Sink & sink, const StringSet & s);
 
 
+MakeError(SerialisationError, Error)
+
+
+template<typename T>
+T readNum(Source & source)
+{
+    unsigned char buf[8];
+    source(buf, sizeof(buf));
+
+    uint64_t n =
+        ((unsigned long long) buf[0]) |
+        ((unsigned long long) buf[1] << 8) |
+        ((unsigned long long) buf[2] << 16) |
+        ((unsigned long long) buf[3] << 24) |
+        ((unsigned long long) buf[4] << 32) |
+        ((unsigned long long) buf[5] << 40) |
+        ((unsigned long long) buf[6] << 48) |
+        ((unsigned long long) buf[7] << 56);
+
+    if (n > std::numeric_limits<T>::max())
+        throw SerialisationError("serialised integer %d is too large for type ‘%s’", n, typeid(T).name());
+
+    return n;
+}
+
+
+inline unsigned int readInt(Source & source)
+{
+    return readNum<unsigned int>(source);
+}
+
+
+inline uint64_t readLongLong(Source & source)
+{
+    return readNum<uint64_t>(source);
+}
+
+
 void readPadding(size_t len, Source & source);
-unsigned int readInt(Source & source);
-unsigned long long readLongLong(Source & source);
 size_t readString(unsigned char * buf, size_t max, Source & source);
 string readString(Source & source);
 template<class T> T readStrings(Source & source);
 
 Source & operator >> (Source & in, string & s);
-Source & operator >> (Source & in, unsigned int & n);
 
+template<typename T>
+Source & operator >> (Source & in, T & n)
+{
+    n = readNum<T>(in);
+    return in;
+}
 
-MakeError(SerialisationError, Error)
+template<typename T>
+Source & operator >> (Source & in, bool & b)
+{
+    b = readNum<uint64_t>(in);
+    return in;
+}
 
 
 }
diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh
index 2aa074299b23..611c900e0a32 100644
--- a/src/libutil/sync.hh
+++ b/src/libutil/sync.hh
@@ -33,6 +33,7 @@ public:
 
     Sync() { }
     Sync(const T & data) : data(data) { }
+    Sync(T && data) noexcept : data(std::move(data)) { }
 
     class Lock
     {
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 97d79af9b5d6..1429c238513b 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -7,6 +7,7 @@
 #include <list>
 #include <set>
 #include <memory>
+#include <map>
 
 #include <boost/format.hpp>
 
@@ -141,6 +142,7 @@ private:
 
 typedef list<string> Strings;
 typedef set<string> StringSet;
+typedef std::map<std::string, std::string> StringMap;
 
 
 /* Paths are just strings. */
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 0a5f796e4eaa..0bd51afd1a9f 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1,6 +1,7 @@
 #include "util.hh"
 #include "affinity.hh"
 #include "sync.hh"
+#include "finally.hh"
 
 #include <cctype>
 #include <cerrno>
@@ -10,6 +11,7 @@
 #include <iostream>
 #include <sstream>
 #include <thread>
+#include <future>
 
 #include <sys/wait.h>
 #include <unistd.h>
@@ -94,6 +96,8 @@ Path absPath(Path path, Path dir)
 
 Path canonPath(const Path & path, bool resolveSymlinks)
 {
+    assert(path != "");
+
     string s;
 
     if (path[0] != '/')
@@ -485,6 +489,7 @@ void readFull(int fd, unsigned char * buf, size_t count)
 void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts)
 {
     while (count) {
+        if (allowInterrupts) checkInterrupt();
         ssize_t res = write(fd, (char *) buf, count);
         if (res == -1 && errno != EINTR)
             throw SysError("writing to file");
@@ -492,7 +497,6 @@ void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterr
             count -= res;
             buf += res;
         }
-        if (allowInterrupts) checkInterrupt();
     }
 }
 
@@ -676,12 +680,11 @@ Pid::operator pid_t()
 }
 
 
-int Pid::kill(bool quiet)
+int Pid::kill()
 {
     assert(pid != -1);
 
-    if (!quiet)
-        printError(format("killing process %1%") % pid);
+    debug(format("killing process %1%") % pid);
 
     /* Send the requested signal to the child.  If it has its own
        process group, send the signal to every process in the child
@@ -837,23 +840,21 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss)
 
 
 string runProgram(Path program, bool searchPath, const Strings & args,
-    const string & input)
+    const std::experimental::optional<std::string> & input)
 {
     checkInterrupt();
 
     /* Create a pipe. */
     Pipe out, in;
     out.create();
-    if (!input.empty()) in.create();
+    if (input) in.create();
 
     /* Fork. */
     Pid pid = startProcess([&]() {
         if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
             throw SysError("dupping stdout");
-        if (!input.empty()) {
-            if (dup2(in.readSide.get(), STDIN_FILENO) == -1)
-                throw SysError("dupping stdin");
-        }
+        if (input && dup2(in.readSide.get(), STDIN_FILENO) == -1)
+            throw SysError("dupping stdin");
 
         Strings args_(args);
         args_.push_front(program);
@@ -870,11 +871,27 @@ string runProgram(Path program, bool searchPath, const Strings & args,
 
     out.writeSide = -1;
 
-    /* FIXME: This can deadlock if the input is too long. */
-    if (!input.empty()) {
+    std::thread writerThread;
+
+    std::promise<void> promise;
+
+    Finally doJoin([&]() {
+        if (writerThread.joinable())
+            writerThread.join();
+    });
+
+
+    if (input) {
         in.readSide = -1;
-        writeFull(in.writeSide.get(), input);
-        in.writeSide = -1;
+        writerThread = std::thread([&]() {
+            try {
+                writeFull(in.writeSide.get(), *input);
+                promise.set_value();
+            } catch (...) {
+                promise.set_exception(std::current_exception());
+            }
+            in.writeSide = -1;
+        });
     }
 
     string result = drainFD(out.readSide.get());
@@ -885,6 +902,9 @@ string runProgram(Path program, bool searchPath, const Strings & args,
         throw ExecError(status, format("program ‘%1%’ %2%")
             % program % statusToString(status));
 
+    /* Wait for the writer thread to finish. */
+    if (input) promise.get_future().get();
+
     return result;
 }
 
@@ -1194,7 +1214,7 @@ static void signalHandlerThread(sigset_t set)
 
 void triggerInterrupt()
 {
-    _isInterrupted = 1;
+    _isInterrupted = true;
 
     {
         auto interruptCallbacks(_interruptCallbacks.lock());
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 2950f7daa5ec..0e6941e4a8db 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -13,6 +13,8 @@
 #include <limits>
 #include <cstdio>
 #include <map>
+#include <sstream>
+#include <experimental/optional>
 
 #ifndef HAVE_STRUCT_DIRENT_D_TYPE
 #define DT_UNKNOWN 0
@@ -201,7 +203,7 @@ public:
     ~Pid();
     void operator =(pid_t pid);
     operator pid_t();
-    int kill(bool quiet = false);
+    int kill();
     int wait();
 
     void setSeparatePG(bool separatePG);
@@ -231,7 +233,8 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = P
 /* Run a program and return its stdout in a string (i.e., like the
    shell backtick operator). */
 string runProgram(Path program, bool searchPath = false,
-    const Strings & args = Strings(), const string & input = "");
+    const Strings & args = Strings(),
+    const std::experimental::optional<std::string> & input = {});
 
 class ExecError : public Error
 {
@@ -448,5 +451,4 @@ struct ReceiveInterrupts
     { }
 };
 
-
 }
generated by cgit-pink 1.4.1 (git 2.44.2) at 2025-01-05T00·43+0000