Merge branch 'master' into new-cli

13 files changed, 515 insertions, 97 deletions

diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 6ee7981432b6..5363496c272e 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -29,7 +29,7 @@ bool useCaseHack =
     false;
 #endif
 
-static string archiveVersion1 = "nix-archive-1";
+const std::string narVersionMagic1 = "nix-archive-1";
 
 static string caseHackSuffix = "~nix~case~hack~";
 
@@ -113,11 +113,17 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
 
 void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
 {
-    sink << archiveVersion1;
+    sink << narVersionMagic1;
     dump(path, sink, filter);
 }
 
 
+void dumpString(const std::string & s, Sink & sink)
+{
+    sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")";
+}
+
+
 static SerialisationError badArchive(string s)
 {
     return SerialisationError("bad archive: " + s);
@@ -214,7 +220,8 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
         }
 
         else if (s == "executable" && type == tpRegular) {
-            readString(source);
+            auto s = readString(source);
+            if (s != "") throw badArchive("executable marker has non-empty value");
             sink.isExecutable();
         }
 
@@ -275,7 +282,7 @@ void parseDump(ParseSink & sink, Source & source)
         /* This generally means the integer at the start couldn't be
            decoded.  Ignore and throw the exception below. */
     }
-    if (version != archiveVersion1)
+    if (version != narVersionMagic1)
         throw badArchive("input doesn't look like a Nix archive");
     parse(sink, source, "");
 }
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index c216e9768fd1..90117f5ff168 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -55,6 +55,8 @@ extern PathFilter defaultPathFilter;
 void dumpPath(const Path & path, Sink & sink,
     PathFilter & filter = defaultPathFilter);
 
+void dumpString(const std::string & s, Sink & sink);
+
 struct ParseSink
 {
     virtual void createDirectory(const Path & path) { };
@@ -76,4 +78,7 @@ void restorePath(const Path & path, Source & source);
 extern bool useCaseHack;
 
 
+extern const std::string narVersionMagic1;
+
+
 }
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index fb4160669a29..a3fa0dab737b 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -6,34 +6,83 @@
 
 namespace nix {
 
+/* RAII wrapper around lzma_stream. */
+struct LzmaStream
+{
+    lzma_stream strm;
+    LzmaStream() : strm(LZMA_STREAM_INIT) { };
+    ~LzmaStream() { lzma_end(&strm); };
+    lzma_stream & operator()() { return strm; }
+};
+
+std::string compressXZ(const std::string & in)
+{
+    LzmaStream strm;
+
+    // FIXME: apply the x86 BCJ filter?
+
+    lzma_ret ret = lzma_easy_encoder(
+        &strm(), 6, LZMA_CHECK_CRC64);
+    if (ret != LZMA_OK)
+        throw Error("unable to initialise lzma encoder");
+
+    lzma_action action = LZMA_RUN;
+    uint8_t outbuf[BUFSIZ];
+    string res;
+    strm().next_in = (uint8_t *) in.c_str();
+    strm().avail_in = in.size();
+    strm().next_out = outbuf;
+    strm().avail_out = sizeof(outbuf);
+
+    while (true) {
+
+        if (strm().avail_in == 0)
+            action = LZMA_FINISH;
+
+        lzma_ret ret = lzma_code(&strm(), action);
+
+        if (strm().avail_out == 0 || ret == LZMA_STREAM_END) {
+            res.append((char *) outbuf, sizeof(outbuf) - strm().avail_out);
+            strm().next_out = outbuf;
+            strm().avail_out = sizeof(outbuf);
+        }
+
+        if (ret == LZMA_STREAM_END)
+            return res;
+
+        if (ret != LZMA_OK)
+            throw Error("error while decompressing xz file");
+    }
+}
+
 std::string decompressXZ(const std::string & in)
 {
-    lzma_stream strm = LZMA_STREAM_INIT;
+    LzmaStream strm;
 
     lzma_ret ret = lzma_stream_decoder(
-        &strm, UINT64_MAX, LZMA_CONCATENATED);
+        &strm(), UINT64_MAX, LZMA_CONCATENATED);
     if (ret != LZMA_OK)
         throw Error("unable to initialise lzma decoder");
 
     lzma_action action = LZMA_RUN;
     uint8_t outbuf[BUFSIZ];
     string res;
-    strm.next_in = (uint8_t *) in.c_str();
-    strm.avail_in = in.size();
-    strm.next_out = outbuf;
-    strm.avail_out = sizeof(outbuf);
+    strm().next_in = (uint8_t *) in.c_str();
+    strm().avail_in = in.size();
+    strm().next_out = outbuf;
+    strm().avail_out = sizeof(outbuf);
 
     while (true) {
 
-        if (strm.avail_in == 0)
+        if (strm().avail_in == 0)
             action = LZMA_FINISH;
 
-        lzma_ret ret = lzma_code(&strm, action);
+        lzma_ret ret = lzma_code(&strm(), action);
 
-        if (strm.avail_out == 0 || ret == LZMA_STREAM_END) {
-            res.append((char *) outbuf, sizeof(outbuf) - strm.avail_out);
-            strm.next_out = outbuf;
-            strm.avail_out = sizeof(outbuf);
+        if (strm().avail_out == 0 || ret == LZMA_STREAM_END) {
+            res.append((char *) outbuf, sizeof(outbuf) - strm().avail_out);
+            strm().next_out = outbuf;
+            strm().avail_out = sizeof(outbuf);
         }
 
         if (ret == LZMA_STREAM_END)
diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh
index 962ce5ac7767..eb1697fc4aa4 100644
--- a/src/libutil/compression.hh
+++ b/src/libutil/compression.hh
@@ -4,6 +4,8 @@
 
 namespace nix {
 
+std::string compressXZ(const std::string & in);
+
 std::string decompressXZ(const std::string & in);
 
 }
diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh
new file mode 100644
index 000000000000..4344d6601bc8
--- /dev/null
+++ b/src/libutil/lru-cache.hh
@@ -0,0 +1,84 @@
+#pragma once
+
+#include <map>
+#include <list>
+
+namespace nix {
+
+/* A simple least-recently used cache. Not thread-safe. */
+template<typename Key, typename Value>
+class LRUCache
+{
+private:
+
+    size_t maxSize;
+
+    // Stupid wrapper to get around circular dependency between Data
+    // and LRU.
+    struct LRUIterator;
+
+    using Data = std::map<Key, std::pair<LRUIterator, Value>>;
+    using LRU = std::list<typename Data::iterator>;
+
+    struct LRUIterator { typename LRU::iterator it; };
+
+    Data data;
+    LRU lru;
+
+public:
+
+    LRUCache(size_t maxSize) : maxSize(maxSize) { }
+
+    /* Insert or upsert an item in the cache. */
+    void upsert(const Key & key, const Value & value)
+    {
+        erase(key);
+
+        if (data.size() >= maxSize) {
+            /* Retire the oldest item. */
+            auto oldest = lru.begin();
+            data.erase(*oldest);
+            lru.erase(oldest);
+        }
+
+        auto res = data.emplace(key, std::make_pair(LRUIterator(), value));
+        assert(res.second);
+        auto & i(res.first);
+
+        auto j = lru.insert(lru.end(), i);
+
+        i->second.first.it = j;
+    }
+
+    bool erase(const Key & key)
+    {
+        auto i = data.find(key);
+        if (i == data.end()) return false;
+        lru.erase(i->second.first.it);
+        data.erase(i);
+        return true;
+    }
+
+    /* Look up an item in the cache. If it exists, it becomes the most
+       recently used item. */
+    // FIXME: use boost::optional?
+    Value * get(const Key & key)
+    {
+        auto i = data.find(key);
+        if (i == data.end()) return 0;
+
+        /* Move this item to the back of the LRU list. */
+        lru.erase(i->second.first.it);
+        auto j = lru.insert(lru.end(), i);
+        i->second.first.it = j;
+
+        return &i->second.second;
+    }
+
+    size_t size()
+    {
+        return data.size();
+    }
+};
+
+}
diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh
new file mode 100644
index 000000000000..f291cd578388
--- /dev/null
+++ b/src/libutil/pool.hh
@@ -0,0 +1,151 @@
+#pragma once
+
+#include <functional>
+#include <limits>
+#include <list>
+#include <memory>
+#include <cassert>
+
+#include "sync.hh"
+#include "ref.hh"
+
+namespace nix {
+
+/* This template class implements a simple pool manager of resources
+   of some type R, such as database connections. It is used as
+   follows:
+
+     class Connection { ... };
+
+     Pool<Connection> pool;
+
+     {
+       auto conn(pool.get());
+       conn->exec("select ...");
+     }
+
+   Here, the Connection object referenced by ‘conn’ is automatically
+   returned to the pool when ‘conn’ goes out of scope.
+*/
+
+template <class R>
+class Pool
+{
+public:
+
+    /* A function that produces new instances of R on demand. */
+    typedef std::function<ref<R>()> Factory;
+
+    /* A function that checks whether an instance of R is still
+       usable. Unusable instances are removed from the pool. */
+    typedef std::function<bool(const ref<R> &)> Validator;
+
+private:
+
+    Factory factory;
+    Validator validator;
+
+    struct State
+    {
+        size_t inUse = 0;
+        size_t max;
+        std::vector<ref<R>> idle;
+    };
+
+    Sync<State> state;
+
+    std::condition_variable wakeup;
+
+public:
+
+    Pool(size_t max = std::numeric_limits<size_t>::max(),
+        const Factory & factory = []() { return make_ref<R>(); },
+        const Validator & validator = [](ref<R> r) { return true; })
+        : factory(factory)
+        , validator(validator)
+    {
+        auto state_(state.lock());
+        state_->max = max;
+    }
+
+    ~Pool()
+    {
+        auto state_(state.lock());
+        assert(!state_->inUse);
+        state_->max = 0;
+        state_->idle.clear();
+    }
+
+    class Handle
+    {
+    private:
+        Pool & pool;
+        std::shared_ptr<R> r;
+
+        friend Pool;
+
+        Handle(Pool & pool, std::shared_ptr<R> r) : pool(pool), r(r) { }
+
+    public:
+        Handle(Handle && h) : pool(h.pool), r(h.r) { h.r.reset(); }
+
+        Handle(const Handle & l) = delete;
+
+        ~Handle()
+        {
+            if (!r) return;
+            {
+                auto state_(pool.state.lock());
+                state_->idle.push_back(ref<R>(r));
+                assert(state_->inUse);
+                state_->inUse--;
+            }
+            pool.wakeup.notify_one();
+        }
+
+        R * operator -> () { return &*r; }
+        R & operator * () { return *r; }
+    };
+
+    Handle get()
+    {
+        {
+            auto state_(state.lock());
+
+            /* If we're over the maximum number of instance, we need
+               to wait until a slot becomes available. */
+            while (state_->idle.empty() && state_->inUse >= state_->max)
+                state_.wait(wakeup);
+
+            while (!state_->idle.empty()) {
+                auto p = state_->idle.back();
+                state_->idle.pop_back();
+                if (validator(p)) {
+                    state_->inUse++;
+                    return Handle(*this, p);
+                }
+            }
+
+            state_->inUse++;
+        }
+
+        /* We need to create a new instance. Because that might take a
+           while, we don't hold the lock in the meantime. */
+        try {
+            Handle h(*this, factory());
+            return h;
+        } catch (...) {
+            auto state_(state.lock());
+            state_->inUse--;
+            throw;
+        }
+    }
+
+    unsigned int count()
+    {
+        auto state_(state.lock());
+        return state_->idle.size() + state_->inUse;
+    }
+};
+
+}
diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh
new file mode 100644
index 000000000000..a6d338d79622
--- /dev/null
+++ b/src/libutil/ref.hh
@@ -0,0 +1,67 @@
+#pragma once
+
+#include <memory>
+#include <exception>
+
+namespace nix {
+
+/* A simple non-nullable reference-counted pointer. Actually a wrapper
+   around std::shared_ptr that prevents non-null constructions. */
+template<typename T>
+class ref
+{
+private:
+
+    std::shared_ptr<T> p;
+
+public:
+
+    ref<T>(const ref<T> & r)
+        : p(r.p)
+    { }
+
+    explicit ref<T>(const std::shared_ptr<T> & p)
+        : p(p)
+    {
+        if (!p)
+            throw std::invalid_argument("null pointer cast to ref");
+    }
+
+    T* operator ->() const
+    {
+        return &*p;
+    }
+
+    T& operator *() const
+    {
+        return *p;
+    }
+
+    operator std::shared_ptr<T> ()
+    {
+        return p;
+    }
+
+    template<typename T2>
+    operator ref<T2> ()
+    {
+        return ref<T2>((std::shared_ptr<T2>) p);
+    }
+
+private:
+
+    template<typename T2, typename... Args>
+    friend ref<T2>
+    make_ref(Args&&... args);
+
+};
+
+template<typename T, typename... Args>
+inline ref<T>
+make_ref(Args&&... args)
+{
+    auto p = std::make_shared<T>(std::forward<Args>(args)...);
+    return ref<T>(p);
+}
+
+}
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index f136a13248ba..c9620e2bf32a 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -72,7 +72,17 @@ void FdSink::write(const unsigned char * data, size_t len)
             warned = true;
         }
     }
-    writeFull(fd, data, len);
+    try {
+        writeFull(fd, data, len);
+    } catch (SysError & e) {
+        _good = true;
+    }
+}
+
+
+bool FdSink::good()
+{
+    return _good;
 }
 
 
@@ -119,12 +129,18 @@ size_t FdSource::readUnbuffered(unsigned char * data, size_t len)
         checkInterrupt();
         n = ::read(fd, (char *) data, bufSize);
     } while (n == -1 && errno == EINTR);
-    if (n == -1) throw SysError("reading from file");
-    if (n == 0) throw EndOfFile("unexpected end-of-file");
+    if (n == -1) { _good = false; throw SysError("reading from file"); }
+    if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); }
     return n;
 }
 
 
+bool FdSource::good()
+{
+    return _good;
+}
+
+
 size_t StringSource::read(unsigned char * data, size_t len)
 {
     if (pos == s.size()) throw EndOfFile("end of string reached");
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 979ff849fcaf..9e269f3923ea 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -12,6 +12,7 @@ struct Sink
 {
     virtual ~Sink() { }
     virtual void operator () (const unsigned char * data, size_t len) = 0;
+    virtual bool good() { return true; }
 };
 
 
@@ -25,7 +26,7 @@ struct BufferedSink : Sink
         : bufSize(bufSize), bufPos(0), buffer(0) { }
     ~BufferedSink();
 
-    void operator () (const unsigned char * data, size_t len);
+    void operator () (const unsigned char * data, size_t len) override;
 
     void flush();
 
@@ -47,6 +48,8 @@ struct Source
        return the number of bytes stored.  If blocks until at least
        one byte is available. */
     virtual size_t read(unsigned char * data, size_t len) = 0;
+
+    virtual bool good() { return true; }
 };
 
 
@@ -60,7 +63,7 @@ struct BufferedSource : Source
         : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { }
     ~BufferedSource();
 
-    size_t read(unsigned char * data, size_t len);
+    size_t read(unsigned char * data, size_t len) override;
 
     /* Underlying read call, to be overridden. */
     virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0;
@@ -80,7 +83,12 @@ struct FdSink : BufferedSink
     FdSink(int fd) : fd(fd), warn(false), written(0) { }
     ~FdSink();
 
-    void write(const unsigned char * data, size_t len);
+    void write(const unsigned char * data, size_t len) override;
+
+    bool good() override;
+
+private:
+    bool _good = true;
 };
 
 
@@ -90,7 +98,10 @@ struct FdSource : BufferedSource
     int fd;
     FdSource() : fd(-1) { }
     FdSource(int fd) : fd(fd) { }
-    size_t readUnbuffered(unsigned char * data, size_t len);
+    size_t readUnbuffered(unsigned char * data, size_t len) override;
+    bool good() override;
+private:
+    bool _good = true;
 };
 
 
@@ -98,7 +109,7 @@ struct FdSource : BufferedSource
 struct StringSink : Sink
 {
     string s;
-    void operator () (const unsigned char * data, size_t len);
+    void operator () (const unsigned char * data, size_t len) override;
 };
 
 
@@ -108,7 +119,7 @@ struct StringSource : Source
     const string & s;
     size_t pos;
     StringSource(const string & _s) : s(_s), pos(0) { }
-    size_t read(unsigned char * data, size_t len);
+    size_t read(unsigned char * data, size_t len) override;
 };
 
 
diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh
new file mode 100644
index 000000000000..c99c098ac9c6
--- /dev/null
+++ b/src/libutil/sync.hh
@@ -0,0 +1,78 @@
+#pragma once
+
+#include <mutex>
+#include <condition_variable>
+#include <cassert>
+
+namespace nix {
+
+/* This template class ensures synchronized access to a value of type
+   T. It is used as follows:
+
+     struct Data { int x; ... };
+
+     Sync<Data> data;
+
+     {
+       auto data_(data.lock());
+       data_->x = 123;
+     }
+
+   Here, "data" is automatically unlocked when "data_" goes out of
+   scope.
+*/
+
+template<class T>
+class Sync
+{
+private:
+    std::mutex mutex;
+    T data;
+
+public:
+
+    Sync() { }
+    Sync(const T & data) : data(data) { }
+
+    class Lock
+    {
+    private:
+        Sync * s;
+        std::unique_lock<std::mutex> lk;
+        friend Sync;
+        Lock(Sync * s) : s(s), lk(s->mutex) { }
+    public:
+        Lock(Lock && l) : s(l.s) { abort(); }
+        Lock(const Lock & l) = delete;
+        ~Lock() { }
+        T * operator -> () { return &s->data; }
+        T & operator * () { return s->data; }
+
+        void wait(std::condition_variable & cv)
+        {
+            assert(s);
+            cv.wait(lk);
+        }
+
+        template<class Rep, class Period, class Predicate>
+        bool wait_for(std::condition_variable & cv,
+            const std::chrono::duration<Rep, Period> & duration,
+            Predicate pred)
+        {
+            assert(s);
+            return cv.wait_for(lk, duration, pred);
+        }
+
+        template<class Clock, class Duration>
+        std::cv_status wait_until(std::condition_variable & cv,
+            const std::chrono::time_point<Clock, Duration> & duration)
+        {
+            assert(s);
+            return cv.wait_until(lk, duration);
+        }
+    };
+
+    Lock lock() { return Lock(this); }
+};
+
+}
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 23eb5251209e..33aaf5fc9c4d 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -2,6 +2,8 @@
 
 #include "config.h"
 
+#include "ref.hh"
+
 #include <string>
 #include <list>
 #include <set>
@@ -97,70 +99,4 @@ typedef enum {
 } Verbosity;
 
 
-/* A simple non-nullable reference-counted pointer. Actually a wrapper
-   around std::shared_ptr that prevents non-null constructions. */
-template<typename T>
-class ref
-{
-private:
-
-    std::shared_ptr<T> p;
-
-public:
-
-    ref<T>(const ref<T> & r)
-        : p(r.p)
-    { }
-
-    explicit ref<T>(const std::shared_ptr<T> & p)
-        : p(p)
-    {
-        if (!p)
-            throw std::invalid_argument("null pointer cast to ref");
-    }
-
-    T* operator ->() const
-    {
-        return &*p;
-    }
-
-    T& operator *() const
-    {
-        return *p;
-    }
-
-    operator std::shared_ptr<T> ()
-    {
-        return p;
-    }
-
-private:
-
-    template<typename T2, typename... Args>
-    friend ref<T2>
-    make_ref(Args&&... args);
-
-    template<typename T2, typename T3, typename... Args>
-    friend ref<T2>
-    make_ref(Args&&... args);
-
-};
-
-template<typename T, typename... Args>
-inline ref<T>
-make_ref(Args&&... args)
-{
-    auto p = std::make_shared<T>(std::forward<Args>(args)...);
-    return ref<T>(p);
-}
-
-template<typename T, typename T2, typename... Args>
-inline ref<T>
-make_ref(Args&&... args)
-{
-    auto p = std::make_shared<T2>(std::forward<Args>(args)...);
-    return ref<T>(p);
-}
-
-
 }
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index def0525abc18..3becbbabc2bf 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -320,9 +320,11 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
 {
     checkInterrupt();
 
-    printMsg(lvlVomit, format("%1%") % path);
-
-    struct stat st = lstat(path);
+    struct stat st;
+    if (lstat(path.c_str(), &st) == -1) {
+        if (errno == ENOENT) return;
+        throw SysError(format("getting status of ‘%1%’") % path);
+    }
 
     if (!S_ISDIR(st.st_mode) && st.st_nlink == 1)
         bytesFreed += st.st_blocks * 512;
@@ -338,8 +340,10 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
             _deletePath(path + "/" + i.name, bytesFreed);
     }
 
-    if (remove(path.c_str()) == -1)
+    if (remove(path.c_str()) == -1) {
+        if (errno == ENOENT) return;
         throw SysError(format("cannot unlink ‘%1%’") % path);
+    }
 }
 
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index b714cdc64a5a..3606f6ec9eb2 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -92,8 +92,8 @@ string readLine(int fd);
 void writeLine(int fd, string s);
 
 /* Delete a path; i.e., in the case of a directory, it is deleted
-   recursively.  Don't use this at home, kids.  The second variant
-   returns the number of bytes and blocks freed. */
+   recursively. It's not an error if the path does not exist. The
+   second variant returns the number of bytes and blocks freed. */
 void deletePath(const Path & path);
 
 void deletePath(const Path & path, unsigned long long & bytesFreed);
@@ -366,6 +366,14 @@ template<class N> bool string2Int(const string & s, N & n)
     return str && str.get() == EOF;
 }
 
+/* Parse a string into a float. */
+template<class N> bool string2Float(const string & s, N & n)
+{
+    std::istringstream str(s);
+    str >> n;
+    return str && str.get() == EOF;
+}
+
 
 /* Return true iff `s' ends in `suffix'. */
 bool hasSuffix(const string & s, const string & suffix);