* Sync with the trunk.

4 files changed, 226 insertions, 70 deletions

diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index b9e7846992f6..bbfe7847fd8a 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -204,6 +204,22 @@ Hash parseHash32(HashType ht, const string & s)
 }
 
 
+Hash parseHash16or32(HashType ht, const string & s)
+{
+    Hash hash(ht);
+    if (s.size() == hash.hashSize * 2)
+        /* hexadecimal representation */
+        hash = parseHash(ht, s);
+    else if (s.size() == hashLength32(hash))
+        /* base-32 representation */
+        hash = parseHash32(ht, s);
+    else
+        throw Error(format("hash `%1%' has wrong length for hash type `%2%'")
+            % s % printHashType(ht));
+    return hash;
+}
+
+
 bool isHash(const string & s)
 {
     if (s.length() != 32) return false;
@@ -290,21 +306,13 @@ HashSink::HashSink(HashType ht) : ht(ht)
     start(ht, *ctx);
 }
     
-HashSink::HashSink(const HashSink & h)
-{
-    ht = h.ht;
-    bytes = h.bytes;
-    ctx = new Ctx;
-    *ctx = *h.ctx;
-}
-    
 HashSink::~HashSink()
 {
+    bufPos = 0;
     delete ctx;
 }
 
-void HashSink::operator ()
-    (const unsigned char * data, unsigned int len)
+void HashSink::write(const unsigned char * data, size_t len)
 {
     bytes += len;
     update(ht, *ctx, data, len);
@@ -312,11 +320,21 @@ void HashSink::operator ()
 
 HashResult HashSink::finish()
 {
+    flush();
     Hash hash(ht);
     nix::finish(ht, *ctx, hash.hash);
     return HashResult(hash, bytes);
 }
 
+HashResult HashSink::currentHash()
+{
+    flush();
+    Ctx ctx2 = *ctx;
+    Hash hash(ht);
+    nix::finish(ht, ctx2, hash.hash);
+    return HashResult(hash, bytes);
+}
+
 
 HashResult hashPath(
     HashType ht, const Path & path, PathFilter & filter)
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 13740954d625..e0b6478cc418 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -58,6 +58,9 @@ string printHash32(const Hash & hash);
 /* Parse a base-32 representation of a hash code. */
 Hash parseHash32(HashType ht, const string & s);
 
+/* Parse a base-16 or base-32 representation of a hash code. */
+Hash parseHash16or32(HashType ht, const string & s);
+
 /* Verify that the given string is a valid hash code. */
 bool isHash(const string & s);
 
@@ -88,7 +91,7 @@ string printHashType(HashType ht);
 
 union Ctx;
 
-class HashSink : public Sink
+class HashSink : public BufferedSink
 {
 private:
     HashType ht;
@@ -99,8 +102,9 @@ public:
     HashSink(HashType ht);
     HashSink(const HashSink & h);
     ~HashSink();
-    virtual void operator () (const unsigned char * data, unsigned int len);
+    void write(const unsigned char * data, size_t len);
     HashResult finish();
+    HashResult currentHash();
 };
 
 
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 9b422271323f..c4563ffd1212 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -2,24 +2,117 @@
 #include "util.hh"
 
 #include <cstring>
+#include <cerrno>
 
 
 namespace nix {
 
 
-void FdSink::operator () (const unsigned char * data, unsigned int len)
+BufferedSink::~BufferedSink()
+{
+    /* We can't call flush() here, because C++ for some insane reason
+       doesn't allow you to call virtual methods from a destructor. */
+    assert(!bufPos);
+    if (buffer) delete[] buffer;
+}
+
+    
+void BufferedSink::operator () (const unsigned char * data, size_t len)
+{
+    if (!buffer) buffer = new unsigned char[bufSize];
+    
+    while (len) {
+        /* Optimisation: bypass the buffer if the data exceeds the
+           buffer size. */
+        if (bufPos + len >= bufSize) {
+            flush();
+            write(data, len);
+            break;
+        }
+        /* Otherwise, copy the bytes to the buffer.  Flush the buffer
+           when it's full. */
+        size_t n = bufPos + len > bufSize ? bufSize - bufPos : len;
+        memcpy(buffer + bufPos, data, n);
+        data += n; bufPos += n; len -= n;
+        if (bufPos == bufSize) flush();
+    }
+}
+
+
+void BufferedSink::flush()
+{
+    if (bufPos == 0) return;
+    size_t n = bufPos;
+    bufPos = 0; // don't trigger the assert() in ~BufferedSink()
+    write(buffer, n);
+}
+
+
+FdSink::~FdSink()
+{
+    try { flush(); } catch (...) { ignoreException(); }
+}
+
+
+void FdSink::write(const unsigned char * data, size_t len)
 {
     writeFull(fd, data, len);
 }
 
 
-void FdSource::operator () (unsigned char * data, unsigned int len)
+void Source::operator () (unsigned char * data, size_t len)
+{
+    while (len) {
+        size_t n = read(data, len);
+        data += n; len -= n;
+    }
+}
+
+
+BufferedSource::~BufferedSource()
 {
-    readFull(fd, data, len);
+    if (buffer) delete[] buffer;
 }
 
 
-void writePadding(unsigned int len, Sink & sink)
+size_t BufferedSource::read(unsigned char * data, size_t len)
+{
+    if (!buffer) buffer = new unsigned char[bufSize];
+
+    if (!bufPosIn) bufPosIn = readUnbuffered(buffer, bufSize);
+            
+    /* Copy out the data in the buffer. */
+    size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len;
+    memcpy(data, buffer + bufPosOut, n);
+    bufPosOut += n;
+    if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0;
+    return n;
+}
+
+
+size_t FdSource::readUnbuffered(unsigned char * data, size_t len)
+{
+    ssize_t n;
+    do {
+        checkInterrupt();
+        n = ::read(fd, (char *) data, bufSize);
+    } while (n == -1 && errno == EINTR);
+    if (n == -1) throw SysError("reading from file");
+    if (n == 0) throw EndOfFile("unexpected end-of-file");
+    return n;
+}
+
+
+size_t StringSource::read(unsigned char * data, size_t len)
+{
+    if (pos == s.size()) throw EndOfFile("end of string reached");
+    size_t n = s.copy((char *) data, len, pos);
+    pos += n;
+    return n;
+}
+
+
+void writePadding(size_t len, Sink & sink)
 {
     if (len % 8) {
         unsigned char zero[8];
@@ -56,28 +149,36 @@ void writeLongLong(unsigned long long n, Sink & sink)
 }
 
 
-void writeString(const string & s, Sink & sink)
+void writeString(const unsigned char * buf, size_t len, Sink & sink)
 {
-    unsigned int len = s.length();
     writeInt(len, sink);
-    sink((const unsigned char *) s.c_str(), len);
+    sink(buf, len);
     writePadding(len, sink);
 }
 
 
-void writeStringSet(const StringSet & ss, Sink & sink)
+void writeString(const string & s, Sink & sink)
+{
+    writeString((const unsigned char *) s.c_str(), s.size(), sink);
+}
+
+
+template<class T> void writeStrings(const T & ss, Sink & sink)
 {
     writeInt(ss.size(), sink);
-    for (StringSet::iterator i = ss.begin(); i != ss.end(); ++i)
+    foreach (typename T::const_iterator, i, ss)
         writeString(*i, sink);
 }
 
+template void writeStrings(const Paths & ss, Sink & sink);
+template void writeStrings(const PathSet & ss, Sink & sink);
 
-void readPadding(unsigned int len, Source & source)
+
+void readPadding(size_t len, Source & source)
 {
     if (len % 8) {
         unsigned char zero[8];
-        unsigned int n = 8 - (len % 8);
+        size_t n = 8 - (len % 8);
         source(zero, n);
         for (unsigned int i = 0; i < n; i++)
             if (zero[i]) throw SerialisationError("non-zero padding");
@@ -115,9 +216,19 @@ unsigned long long readLongLong(Source & source)
 }
 
 
+size_t readString(unsigned char * buf, size_t max, Source & source)
+{
+    size_t len = readInt(source);
+    if (len > max) throw Error("string is too long");
+    source(buf, len);
+    readPadding(len, source);
+    return len;
+}
+
+ 
 string readString(Source & source)
 {
-    unsigned int len = readInt(source);
+    size_t len = readInt(source);
     unsigned char * buf = new unsigned char[len];
     AutoDeleteArray<unsigned char> d(buf);
     source(buf, len);
@@ -126,14 +237,17 @@ string readString(Source & source)
 }
 
  
-StringSet readStringSet(Source & source)
+template<class T> T readStrings(Source & source)
 {
     unsigned int count = readInt(source);
-    StringSet ss;
+    T ss;
     while (count--)
-        ss.insert(readString(source));
+        ss.insert(ss.end(), readString(source));
     return ss;
 }
 
+template Paths readStrings(Source & source);
+template PathSet readStrings(Source & source);
+
 
 }
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 0e797d63bca9..ded4b12a046e 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -11,7 +11,25 @@ namespace nix {
 struct Sink 
 {
     virtual ~Sink() { }
-    virtual void operator () (const unsigned char * data, unsigned int len) = 0;
+    virtual void operator () (const unsigned char * data, size_t len) = 0;
+};
+
+
+/* A buffered abstract sink. */
+struct BufferedSink : Sink
+{
+    size_t bufSize, bufPos;
+    unsigned char * buffer;
+
+    BufferedSink(size_t bufSize = 32 * 1024)
+        : bufSize(bufSize), bufPos(0), buffer(0) { }
+    ~BufferedSink();
+
+    void operator () (const unsigned char * data, size_t len);
+    
+    void flush();
+    
+    virtual void write(const unsigned char * data, size_t len) = 0;
 };
 
 
@@ -20,49 +38,55 @@ struct Source
 {
     virtual ~Source() { }
     
-    /* The callee should store exactly *len bytes in the buffer
-       pointed to by data.  It should block if that much data is not
-       yet available, or throw an error if it is not going to be
-       available. */
-    virtual void operator () (unsigned char * data, unsigned int len) = 0;
+    /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’.
+       It blocks until all the requested data is available, or throws
+       an error if it is not going to be available.   */
+    void operator () (unsigned char * data, size_t len);
+
+    /* Store up to ‘len’ in the buffer pointed to by ‘data’, and
+       return the number of bytes stored.  If blocks until at least
+       one byte is available. */
+    virtual size_t read(unsigned char * data, size_t len) = 0;
 };
 
 
-/* A sink that writes data to a file descriptor. */
-struct FdSink : Sink
+/* A buffered abstract source. */
+struct BufferedSource : Source
 {
-    int fd;
+    size_t bufSize, bufPosIn, bufPosOut;
+    unsigned char * buffer;
 
-    FdSink()
-    {
-        fd = -1;
-    }
+    BufferedSource(size_t bufSize = 32 * 1024)
+        : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { }
+    ~BufferedSource();
     
-    FdSink(int fd) 
-    {
-        this->fd = fd;
-    }
+    size_t read(unsigned char * data, size_t len);
     
-    void operator () (const unsigned char * data, unsigned int len);
+    /* Underlying read call, to be overriden. */
+    virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0;
 };
 
 
-/* A source that reads data from a file descriptor. */
-struct FdSource : Source
+/* A sink that writes data to a file descriptor. */
+struct FdSink : BufferedSink
 {
     int fd;
 
-    FdSource()
-    {
-        fd = -1;
-    }
+    FdSink() : fd(-1) { }
+    FdSink(int fd) : fd(fd) { }
+    ~FdSink();
     
-    FdSource(int fd) 
-    {
-        this->fd = fd;
-    }
-    
-    void operator () (unsigned char * data, unsigned int len);
+    void write(const unsigned char * data, size_t len);
+};
+
+
+/* A source that reads data from a file descriptor. */
+struct FdSource : BufferedSource
+{
+    int fd;
+    FdSource() : fd(-1) { }
+    FdSource(int fd) : fd(fd) { }
+    size_t readUnbuffered(unsigned char * data, size_t len);
 };
 
 
@@ -70,7 +94,7 @@ struct FdSource : Source
 struct StringSink : Sink
 {
     string s;
-    virtual void operator () (const unsigned char * data, unsigned int len)
+    void operator () (const unsigned char * data, size_t len)
     {
         s.append((const char *) data, len);
     }
@@ -81,29 +105,25 @@ struct StringSink : Sink
 struct StringSource : Source
 {
     const string & s;
-    unsigned int pos;
+    size_t pos;
     StringSource(const string & _s) : s(_s), pos(0) { }
-    virtual void operator () (unsigned char * data, unsigned int len)
-    {
-        s.copy((char *) data, len, pos);
-        pos += len;
-        if (pos > s.size())
-            throw Error("end of string reached");
-    }
+    size_t read(unsigned char * data, size_t len);    
 };
 
 
-void writePadding(unsigned int len, Sink & sink);
+void writePadding(size_t len, Sink & sink);
 void writeInt(unsigned int n, Sink & sink);
 void writeLongLong(unsigned long long n, Sink & sink);
+void writeString(const unsigned char * buf, size_t len, Sink & sink);
 void writeString(const string & s, Sink & sink);
-void writeStringSet(const StringSet & ss, Sink & sink);
+template<class T> void writeStrings(const T & ss, Sink & sink);
 
-void readPadding(unsigned int len, Source & source);
+void readPadding(size_t len, Source & source);
 unsigned int readInt(Source & source);
 unsigned long long readLongLong(Source & source);
+size_t readString(unsigned char * buf, size_t max, Source & source);
 string readString(Source & source);
-StringSet readStringSet(Source & source);
+template<class T> T readStrings(Source & source);
 
 
 MakeError(SerialisationError, Error)