From 49f59dceca37636353cf2f5f60135d7705ea154e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 2 Dec 2011 11:47:06 +0000 Subject: * Move parseHash16or32 into libutil, and use in nix-hash. --- src/libutil/hash.cc | 16 ++++++++++++++++ src/libutil/hash.hh | 3 +++ 2 files changed, 19 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index b9e7846992f6..53342344107d 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -204,6 +204,22 @@ Hash parseHash32(HashType ht, const string & s) } +Hash parseHash16or32(HashType ht, const string & s) +{ + Hash hash(ht); + if (s.size() == hash.hashSize * 2) + /* hexadecimal representation */ + hash = parseHash(ht, s); + else if (s.size() == hashLength32(hash)) + /* base-32 representation */ + hash = parseHash32(ht, s); + else + throw Error(format("hash `%1%' has wrong length for hash type `%2%'") + % s % printHashType(ht)); + return hash; +} + + bool isHash(const string & s) { if (s.length() != 32) return false; diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 13740954d625..cbdcf4c8d4ff 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -58,6 +58,9 @@ string printHash32(const Hash & hash); /* Parse a base-32 representation of a hash code. */ Hash parseHash32(HashType ht, const string & s); +/* Parse a base-16 or base-32 representation of a hash code. */ +Hash parseHash16or32(HashType ht, const string & s); + /* Verify that the given string is a valid hash code. */ bool isHash(const string & s); -- cgit 1.4.1 From 3a48282b0681d68147e18f7464eaddf1d188c3be Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 Dec 2011 23:30:06 +0000 Subject: * Buffer writes in FdSink. This significantly reduces the number of system calls / context switches when dumping a NAR and in the worker protocol. --- src/libstore/remote-store.cc | 4 ++++ src/libutil/serialise.cc | 25 ++++++++++++++++++++++++- src/libutil/serialise.hh | 19 +++++++++++++------ src/nix-worker/nix-worker.cc | 16 +++++++++------- 4 files changed, 50 insertions(+), 14 deletions(-) (limited to 'src/libutil') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 0c6a1c37d1ec..8269b6a83168 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -65,6 +65,7 @@ void RemoteStore::openConnection() /* Send the magic greeting, check for the reply. */ try { writeInt(WORKER_MAGIC_1, to); + to.flush(); unsigned int magic = readInt(from); if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); @@ -166,6 +167,7 @@ void RemoteStore::connectToDaemon() RemoteStore::~RemoteStore() { try { + to.flush(); fdSocket.close(); if (child != -1) child.wait(true); @@ -488,6 +490,7 @@ void RemoteStore::clearFailedPaths(const PathSet & paths) void RemoteStore::processStderr(Sink * sink, Source * source) { + to.flush(); unsigned int msg; while ((msg = readInt(from)) == STDERR_NEXT || msg == STDERR_READ || msg == STDERR_WRITE) { @@ -503,6 +506,7 @@ void RemoteStore::processStderr(Sink * sink, Source * source) AutoDeleteArray d(buf); (*source)(buf, len); writeString(string((const char *) buf, len), to); + to.flush(); } else { string s = readString(from); diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 9b422271323f..66a64a6be4c8 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -9,7 +9,30 @@ namespace nix { void FdSink::operator () (const unsigned char * data, unsigned int len) { - writeFull(fd, data, len); + if (!buffer) buffer = new unsigned char[bufSize]; + + while (len) { + /* Optimisation: bypass the buffer if the data exceeds the + buffer size and there is no unflushed data. */ + if (bufPos == 0 && len >= bufSize) { + writeFull(fd, data, len); + break; + } + /* Otherwise, copy the bytes to the buffer. Flush the buffer + when it's full. */ + size_t n = bufPos + len > bufSize ? bufSize - bufPos : len; + memcpy(buffer + bufPos, data, n); + data += n; bufPos += n; len -= n; + if (bufPos == bufSize) flush(); + } +} + + +void FdSink::flush() +{ + if (fd == -1 || bufPos == 0) return; + writeFull(fd, buffer, bufPos); + bufPos = 0; } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 0e797d63bca9..711bd5e6c7ce 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -28,22 +28,29 @@ struct Source }; -/* A sink that writes data to a file descriptor. */ +/* A sink that writes data to a file descriptor (using a buffer). */ struct FdSink : Sink { int fd; + unsigned int bufSize, bufPos; + unsigned char * buffer; - FdSink() + FdSink() : fd(-1), bufSize(32 * 1024), bufPos(0), buffer(0) { } + + FdSink(int fd, unsigned int bufSize = 32 * 1024) + : fd(fd), bufSize(bufSize), bufPos(0), buffer(0) { - fd = -1; } - - FdSink(int fd) + + ~FdSink() { - this->fd = fd; + flush(); + if (buffer) delete[] buffer; } void operator () (const unsigned char * data, unsigned int len); + + void flush(); }; diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 8950f73ef25f..6c222420e0ce 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -57,6 +57,7 @@ static void tunnelStderr(const unsigned char * buf, size_t count) try { writeInt(STDERR_NEXT, to); writeString(string((char *) buf, count), to); + to.flush(); } catch (...) { /* Write failed; that means that the other side is gone. */ @@ -200,9 +201,7 @@ static void stopWork(bool success = true, const string & msg = "", unsigned int struct TunnelSink : Sink { Sink & to; - TunnelSink(Sink & to) : to(to) - { - } + TunnelSink(Sink & to) : to(to) { } virtual void operator () (const unsigned char * data, unsigned int len) { @@ -215,9 +214,7 @@ struct TunnelSink : Sink struct TunnelSource : Source { Source & from; - TunnelSource(Source & from) : from(from) - { - } + TunnelSource(Source & from) : from(from) { } virtual void operator () (unsigned char * data, unsigned int len) { @@ -228,6 +225,7 @@ struct TunnelSource : Source writeInt(STDERR_READ, to); writeInt(len, to); + to.flush(); string s = readString(from); if (s.size() != len) throw Error("not enough data"); memcpy(data, (const unsigned char *) s.c_str(), len); @@ -596,8 +594,8 @@ static void processConnection() unsigned int magic = readInt(from); if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); writeInt(WORKER_MAGIC_2, to); - writeInt(PROTOCOL_VERSION, to); + to.flush(); unsigned int clientVersion = readInt(from); /* Send startup error messages to the client. */ @@ -619,9 +617,11 @@ static void processConnection() store = boost::shared_ptr(new LocalStore()); stopWork(); + to.flush(); } catch (Error & e) { stopWork(false, e.msg()); + to.flush(); return; } @@ -652,6 +652,8 @@ static void processConnection() if (!errorAllowed) break; } + to.flush(); + assert(!canSendStderr); }; -- cgit 1.4.1 From a3e0656cbbfadba28518e0a29c324edaabb9874a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 15 Dec 2011 12:32:08 +0000 Subject: * Buffer reads in FdSource. Together with write buffering, this significantly cuts down the number of syscalls (e.g., for "nix-store -qR /var/run/current-system" via the daemon, it reduced the number of syscalls in the client from 29134 to 4766 and in the daemon from 44266 to 20666). --- src/libutil/serialise.cc | 24 +++++++++++++++++++++++- src/libutil/serialise.hh | 18 +++++++++--------- 2 files changed, 32 insertions(+), 10 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 66a64a6be4c8..e3a53c0d01dc 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -2,6 +2,7 @@ #include "util.hh" #include +#include namespace nix { @@ -38,7 +39,28 @@ void FdSink::flush() void FdSource::operator () (unsigned char * data, unsigned int len) { - readFull(fd, data, len); + if (!buffer) buffer = new unsigned char[bufSize]; + + while (len) { + if (!bufPosIn) { + /* Read as much data as is available (up to the buffer + size). */ + checkInterrupt(); + ssize_t n = read(fd, (char *) buffer, bufSize); + if (n == -1) { + if (errno == EINTR) continue; + throw SysError("reading from file"); + } + if (n == 0) throw EndOfFile("unexpected end-of-file"); + bufPosIn = n; + } + + /* Copy out the data in the buffer. */ + size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; + memcpy(data, buffer + bufPosOut, n); + data += n; bufPosOut += n; len -= n; + if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0; + } } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 711bd5e6c7ce..b8d4d7a849d2 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -38,9 +38,7 @@ struct FdSink : Sink FdSink() : fd(-1), bufSize(32 * 1024), bufPos(0), buffer(0) { } FdSink(int fd, unsigned int bufSize = 32 * 1024) - : fd(fd), bufSize(bufSize), bufPos(0), buffer(0) - { - } + : fd(fd), bufSize(bufSize), bufPos(0), buffer(0) { } ~FdSink() { @@ -58,15 +56,17 @@ struct FdSink : Sink struct FdSource : Source { int fd; + unsigned int bufSize, bufPosIn, bufPosOut; + unsigned char * buffer; - FdSource() - { - fd = -1; - } + FdSource() : fd(-1), bufSize(32 * 1024), bufPosIn(0), bufPosOut(0), buffer(0) { } + + FdSource(int fd, unsigned int bufSize = 32 * 1024) + : fd(fd), bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { } - FdSource(int fd) + ~FdSource() { - this->fd = fd; + if (buffer) delete[] buffer; } void operator () (unsigned char * data, unsigned int len); -- cgit 1.4.1 From 5a1b9ed0aa3a0c240b667dbe504b61b2b68e4d16 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 15 Dec 2011 16:19:53 +0000 Subject: * Refactoring: move sink/source buffering into separate classes. * Buffer the HashSink. This speeds up hashing a bit because it prevents lots of calls to the hash update functions (e.g. nix-hash went from 9.3s to 8.7s of user time on the closure of my /var/run/current-system). --- src/libstore/local-store.cc | 9 ++--- src/libstore/references.cc | 4 +- src/libstore/remote-store.cc | 2 +- src/libutil/hash.cc | 22 +++++----- src/libutil/hash.hh | 5 ++- src/libutil/serialise.cc | 69 ++++++++++++++++++++----------- src/libutil/serialise.hh | 96 +++++++++++++++++++++++++------------------- src/nix-worker/nix-worker.cc | 8 ++-- 8 files changed, 125 insertions(+), 90 deletions(-) (limited to 'src/libutil') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5107a93de299..525e5fc7bbe3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1103,16 +1103,14 @@ struct HashAndWriteSink : Sink HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) { } - virtual void operator () - (const unsigned char * data, unsigned int len) + virtual void operator () (const unsigned char * data, size_t len) { writeSink(data, len); hashSink(data, len); } Hash currentHash() { - HashSink hashSinkClone(hashSink); - return hashSinkClone.finish().first; + return hashSink.currentHash().first; } }; @@ -1201,8 +1199,7 @@ struct HashAndReadSource : Source { hashing = true; } - virtual void operator () - (unsigned char * data, unsigned int len) + virtual void operator () (unsigned char * data, size_t len) { readSource(data, len); if (hashing) hashSink(data, len); diff --git a/src/libstore/references.cc b/src/libstore/references.cc index ade9c9aa20e3..c1f9e3ba7711 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -57,11 +57,11 @@ struct RefScanSink : Sink RefScanSink() : hashSink(htSHA256) { } - void operator () (const unsigned char * data, unsigned int len); + void operator () (const unsigned char * data, size_t len); }; -void RefScanSink::operator () (const unsigned char * data, unsigned int len) +void RefScanSink::operator () (const unsigned char * data, size_t len) { hashSink(data, len); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8269b6a83168..7bf0ad7bd4c7 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -374,7 +374,7 @@ Path RemoteStore::importPath(bool requireSignature, Source & source) openConnection(); writeInt(wopImportPath, to); /* We ignore requireSignature, since the worker forces it to true - anyway. */ + anyway. */ processStderr(0, &source); return readStorePath(from); } diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 53342344107d..bbfe7847fd8a 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -306,21 +306,13 @@ HashSink::HashSink(HashType ht) : ht(ht) start(ht, *ctx); } -HashSink::HashSink(const HashSink & h) -{ - ht = h.ht; - bytes = h.bytes; - ctx = new Ctx; - *ctx = *h.ctx; -} - HashSink::~HashSink() { + bufPos = 0; delete ctx; } -void HashSink::operator () - (const unsigned char * data, unsigned int len) +void HashSink::write(const unsigned char * data, size_t len) { bytes += len; update(ht, *ctx, data, len); @@ -328,11 +320,21 @@ void HashSink::operator () HashResult HashSink::finish() { + flush(); Hash hash(ht); nix::finish(ht, *ctx, hash.hash); return HashResult(hash, bytes); } +HashResult HashSink::currentHash() +{ + flush(); + Ctx ctx2 = *ctx; + Hash hash(ht); + nix::finish(ht, ctx2, hash.hash); + return HashResult(hash, bytes); +} + HashResult hashPath( HashType ht, const Path & path, PathFilter & filter) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index cbdcf4c8d4ff..e0b6478cc418 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -91,7 +91,7 @@ string printHashType(HashType ht); union Ctx; -class HashSink : public Sink +class HashSink : public BufferedSink { private: HashType ht; @@ -102,8 +102,9 @@ public: HashSink(HashType ht); HashSink(const HashSink & h); ~HashSink(); - virtual void operator () (const unsigned char * data, unsigned int len); + void write(const unsigned char * data, size_t len); HashResult finish(); + HashResult currentHash(); }; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index e3a53c0d01dc..a82262704ea3 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -8,7 +8,16 @@ namespace nix { -void FdSink::operator () (const unsigned char * data, unsigned int len) +BufferedSink::~BufferedSink() +{ + /* We can't call flush() here, because C++ for some insane reason + doesn't allow you to call virtual methods from a destructor. */ + assert(!bufPos); + if (buffer) delete[] buffer; +} + + +void BufferedSink::operator () (const unsigned char * data, size_t len) { if (!buffer) buffer = new unsigned char[bufSize]; @@ -16,7 +25,7 @@ void FdSink::operator () (const unsigned char * data, unsigned int len) /* Optimisation: bypass the buffer if the data exceeds the buffer size and there is no unflushed data. */ if (bufPos == 0 && len >= bufSize) { - writeFull(fd, data, len); + write(data, len); break; } /* Otherwise, copy the bytes to the buffer. Flush the buffer @@ -29,31 +38,32 @@ void FdSink::operator () (const unsigned char * data, unsigned int len) } -void FdSink::flush() +void BufferedSink::flush() { - if (fd == -1 || bufPos == 0) return; - writeFull(fd, buffer, bufPos); + if (bufPos == 0) return; + write(buffer, bufPos); bufPos = 0; } -void FdSource::operator () (unsigned char * data, unsigned int len) +void FdSink::write(const unsigned char * data, size_t len) +{ + writeFull(fd, data, len); +} + + +BufferedSource::~BufferedSource() +{ + if (buffer) delete[] buffer; +} + + +void BufferedSource::operator () (unsigned char * data, size_t len) { if (!buffer) buffer = new unsigned char[bufSize]; while (len) { - if (!bufPosIn) { - /* Read as much data as is available (up to the buffer - size). */ - checkInterrupt(); - ssize_t n = read(fd, (char *) buffer, bufSize); - if (n == -1) { - if (errno == EINTR) continue; - throw SysError("reading from file"); - } - if (n == 0) throw EndOfFile("unexpected end-of-file"); - bufPosIn = n; - } + if (!bufPosIn) bufPosIn = read(buffer, bufSize); /* Copy out the data in the buffer. */ size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; @@ -64,7 +74,20 @@ void FdSource::operator () (unsigned char * data, unsigned int len) } -void writePadding(unsigned int len, Sink & sink) +size_t FdSource::read(unsigned char * data, size_t len) +{ + ssize_t n; + do { + checkInterrupt(); + n = ::read(fd, (char *) data, bufSize); + } while (n == -1 && errno == EINTR); + if (n == -1) throw SysError("reading from file"); + if (n == 0) throw EndOfFile("unexpected end-of-file"); + return n; +} + + +void writePadding(size_t len, Sink & sink) { if (len % 8) { unsigned char zero[8]; @@ -103,7 +126,7 @@ void writeLongLong(unsigned long long n, Sink & sink) void writeString(const string & s, Sink & sink) { - unsigned int len = s.length(); + size_t len = s.length(); writeInt(len, sink); sink((const unsigned char *) s.c_str(), len); writePadding(len, sink); @@ -118,11 +141,11 @@ void writeStringSet(const StringSet & ss, Sink & sink) } -void readPadding(unsigned int len, Source & source) +void readPadding(size_t len, Source & source) { if (len % 8) { unsigned char zero[8]; - unsigned int n = 8 - (len % 8); + size_t n = 8 - (len % 8); source(zero, n); for (unsigned int i = 0; i < n; i++) if (zero[i]) throw SerialisationError("non-zero padding"); @@ -162,7 +185,7 @@ unsigned long long readLongLong(Source & source) string readString(Source & source) { - unsigned int len = readInt(source); + size_t len = readInt(source); unsigned char * buf = new unsigned char[len]; AutoDeleteArray d(buf); source(buf, len); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index b8d4d7a849d2..a0588668f037 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -11,7 +11,25 @@ namespace nix { struct Sink { virtual ~Sink() { } - virtual void operator () (const unsigned char * data, unsigned int len) = 0; + virtual void operator () (const unsigned char * data, size_t len) = 0; +}; + + +/* A buffered abstract sink. */ +struct BufferedSink : Sink +{ + size_t bufSize, bufPos; + unsigned char * buffer; + + BufferedSink(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPos(0), buffer(0) { } + ~BufferedSink(); + + void operator () (const unsigned char * data, size_t len); + + void flush(); + + virtual void write(const unsigned char * data, size_t len) = 0; }; @@ -20,56 +38,52 @@ struct Source { virtual ~Source() { } - /* The callee should store exactly *len bytes in the buffer - pointed to by data. It should block if that much data is not - yet available, or throw an error if it is not going to be - available. */ - virtual void operator () (unsigned char * data, unsigned int len) = 0; + /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. + It blocks if that much data is not yet available, or throws an + error if it is not going to be available. */ + virtual void operator () (unsigned char * data, size_t len) = 0; }; -/* A sink that writes data to a file descriptor (using a buffer). */ -struct FdSink : Sink +/* A buffered abstract source. */ +struct BufferedSource : Source { - int fd; - unsigned int bufSize, bufPos; + size_t bufSize, bufPosIn, bufPosOut; unsigned char * buffer; - FdSink() : fd(-1), bufSize(32 * 1024), bufPos(0), buffer(0) { } + BufferedSource(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { } + ~BufferedSource(); - FdSink(int fd, unsigned int bufSize = 32 * 1024) - : fd(fd), bufSize(bufSize), bufPos(0), buffer(0) { } - - ~FdSink() - { - flush(); - if (buffer) delete[] buffer; - } + void operator () (unsigned char * data, size_t len); - void operator () (const unsigned char * data, unsigned int len); - - void flush(); + /* Store up to ‘len’ in the buffer pointed to by ‘data’, and + return the number of bytes stored. If should block until at + least one byte is available. */ + virtual size_t read(unsigned char * data, size_t len) = 0; }; -/* A source that reads data from a file descriptor. */ -struct FdSource : Source +/* A sink that writes data to a file descriptor. */ +struct FdSink : BufferedSink { int fd; - unsigned int bufSize, bufPosIn, bufPosOut; - unsigned char * buffer; - FdSource() : fd(-1), bufSize(32 * 1024), bufPosIn(0), bufPosOut(0), buffer(0) { } - - FdSource(int fd, unsigned int bufSize = 32 * 1024) - : fd(fd), bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { } - - ~FdSource() - { - if (buffer) delete[] buffer; - } + FdSink() : fd(-1) { } + FdSink(int fd) : fd(fd) { } + ~FdSink() { flush(); } - void operator () (unsigned char * data, unsigned int len); + void write(const unsigned char * data, size_t len); +}; + + +/* A source that reads data from a file descriptor. */ +struct FdSource : BufferedSource +{ + int fd; + FdSource() : fd(-1) { } + FdSource(int fd) : fd(fd) { } + size_t read(unsigned char * data, size_t len); }; @@ -77,7 +91,7 @@ struct FdSource : Source struct StringSink : Sink { string s; - virtual void operator () (const unsigned char * data, unsigned int len) + void operator () (const unsigned char * data, size_t len) { s.append((const char *) data, len); } @@ -88,9 +102,9 @@ struct StringSink : Sink struct StringSource : Source { const string & s; - unsigned int pos; + size_t pos; StringSource(const string & _s) : s(_s), pos(0) { } - virtual void operator () (unsigned char * data, unsigned int len) + virtual void operator () (unsigned char * data, size_t len) { s.copy((char *) data, len, pos); pos += len; @@ -100,13 +114,13 @@ struct StringSource : Source }; -void writePadding(unsigned int len, Sink & sink); +void writePadding(size_t len, Sink & sink); void writeInt(unsigned int n, Sink & sink); void writeLongLong(unsigned long long n, Sink & sink); void writeString(const string & s, Sink & sink); void writeStringSet(const StringSet & ss, Sink & sink); -void readPadding(unsigned int len, Source & source); +void readPadding(size_t len, Source & source); unsigned int readInt(Source & source); unsigned long long readLongLong(Source & source); string readString(Source & source); diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 6c222420e0ce..a89852638214 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -202,8 +202,7 @@ struct TunnelSink : Sink { Sink & to; TunnelSink(Sink & to) : to(to) { } - virtual void operator () - (const unsigned char * data, unsigned int len) + virtual void operator () (const unsigned char * data, size_t len) { writeInt(STDERR_WRITE, to); writeString(string((const char *) data, len), to); @@ -215,8 +214,7 @@ struct TunnelSource : Source { Source & from; TunnelSource(Source & from) : from(from) { } - virtual void operator () - (unsigned char * data, unsigned int len) + virtual void operator () (unsigned char * data, size_t len) { /* Careful: we're going to receive data from the client now, so we have to disable the SIGPOLL handler. */ @@ -267,7 +265,7 @@ struct SavingSourceAdapter : Source Source & orig; string s; SavingSourceAdapter(Source & orig) : orig(orig) { } - void operator () (unsigned char * data, unsigned int len) + void operator () (unsigned char * data, size_t len) { orig(data, len); s.append((const char *) data, len); -- cgit 1.4.1 From 78598d06f0240a15b74720d8f987daeb702318d7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 16 Dec 2011 15:45:42 +0000 Subject: * Clean up exception handling. --- src/libutil/serialise.cc | 11 +++++++++-- src/libutil/serialise.hh | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index a82262704ea3..76f2e721a535 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -41,8 +41,15 @@ void BufferedSink::operator () (const unsigned char * data, size_t len) void BufferedSink::flush() { if (bufPos == 0) return; - write(buffer, bufPos); - bufPos = 0; + size_t n = bufPos; + bufPos = 0; // don't trigger the assert() in ~BufferedSink() + write(buffer, n); +} + + +FdSink::~FdSink() +{ + try { flush(); } catch (...) { ignoreException(); } } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index a0588668f037..a155f6681e2b 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -71,7 +71,7 @@ struct FdSink : BufferedSink FdSink() : fd(-1) { } FdSink(int fd) : fd(fd) { } - ~FdSink() { flush(); } + ~FdSink(); void write(const unsigned char * data, size_t len); }; -- cgit 1.4.1 From e0bd307802d13476055f8ba99ab7808de0fd71e5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 16 Dec 2011 19:44:13 +0000 Subject: * Make the import operation through the daemon much more efficient (way fewer roundtrips) by allowing the client to send data in bigger chunks. * Some refactoring. --- src/libstore/local-store.cc | 7 ++++--- src/libstore/remote-store.cc | 6 +++--- src/libstore/worker-protocol.hh | 2 +- src/libutil/serialise.cc | 42 +++++++++++++++++++++++++++++------------ src/libutil/serialise.hh | 31 ++++++++++++++---------------- src/nix-worker/nix-worker.cc | 24 +++++++++++++++-------- 6 files changed, 68 insertions(+), 44 deletions(-) (limited to 'src/libutil') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 525e5fc7bbe3..65b1cdbc87a3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1199,10 +1199,11 @@ struct HashAndReadSource : Source { hashing = true; } - virtual void operator () (unsigned char * data, size_t len) + size_t read(unsigned char * data, size_t len) { - readSource(data, len); - if (hashing) hashSink(data, len); + size_t n = readSource.read(data, len); + if (hashing) hashSink(data, n); + return n; } }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 7bf0ad7bd4c7..e976e8fa57ae 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -501,11 +501,11 @@ void RemoteStore::processStderr(Sink * sink, Source * source) } else if (msg == STDERR_READ) { if (!source) throw Error("no source"); - unsigned int len = readInt(from); + size_t len = readInt(from); unsigned char * buf = new unsigned char[len]; AutoDeleteArray d(buf); - (*source)(buf, len); - writeString(string((const char *) buf, len), to); + size_t n = source->read(buf, len); + writeString(string((const char *) buf, n), to); // !!! inefficient to.flush(); } else { diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index acb8bc8b2948..acabd6ca30d2 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -8,7 +8,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x108 +#define PROTOCOL_VERSION 0x109 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 76f2e721a535..640267a131bf 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -23,8 +23,9 @@ void BufferedSink::operator () (const unsigned char * data, size_t len) while (len) { /* Optimisation: bypass the buffer if the data exceeds the - buffer size and there is no unflushed data. */ - if (bufPos == 0 && len >= bufSize) { + buffer size. */ + if (bufPos + len >= bufSize) { + flush(); write(data, len); break; } @@ -59,29 +60,37 @@ void FdSink::write(const unsigned char * data, size_t len) } +void Source::operator () (unsigned char * data, size_t len) +{ + while (len) { + size_t n = read(data, len); + data += n; len -= n; + } +} + + BufferedSource::~BufferedSource() { if (buffer) delete[] buffer; } -void BufferedSource::operator () (unsigned char * data, size_t len) +size_t BufferedSource::read(unsigned char * data, size_t len) { if (!buffer) buffer = new unsigned char[bufSize]; - while (len) { - if (!bufPosIn) bufPosIn = read(buffer, bufSize); + if (!bufPosIn) bufPosIn = readUnbuffered(buffer, bufSize); - /* Copy out the data in the buffer. */ - size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; - memcpy(data, buffer + bufPosOut, n); - data += n; bufPosOut += n; len -= n; - if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0; - } + /* Copy out the data in the buffer. */ + size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; + memcpy(data, buffer + bufPosOut, n); + bufPosOut += n; + if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0; + return n; } -size_t FdSource::read(unsigned char * data, size_t len) +size_t FdSource::readUnbuffered(unsigned char * data, size_t len) { ssize_t n; do { @@ -94,6 +103,15 @@ size_t FdSource::read(unsigned char * data, size_t len) } +size_t StringSource::read(unsigned char * data, size_t len) +{ + if (pos == s.size()) throw EndOfFile("end of string reached"); + size_t n = s.copy((char *) data, len, pos); + pos += n; + return n; +} + + void writePadding(size_t len, Sink & sink) { if (len % 8) { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index a155f6681e2b..25398b09d7e6 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -24,7 +24,7 @@ struct BufferedSink : Sink BufferedSink(size_t bufSize = 32 * 1024) : bufSize(bufSize), bufPos(0), buffer(0) { } ~BufferedSink(); - + void operator () (const unsigned char * data, size_t len); void flush(); @@ -39,9 +39,14 @@ struct Source virtual ~Source() { } /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. - It blocks if that much data is not yet available, or throws an - error if it is not going to be available. */ - virtual void operator () (unsigned char * data, size_t len) = 0; + It blocks until all the requested data is available, or throws + an error if it is not going to be available. */ + void operator () (unsigned char * data, size_t len); + + /* Store up to ‘len’ in the buffer pointed to by ‘data’, and + return the number of bytes stored. If blocks until at least + one byte is available. */ + virtual size_t read(unsigned char * data, size_t len) = 0; }; @@ -55,12 +60,10 @@ struct BufferedSource : Source : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { } ~BufferedSource(); - void operator () (unsigned char * data, size_t len); + size_t read(unsigned char * data, size_t len); - /* Store up to ‘len’ in the buffer pointed to by ‘data’, and - return the number of bytes stored. If should block until at - least one byte is available. */ - virtual size_t read(unsigned char * data, size_t len) = 0; + /* Underlying read call, to be overriden. */ + virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; }; @@ -83,7 +86,7 @@ struct FdSource : BufferedSource int fd; FdSource() : fd(-1) { } FdSource(int fd) : fd(fd) { } - size_t read(unsigned char * data, size_t len); + size_t readUnbuffered(unsigned char * data, size_t len); }; @@ -104,13 +107,7 @@ struct StringSource : Source const string & s; size_t pos; StringSource(const string & _s) : s(_s), pos(0) { } - virtual void operator () (unsigned char * data, size_t len) - { - s.copy((char *) data, len, pos); - pos += len; - if (pos > s.size()) - throw Error("end of string reached"); - } + size_t read(unsigned char * data, size_t len); }; diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index a89852638214..695e4c38d5d1 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -210,11 +210,11 @@ struct TunnelSink : Sink }; -struct TunnelSource : Source +struct TunnelSource : BufferedSource { Source & from; TunnelSource(Source & from) : from(from) { } - virtual void operator () (unsigned char * data, size_t len) + size_t readUnbuffered(unsigned char * data, size_t len) { /* Careful: we're going to receive data from the client now, so we have to disable the SIGPOLL handler. */ @@ -224,11 +224,16 @@ struct TunnelSource : Source writeInt(STDERR_READ, to); writeInt(len, to); to.flush(); - string s = readString(from); - if (s.size() != len) throw Error("not enough data"); - memcpy(data, (const unsigned char *) s.c_str(), len); + string s = readString(from); // !!! inefficient startWork(); + + if (s.empty()) throw EndOfFile("unexpected end-of-file"); + if (s.size() > len) throw Error("client sent too much data"); + + memcpy(data, (const unsigned char *) s.c_str(), s.size()); + + return s.size(); } }; @@ -265,10 +270,11 @@ struct SavingSourceAdapter : Source Source & orig; string s; SavingSourceAdapter(Source & orig) : orig(orig) { } - void operator () (unsigned char * data, size_t len) + size_t read(unsigned char * data, size_t len) { - orig(data, len); - s.append((const char *) data, len); + size_t n = orig.read(data, len); + s.append((const char *) data, n); + return n; } }; @@ -397,6 +403,8 @@ static void performOp(unsigned int clientVersion, case wopImportPath: { startWork(); + if (GET_PROTOCOL_MINOR(clientVersion) < 9) + throw Error("import not supported; upgrade your client"); TunnelSource source(from); Path path = store->importPath(true, source); stopWork(); -- cgit 1.4.1 From 8d3dfa2c1782e955d2b7796d19dc0d0381596b98 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 16 Dec 2011 21:29:46 +0000 Subject: * Avoid expensive conversions from char arrays to STL strings. --- src/libutil/serialise.cc | 21 ++++++++++++++++++--- src/libutil/serialise.hh | 2 ++ src/nix-worker/nix-worker.cc | 15 +++++---------- 3 files changed, 25 insertions(+), 13 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 640267a131bf..ba549c214f10 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -149,15 +149,20 @@ void writeLongLong(unsigned long long n, Sink & sink) } -void writeString(const string & s, Sink & sink) +void writeString(const unsigned char * buf, size_t len, Sink & sink) { - size_t len = s.length(); writeInt(len, sink); - sink((const unsigned char *) s.c_str(), len); + sink(buf, len); writePadding(len, sink); } +void writeString(const string & s, Sink & sink) +{ + writeString((const unsigned char *) s.c_str(), s.size(), sink); +} + + void writeStringSet(const StringSet & ss, Sink & sink) { writeInt(ss.size(), sink); @@ -208,6 +213,16 @@ unsigned long long readLongLong(Source & source) } +size_t readString(unsigned char * buf, size_t max, Source & source) +{ + size_t len = readInt(source); + if (len > max) throw Error("string is too long"); + source(buf, len); + readPadding(len, source); + return len; +} + + string readString(Source & source) { size_t len = readInt(source); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 25398b09d7e6..efd8e2a060eb 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -114,12 +114,14 @@ struct StringSource : Source void writePadding(size_t len, Sink & sink); void writeInt(unsigned int n, Sink & sink); void writeLongLong(unsigned long long n, Sink & sink); +void writeString(const unsigned char * buf, size_t len, Sink & sink); void writeString(const string & s, Sink & sink); void writeStringSet(const StringSet & ss, Sink & sink); void readPadding(size_t len, Source & source); unsigned int readInt(Source & source); unsigned long long readLongLong(Source & source); +size_t readString(unsigned char * buf, size_t max, Source & source); string readString(Source & source); StringSet readStringSet(Source & source); diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 695e4c38d5d1..85e2105b2757 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -56,7 +56,7 @@ static void tunnelStderr(const unsigned char * buf, size_t count) if (canSendStderr && myPid == getpid()) { try { writeInt(STDERR_NEXT, to); - writeString(string((char *) buf, count), to); + writeString(buf, count, to); to.flush(); } catch (...) { /* Write failed; that means that the other side is @@ -205,7 +205,7 @@ struct TunnelSink : Sink virtual void operator () (const unsigned char * data, size_t len) { writeInt(STDERR_WRITE, to); - writeString(string((const char *) data, len), to); + writeString(data, len, to); } }; @@ -224,16 +224,11 @@ struct TunnelSource : BufferedSource writeInt(STDERR_READ, to); writeInt(len, to); to.flush(); - string s = readString(from); // !!! inefficient + size_t n = readString(data, len, from); startWork(); - - if (s.empty()) throw EndOfFile("unexpected end-of-file"); - if (s.size() > len) throw Error("client sent too much data"); - - memcpy(data, (const unsigned char *) s.c_str(), s.size()); - - return s.size(); + if (n == 0) throw EndOfFile("unexpected end-of-file"); + return n; } }; -- cgit 1.4.1 From 273b288a7e862ac1918064537ff130cc751fa9fd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 16 Dec 2011 22:31:25 +0000 Subject: * importPath() -> importPaths(). Because of buffering of the input stream it's now necessary for the daemon to process the entire sequence of exported paths, rather than letting the client do it. --- src/libstore/local-store.cc | 17 +++++++++++++++-- src/libstore/local-store.hh | 4 +++- src/libstore/remote-store.cc | 41 +++++++++++++++++++++-------------------- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.hh | 6 +++--- src/libstore/worker-protocol.hh | 4 ++-- src/libutil/serialise.cc | 16 +++++++++++----- src/libutil/serialise.hh | 4 ++-- src/nix-store/nix-store.cc | 10 ++++------ src/nix-worker/nix-worker.cc | 28 +++++++++++++--------------- 10 files changed, 75 insertions(+), 57 deletions(-) (limited to 'src/libutil') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 65b1cdbc87a3..cf0e2ad1b13b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1156,7 +1156,7 @@ void LocalStore::exportPath(const Path & path, bool sign, PathSet references; queryReferences(path, references); - writeStringSet(references, hashAndWriteSink); + writeStrings(references, hashAndWriteSink); Path deriver = queryDeriver(path); writeString(deriver, hashAndWriteSink); @@ -1243,7 +1243,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) Path dstPath = readStorePath(hashAndReadSource); - PathSet references = readStorePaths(hashAndReadSource); + PathSet references = readStorePaths(hashAndReadSource); Path deriver = readString(hashAndReadSource); if (deriver != "") assertStorePath(deriver); @@ -1330,6 +1330,19 @@ Path LocalStore::importPath(bool requireSignature, Source & source) } +Paths LocalStore::importPaths(bool requireSignature, Source & source) +{ + Paths res; + while (true) { + unsigned long long n = readLongLong(source); + if (n == 0) break; + if (n != 1) throw Error("input doesn't look like something created by `nix-store --export'"); + res.push_back(importPath(requireSignature, source)); + } + return res; +} + + void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFreed, unsigned long long & blocksFreed) { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 7ef01b2644b4..4cb905f67231 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -146,7 +146,7 @@ public: void exportPath(const Path & path, bool sign, Sink & sink); - Path importPath(bool requireSignature, Source & source); + Paths importPaths(bool requireSignature, Source & source); void buildDerivations(const PathSet & drvPaths); @@ -259,6 +259,8 @@ private: Path createTempDirInStore(); + Path importPath(bool requireSignature, Source & source); + void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index e976e8fa57ae..942c5bcf1c7c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -27,13 +27,15 @@ Path readStorePath(Source & from) } -PathSet readStorePaths(Source & from) +template T readStorePaths(Source & from) { - PathSet paths = readStringSet(from); - foreach (PathSet::iterator, i, paths) assertStorePath(*i); + T paths = readStrings(from); + foreach (typename T::iterator, i, paths) assertStorePath(*i); return paths; } +template PathSet readStorePaths(Source & from); + RemoteStore::RemoteStore() { @@ -215,7 +217,7 @@ PathSet RemoteStore::queryValidPaths() openConnection(); writeInt(wopQueryValidPaths, to); processStderr(); - return readStorePaths(from); + return readStorePaths(from); } @@ -242,7 +244,7 @@ bool RemoteStore::querySubstitutablePathInfo(const Path & path, if (reply == 0) return false; info.deriver = readString(from); if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths(from); + info.references = readStorePaths(from); info.downloadSize = readLongLong(from); info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0; return true; @@ -260,7 +262,7 @@ ValidPathInfo RemoteStore::queryPathInfo(const Path & path) info.deriver = readString(from); if (info.deriver != "") assertStorePath(info.deriver); info.hash = parseHash(htSHA256, readString(from)); - info.references = readStorePaths(from); + info.references = readStorePaths(from); info.registrationTime = readInt(from); info.narSize = readLongLong(from); return info; @@ -285,7 +287,7 @@ void RemoteStore::queryReferences(const Path & path, writeInt(wopQueryReferences, to); writeString(path, to); processStderr(); - PathSet references2 = readStorePaths(from); + PathSet references2 = readStorePaths(from); references.insert(references2.begin(), references2.end()); } @@ -297,7 +299,7 @@ void RemoteStore::queryReferrers(const Path & path, writeInt(wopQueryReferrers, to); writeString(path, to); processStderr(); - PathSet referrers2 = readStorePaths(from); + PathSet referrers2 = readStorePaths(from); referrers.insert(referrers2.begin(), referrers2.end()); } @@ -320,7 +322,7 @@ PathSet RemoteStore::queryDerivationOutputs(const Path & path) writeInt(wopQueryDerivationOutputs, to); writeString(path, to); processStderr(); - return readStorePaths(from); + return readStorePaths(from); } @@ -350,7 +352,7 @@ Path RemoteStore::addTextToStore(const string & name, const string & s, writeInt(wopAddTextToStore, to); writeString(name, to); writeString(s, to); - writeStringSet(references, to); + writeStrings(references, to); processStderr(); return readStorePath(from); @@ -369,14 +371,14 @@ void RemoteStore::exportPath(const Path & path, bool sign, } -Path RemoteStore::importPath(bool requireSignature, Source & source) +Paths RemoteStore::importPaths(bool requireSignature, Source & source) { openConnection(); - writeInt(wopImportPath, to); + writeInt(wopImportPaths, to); /* We ignore requireSignature, since the worker forces it to true anyway. */ processStderr(0, &source); - return readStorePath(from); + return readStorePaths(from); } @@ -384,7 +386,7 @@ void RemoteStore::buildDerivations(const PathSet & drvPaths) { openConnection(); writeInt(wopBuildDerivations, to); - writeStringSet(drvPaths, to); + writeStrings(drvPaths, to); processStderr(); readInt(from); } @@ -451,7 +453,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) writeInt(wopCollectGarbage, to); writeInt(options.action, to); - writeStringSet(options.pathsToDelete, to); + writeStrings(options.pathsToDelete, to); writeInt(options.ignoreLiveness, to); writeLongLong(options.maxFreed, to); writeInt(options.maxLinks, to); @@ -463,7 +465,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) processStderr(); - results.paths = readStringSet(from); + results.paths = readStrings(from); results.bytesFreed = readLongLong(from); results.blocksFreed = readLongLong(from); } @@ -474,7 +476,7 @@ PathSet RemoteStore::queryFailedPaths() openConnection(); writeInt(wopQueryFailedPaths, to); processStderr(); - return readStorePaths(from); + return readStorePaths(from); } @@ -482,7 +484,7 @@ void RemoteStore::clearFailedPaths(const PathSet & paths) { openConnection(); writeInt(wopClearFailedPaths, to); - writeStringSet(paths, to); + writeStrings(paths, to); processStderr(); readInt(from); } @@ -504,8 +506,7 @@ void RemoteStore::processStderr(Sink * sink, Source * source) size_t len = readInt(from); unsigned char * buf = new unsigned char[len]; AutoDeleteArray d(buf); - size_t n = source->read(buf, len); - writeString(string((const char *) buf, n), to); // !!! inefficient + writeString(buf, source->read(buf, len), to); to.flush(); } else { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 519f46fd1a6d..34a2d91dfcd1 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -56,7 +56,7 @@ public: void exportPath(const Path & path, bool sign, Sink & sink); - Path importPath(bool requireSignature, Source & source); + Paths importPaths(bool requireSignature, Source & source); void buildDerivations(const PathSet & drvPaths); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e3a2c0daa20c..d4997c886207 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -169,9 +169,9 @@ public: virtual void exportPath(const Path & path, bool sign, Sink & sink) = 0; - /* Import a NAR dump created by exportPath() into the Nix - store. */ - virtual Path importPath(bool requireSignature, Source & source) = 0; + /* Import a sequence of NAR dumps created by exportPaths() into + the Nix store. */ + virtual Paths importPaths(bool requireSignature, Source & source) = 0; /* Ensure that the output paths of the derivation are valid. If they are already valid, this is a no-op. Otherwise, validity diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index acabd6ca30d2..760d08a747ae 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -29,7 +29,6 @@ typedef enum { wopSyncWithGC = 13, wopFindRoots = 14, wopExportPath = 16, - wopImportPath = 17, wopQueryDeriver = 18, wopSetOptions = 19, wopCollectGarbage = 20, @@ -39,6 +38,7 @@ typedef enum { wopQueryFailedPaths = 24, wopClearFailedPaths = 25, wopQueryPathInfo = 26, + wopImportPaths = 27, } WorkerOp; @@ -58,7 +58,7 @@ typedef enum { Path readStorePath(Source & from); -PathSet readStorePaths(Source & from); +template T readStorePaths(Source & from); } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index ba549c214f10..c4563ffd1212 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -163,13 +163,16 @@ void writeString(const string & s, Sink & sink) } -void writeStringSet(const StringSet & ss, Sink & sink) +template void writeStrings(const T & ss, Sink & sink) { writeInt(ss.size(), sink); - for (StringSet::iterator i = ss.begin(); i != ss.end(); ++i) + foreach (typename T::const_iterator, i, ss) writeString(*i, sink); } +template void writeStrings(const Paths & ss, Sink & sink); +template void writeStrings(const PathSet & ss, Sink & sink); + void readPadding(size_t len, Source & source) { @@ -234,14 +237,17 @@ string readString(Source & source) } -StringSet readStringSet(Source & source) +template T readStrings(Source & source) { unsigned int count = readInt(source); - StringSet ss; + T ss; while (count--) - ss.insert(readString(source)); + ss.insert(ss.end(), readString(source)); return ss; } +template Paths readStrings(Source & source); +template PathSet readStrings(Source & source); + } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index efd8e2a060eb..ded4b12a046e 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -116,14 +116,14 @@ void writeInt(unsigned int n, Sink & sink); void writeLongLong(unsigned long long n, Sink & sink); void writeString(const unsigned char * buf, size_t len, Sink & sink); void writeString(const string & s, Sink & sink); -void writeStringSet(const StringSet & ss, Sink & sink); +template void writeStrings(const T & ss, Sink & sink); void readPadding(size_t len, Source & source); unsigned int readInt(Source & source); unsigned long long readLongLong(Source & source); size_t readString(unsigned char * buf, size_t max, Source & source); string readString(Source & source); -StringSet readStringSet(Source & source); +template T readStrings(Source & source); MakeError(SerialisationError, Error) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 740033b453d7..e92ccb153d12 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -600,12 +600,10 @@ static void opImport(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); FdSource source(STDIN_FILENO); - while (true) { - unsigned long long n = readLongLong(source); - if (n == 0) break; - if (n != 1) throw Error("input doesn't look like something created by `nix-store --export'"); - cout << format("%1%\n") % store->importPath(requireSignature, source) << std::flush; - } + Paths paths = store->importPaths(requireSignature, source); + + foreach (Paths::iterator, i, paths) + cout << format("%1%\n") % *i << std::flush; } diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 85e2105b2757..5f57b2981d9a 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -327,7 +327,7 @@ static void performOp(unsigned int clientVersion, store->queryReferrers(path, paths); else paths = store->queryDerivationOutputs(path); stopWork(); - writeStringSet(paths, to); + writeStrings(paths, to); break; } @@ -377,7 +377,7 @@ static void performOp(unsigned int clientVersion, case wopAddTextToStore: { string suffix = readString(from); string s = readString(from); - PathSet refs = readStorePaths(from); + PathSet refs = readStorePaths(from); startWork(); Path path = store->addTextToStore(suffix, s, refs); stopWork(); @@ -396,19 +396,17 @@ static void performOp(unsigned int clientVersion, break; } - case wopImportPath: { + case wopImportPaths: { startWork(); - if (GET_PROTOCOL_MINOR(clientVersion) < 9) - throw Error("import not supported; upgrade your client"); TunnelSource source(from); - Path path = store->importPath(true, source); + Paths paths = store->importPaths(true, source); stopWork(); - writeString(path, to); + writeStrings(paths, to); break; } case wopBuildDerivations: { - PathSet drvs = readStorePaths(from); + PathSet drvs = readStorePaths(from); startWork(); store->buildDerivations(drvs); stopWork(); @@ -466,7 +464,7 @@ static void performOp(unsigned int clientVersion, case wopCollectGarbage: { GCOptions options; options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = readStorePaths(from); + options.pathsToDelete = readStorePaths(from); options.ignoreLiveness = readInt(from); options.maxFreed = readLongLong(from); options.maxLinks = readInt(from); @@ -484,7 +482,7 @@ static void performOp(unsigned int clientVersion, store->collectGarbage(options, results); stopWork(); - writeStringSet(results.paths, to); + writeStrings(results.paths, to); writeLongLong(results.bytesFreed, to); writeLongLong(results.blocksFreed, to); @@ -522,7 +520,7 @@ static void performOp(unsigned int clientVersion, writeInt(res ? 1 : 0, to); if (res) { writeString(info.deriver, to); - writeStringSet(info.references, to); + writeStrings(info.references, to); writeLongLong(info.downloadSize, to); if (GET_PROTOCOL_MINOR(clientVersion) >= 7) writeLongLong(info.narSize, to); @@ -534,7 +532,7 @@ static void performOp(unsigned int clientVersion, startWork(); PathSet paths = store->queryValidPaths(); stopWork(); - writeStringSet(paths, to); + writeStrings(paths, to); break; } @@ -542,12 +540,12 @@ static void performOp(unsigned int clientVersion, startWork(); PathSet paths = store->queryFailedPaths(); stopWork(); - writeStringSet(paths, to); + writeStrings(paths, to); break; } case wopClearFailedPaths: { - PathSet paths = readStringSet(from); + PathSet paths = readStrings(from); startWork(); store->clearFailedPaths(paths); stopWork(); @@ -562,7 +560,7 @@ static void performOp(unsigned int clientVersion, stopWork(); writeString(info.deriver, to); writeString(printHash(info.hash), to); - writeStringSet(info.references, to); + writeStrings(info.references, to); writeInt(info.registrationTime, to); writeLongLong(info.narSize, to); break; -- cgit 1.4.1