From c0b706213dad330bd51607ff73059c87f0ec5b93 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 7 Nov 2011 21:11:59 +0000 Subject: * Boldly make SQLite WAL mode the default again. Hopefully the intermittent problems are gone by now. WAL mode is preferrable because it does way fewer fsyncs. --- src/libstore/local-store.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'src/libstore/local-store.cc') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 702ff67e793e..06cadcb0ff0e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -327,10 +327,9 @@ void LocalStore::openDB(bool create) if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) throwSQLiteError(db, "setting synchronous mode"); - /* Set the SQLite journal mode. WAL mode is fastest, but doesn't - seem entirely stable at the moment (Oct. 2010). Thus, use - truncate mode by default. */ - string mode = queryBoolSetting("use-sqlite-wal", false) ? "wal" : "truncate"; + /* Set the SQLite journal mode. WAL mode is fastest, so it's the + default. */ + string mode = queryBoolSetting("use-sqlite-wal", true) ? "wal" : "truncate"; string prevMode; { SQLiteStmt stmt; -- cgit 1.4.1 From 5bbd693caedd5d50994938555b3a4b535875347e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Nov 2011 15:13:37 +0000 Subject: * Add an API function exportPaths() that provides the functionality of ‘nix-store --export’. * Add a Perl module that provides the functionality of ‘nix-copy-closure --to’. This is used by build-remote.pl so it no longer needs to start a separate nix-copy-closure process. Also, it uses the Perl API to do the export, so it doesn't need to start a separate nix-store process either. As a result, nix-copy-closure and build-remote.pl should no longer fail on very large closures due to an "Argument list too long" error. (Note that having very many dependencies in a single derivation can still fail because the environment can become too large. Can't be helped though.) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- perl/Makefile.am | 2 +- perl/lib/Nix/CopyClosure.pm | 46 +++++++++++++++++++++++++++++++++++++++++++++ perl/lib/Nix/Store.pm | 2 +- perl/lib/Nix/Store.xs | 13 +++++++++++++ scripts/build-remote.pl.in | 4 ++-- scripts/nix-copy-closure.in | 36 ++++++++--------------------------- src/libstore/local-store.cc | 2 +- src/libstore/store-api.cc | 11 +++++++++++ src/libstore/store-api.hh | 6 ++++++ src/nix-store/nix-store.cc | 6 +----- 10 files changed, 90 insertions(+), 38 deletions(-) create mode 100644 perl/lib/Nix/CopyClosure.pm (limited to 'src/libstore/local-store.cc') diff --git a/perl/Makefile.am b/perl/Makefile.am index eded469f924b..93f5415c8bb6 100644 --- a/perl/Makefile.am +++ b/perl/Makefile.am @@ -2,7 +2,7 @@ perlversion := $(shell perl -e 'use Config; print $$Config{version};') perlarchname := $(shell perl -e 'use Config; print $$Config{archname};') perllibdir = $(libdir)/perl5/site_perl/$(perlversion)/$(perlarchname) -PERL_MODULES = lib/Nix/Store.pm lib/Nix/Manifest.pm lib/Nix/GeneratePatches.pm lib/Nix/SSH.pm lib/Nix/Config.pm.in +PERL_MODULES = lib/Nix/Store.pm lib/Nix/Manifest.pm lib/Nix/GeneratePatches.pm lib/Nix/SSH.pm lib/Nix/CopyClosure.pm lib/Nix/Config.pm.in all: $(PERL_MODULES:.in=) ln -sfn $(abs_builddir)/.libs/libNixStore.so lib/Store.so diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm new file mode 100644 index 000000000000..045f6bfaf156 --- /dev/null +++ b/perl/lib/Nix/CopyClosure.pm @@ -0,0 +1,46 @@ +package Nix::CopyClosure; + +use strict; +use Nix::Config; +use Nix::Store; + + +sub copyTo { + my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor, $includeOutputs, $dryRun, $sign) = @_; + + $compressor = "$compressor |" if $compressor ne ""; + $decompressor = "$decompressor |" if $decompressor ne ""; + + # Get the closure of this path. + my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs, + map { followLinksToStorePath $_ } @{$storePaths}))); + + # Ask the remote host which paths are invalid. Because of limits + # to the command line length, do this in chunks. Eventually, + # we'll want to use ‘--from-stdin’, but we can't rely on the + # target having this option yet. + my @missing = (); + while (scalar(@closure) > 0) { + my @ps = splice(@closure, 0, 1500); + open(READ, "set -f; ssh $sshHost @{$sshOpts} nix-store --check-validity --print-invalid @ps|"); + while () { + chomp; + push @missing, $_; + } + close READ or die; + } + + # Export the store paths and import them on the remote machine. + if (scalar @missing > 0) { + print STDERR "copying ", scalar @missing, " missing paths to ‘$sshHost’...\n"; + #print STDERR " $_\n" foreach @missing; + unless ($dryRun) { + open SSH, "| $compressor ssh $sshHost @{$sshOpts} '$decompressor nix-store --import'" or die; + exportPaths(fileno(SSH), $sign, @missing); + close SSH or die "copying store paths to remote machine `$sshHost' failed: $?"; + } + } +} + + +1; diff --git a/perl/lib/Nix/Store.pm b/perl/lib/Nix/Store.pm index bef6e7460ca7..d96f8e9ab652 100644 --- a/perl/lib/Nix/Store.pm +++ b/perl/lib/Nix/Store.pm @@ -12,7 +12,7 @@ our %EXPORT_TAGS = ( 'all' => [ qw( ) ] ); our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } ); -our @EXPORT = qw(isValidPath topoSortPaths computeFSClosure followLinksToStorePath); +our @EXPORT = qw(isValidPath topoSortPaths computeFSClosure followLinksToStorePath exportPaths); our $VERSION = '0.15'; diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 9e51ea337c7e..b50451f45f10 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -146,3 +146,16 @@ SV * followLinksToStorePath(char * path) } OUTPUT: RETVAL + + +void exportPaths(int fd, int sign, ...) + PPCODE: + try { + doInit(); + Paths paths; + for (int n = 2; n < items; ++n) paths.push_back(SvPV_nolen(ST(n))); + FdSink sink(fd); + exportPaths(*store, paths, sign, sink); + } catch (Error & e) { + croak(e.what()); + } diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index 110c95f223a0..72d09565bc25 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -5,6 +5,7 @@ use English '-no_match_vars'; use IO::Handle; use Nix::Config; use Nix::SSH qw/sshOpts openSSHConnection/; +use Nix::CopyClosure; no warnings('once'); @@ -225,8 +226,7 @@ sub removeRoots { # Copy the derivation and its dependencies to the build machine. -system("NIX_SSHOPTS=\"@sshOpts\" @bindir@/nix-copy-closure $hostName $maybeSign $drvPath @inputs") == 0 - or die "cannot copy inputs to $hostName: $?"; +Nix::CopyClosure::copyTo($hostName, [ @sshOpts ], [ $drvPath, @inputs ], "", "", 0, 0, $maybeSign ne ""); # Perform the build. diff --git a/scripts/nix-copy-closure.in b/scripts/nix-copy-closure.in index 172acd9e7da2..2eac56e3f229 100755 --- a/scripts/nix-copy-closure.in +++ b/scripts/nix-copy-closure.in @@ -3,6 +3,7 @@ use Nix::SSH; use Nix::Config; use Nix::Store; +use Nix::CopyClosure; if (scalar @ARGV < 1) { @@ -39,8 +40,8 @@ while (@ARGV) { $sign = 1; } elsif ($arg eq "--gzip") { - $compressor = "| gzip"; - $decompressor = "gunzip |"; + $compressor = "gzip"; + $decompressor = "gunzip"; } elsif ($arg eq "--from") { $toMode = 0; @@ -67,30 +68,7 @@ openSSHConnection $sshHost or die "$0: unable to start SSH\n"; if ($toMode) { # Copy TO the remote machine. - - # Get the closure of this path. - my @allStorePaths = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs, map { followLinksToStorePath $_ } @storePaths))); - - # Ask the remote host which paths are invalid. - open(READ, "set -f; ssh $sshHost @sshOpts nix-store --check-validity --print-invalid @allStorePaths|"); - my @missing = (); - while () { - chomp; - push @missing, $_; - } - close READ or die; - - # Export the store paths and import them on the remote machine. - if (scalar @missing > 0) { - print STDERR "copying these missing paths:\n"; - print STDERR " $_\n" foreach @missing; - unless ($dryRun) { - my $extraOpts = $sign ? "--sign" : ""; - system("set -f; nix-store --export $extraOpts @missing $compressor | ssh $sshHost @sshOpts '$decompressor nix-store --import'") == 0 - or die "copying store paths to remote machine `$sshHost' failed: $?"; - } - } - + Nix::CopyClosure::copyTo($sshHost, [ @sshOpts ], [ @storePaths ], $compressor, $decompressor, $includeOutputs, $dryRun, $sign); } else { # Copy FROM the remote machine. @@ -112,8 +90,10 @@ else { # Copy FROM the remote machine. # Export the store paths on the remote machine and import them on locally. if (scalar @missing > 0) { - print STDERR "copying these missing paths:\n"; - print STDERR " $_\n" foreach @missing; + print STDERR "copying ", scalar @missing, " missing paths from ‘$sshHost’...\n"; + #print STDERR " $_\n" foreach @missing; + $compressor = "| $compressor" if $compressor ne ""; + $decompressor = "$decompressor |" if $decompressor ne ""; unless ($dryRun) { my $extraOpts = $sign ? "--sign" : ""; system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor $Nix::Config::binDir/nix-store --import") == 0 diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 06cadcb0ff0e..3c1f2ecacdcb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1255,7 +1255,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) bool haveSignature = readInt(hashAndReadSource) == 1; if (requireSignature && !haveSignature) - throw Error("imported archive lacks a signature"); + throw Error(format("imported archive of `%1%' lacks a signature") % dstPath); if (haveSignature) { string signature = readString(hashAndReadSource); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index d67ff2c77299..36ade2170876 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -298,6 +298,17 @@ string showPaths(const PathSet & paths) } +void exportPaths(StoreAPI & store, const Paths & paths, + bool sign, Sink & sink) +{ + foreach (Paths::const_iterator, i, paths) { + writeInt(1, sink); + store.exportPath(*i, sign, sink); + } + writeInt(0, sink); +} + + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b3e67436c6c3..8bfb09880edd 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -342,6 +342,12 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven = false); +/* Export multiple paths in the format expected by ‘nix-store + --import’. */ +void exportPaths(StoreAPI & store, const Paths & paths, + bool sign, Sink & sink); + + } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 371ca54af07c..84d3da032c2e 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -594,11 +594,7 @@ static void opExport(Strings opFlags, Strings opArgs) else throw UsageError(format("unknown flag `%1%'") % *i); FdSink sink(STDOUT_FILENO); - for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); ++i) { - writeInt(1, sink); - store->exportPath(*i, sign, sink); - } - writeInt(0, sink); + exportPaths(*store, opArgs, sign, sink); } -- cgit 1.4.1 From 000160f5b915ce784e740c139f81e0cbeda751c4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 2 Dec 2011 17:52:18 +0000 Subject: * In ‘nix-store --verify --check-contents’, repair missing hashes rather than complain about them. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/libstore/local-store.cc | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) (limited to 'src/libstore/local-store.cc') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 3c1f2ecacdcb..a353168ffb61 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -366,7 +366,7 @@ void LocalStore::openDB(bool create) stmtRegisterValidPath.create(db, "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);"); stmtUpdatePathInfo.create(db, - "update ValidPaths set narSize = ? where path = ?;"); + "update ValidPaths set narSize = ?, hash = ? where path = ?;"); stmtAddReference.create(db, "insert or replace into Refs (referrer, reference) values (?, ?);"); stmtQueryPathInfo.create(db, @@ -683,7 +683,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) } -/* Update path info in the database. Currently only updated the +/* Update path info in the database. Currently only updates the narSize field. */ void LocalStore::updatePathInfo(const ValidPathInfo & info) { @@ -692,6 +692,7 @@ void LocalStore::updatePathInfo(const ValidPathInfo & info) stmtUpdatePathInfo.bind64(info.narSize); else stmtUpdatePathInfo.bind(); // null + stmtUpdatePathInfo.bind("sha256:" + printHash(info.hash)); stmtUpdatePathInfo.bind(info.path); if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE) throwSQLiteError(db, format("updating info of path `%1%' in database") % info.path); @@ -1386,6 +1387,8 @@ void LocalStore::verifyStore(bool checkContents) if (checkContents) { printMsg(lvlInfo, "checking hashes..."); + Hash nullHash(htSHA256); + foreach (PathSet::iterator, i, validPaths) { try { ValidPathInfo info = queryPathInfo(*i); @@ -1394,17 +1397,30 @@ void LocalStore::verifyStore(bool checkContents) printMsg(lvlTalkative, format("checking contents of `%1%'") % *i); HashResult current = hashPath(info.hash.type, *i); - if (current.first != info.hash) { + if (info.hash != nullHash && info.hash != current.first) { printMsg(lvlError, format("path `%1%' was modified! " "expected hash `%2%', got `%3%'") % *i % printHash(info.hash) % printHash(current.first)); } else { + + bool update = false; + + /* Fill in missing hashes. */ + if (info.hash == nullHash) { + printMsg(lvlError, format("fixing missing hash on `%1%'") % *i); + info.hash = current.first; + update = true; + } + /* Fill in missing narSize fields (from old stores). */ if (info.narSize == 0) { printMsg(lvlError, format("updating size field on `%1%' to %2%") % *i % current.second); info.narSize = current.second; - updatePathInfo(info); + update = true; } + + if (update) updatePathInfo(info); + } } catch (Error & e) { -- cgit 1.4.1 From c8c0380744afd107611bba17127a182ecebd4e0b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 5 Dec 2011 21:04:20 +0000 Subject: * Remove unnecessary quotes. showPaths() already adds quotes. --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore/local-store.cc') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a353168ffb61..5107a93de299 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1347,7 +1347,7 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr PathSet referrers; queryReferrers(path, referrers); referrers.erase(path); /* ignore self-references */ if (!referrers.empty()) - throw PathInUse(format("cannot delete path `%1%' because it is in use by `%2%'") + throw PathInUse(format("cannot delete path `%1%' because it is in use by %2%") % path % showPaths(referrers)); invalidatePath(path); } -- cgit 1.4.1 From 5a1b9ed0aa3a0c240b667dbe504b61b2b68e4d16 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 15 Dec 2011 16:19:53 +0000 Subject: * Refactoring: move sink/source buffering into separate classes. * Buffer the HashSink. This speeds up hashing a bit because it prevents lots of calls to the hash update functions (e.g. nix-hash went from 9.3s to 8.7s of user time on the closure of my /var/run/current-system). --- src/libstore/local-store.cc | 9 ++--- src/libstore/references.cc | 4 +- src/libstore/remote-store.cc | 2 +- src/libutil/hash.cc | 22 +++++----- src/libutil/hash.hh | 5 ++- src/libutil/serialise.cc | 69 ++++++++++++++++++++----------- src/libutil/serialise.hh | 96 +++++++++++++++++++++++++------------------- src/nix-worker/nix-worker.cc | 8 ++-- 8 files changed, 125 insertions(+), 90 deletions(-) (limited to 'src/libstore/local-store.cc') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5107a93de299..525e5fc7bbe3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1103,16 +1103,14 @@ struct HashAndWriteSink : Sink HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) { } - virtual void operator () - (const unsigned char * data, unsigned int len) + virtual void operator () (const unsigned char * data, size_t len) { writeSink(data, len); hashSink(data, len); } Hash currentHash() { - HashSink hashSinkClone(hashSink); - return hashSinkClone.finish().first; + return hashSink.currentHash().first; } }; @@ -1201,8 +1199,7 @@ struct HashAndReadSource : Source { hashing = true; } - virtual void operator () - (unsigned char * data, unsigned int len) + virtual void operator () (unsigned char * data, size_t len) { readSource(data, len); if (hashing) hashSink(data, len); diff --git a/src/libstore/references.cc b/src/libstore/references.cc index ade9c9aa20e3..c1f9e3ba7711 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -57,11 +57,11 @@ struct RefScanSink : Sink RefScanSink() : hashSink(htSHA256) { } - void operator () (const unsigned char * data, unsigned int len); + void operator () (const unsigned char * data, size_t len); }; -void RefScanSink::operator () (const unsigned char * data, unsigned int len) +void RefScanSink::operator () (const unsigned char * data, size_t len) { hashSink(data, len); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8269b6a83168..7bf0ad7bd4c7 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -374,7 +374,7 @@ Path RemoteStore::importPath(bool requireSignature, Source & source) openConnection(); writeInt(wopImportPath, to); /* We ignore requireSignature, since the worker forces it to true - anyway. */ + anyway. */ processStderr(0, &source); return readStorePath(from); } diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 53342344107d..bbfe7847fd8a 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -306,21 +306,13 @@ HashSink::HashSink(HashType ht) : ht(ht) start(ht, *ctx); } -HashSink::HashSink(const HashSink & h) -{ - ht = h.ht; - bytes = h.bytes; - ctx = new Ctx; - *ctx = *h.ctx; -} - HashSink::~HashSink() { + bufPos = 0; delete ctx; } -void HashSink::operator () - (const unsigned char * data, unsigned int len) +void HashSink::write(const unsigned char * data, size_t len) { bytes += len; update(ht, *ctx, data, len); @@ -328,11 +320,21 @@ void HashSink::operator () HashResult HashSink::finish() { + flush(); Hash hash(ht); nix::finish(ht, *ctx, hash.hash); return HashResult(hash, bytes); } +HashResult HashSink::currentHash() +{ + flush(); + Ctx ctx2 = *ctx; + Hash hash(ht); + nix::finish(ht, ctx2, hash.hash); + return HashResult(hash, bytes); +} + HashResult hashPath( HashType ht, const Path & path, PathFilter & filter) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index cbdcf4c8d4ff..e0b6478cc418 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -91,7 +91,7 @@ string printHashType(HashType ht); union Ctx; -class HashSink : public Sink +class HashSink : public BufferedSink { private: HashType ht; @@ -102,8 +102,9 @@ public: HashSink(HashType ht); HashSink(const HashSink & h); ~HashSink(); - virtual void operator () (const unsigned char * data, unsigned int len); + void write(const unsigned char * data, size_t len); HashResult finish(); + HashResult currentHash(); }; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index e3a53c0d01dc..a82262704ea3 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -8,7 +8,16 @@ namespace nix { -void FdSink::operator () (const unsigned char * data, unsigned int len) +BufferedSink::~BufferedSink() +{ + /* We can't call flush() here, because C++ for some insane reason + doesn't allow you to call virtual methods from a destructor. */ + assert(!bufPos); + if (buffer) delete[] buffer; +} + + +void BufferedSink::operator () (const unsigned char * data, size_t len) { if (!buffer) buffer = new unsigned char[bufSize]; @@ -16,7 +25,7 @@ void FdSink::operator () (const unsigned char * data, unsigned int len) /* Optimisation: bypass the buffer if the data exceeds the buffer size and there is no unflushed data. */ if (bufPos == 0 && len >= bufSize) { - writeFull(fd, data, len); + write(data, len); break; } /* Otherwise, copy the bytes to the buffer. Flush the buffer @@ -29,31 +38,32 @@ void FdSink::operator () (const unsigned char * data, unsigned int len) } -void FdSink::flush() +void BufferedSink::flush() { - if (fd == -1 || bufPos == 0) return; - writeFull(fd, buffer, bufPos); + if (bufPos == 0) return; + write(buffer, bufPos); bufPos = 0; } -void FdSource::operator () (unsigned char * data, unsigned int len) +void FdSink::write(const unsigned char * data, size_t len) +{ + writeFull(fd, data, len); +} + + +BufferedSource::~BufferedSource() +{ + if (buffer) delete[] buffer; +} + + +void BufferedSource::operator () (unsigned char * data, size_t len) { if (!buffer) buffer = new unsigned char[bufSize]; while (len) { - if (!bufPosIn) { - /* Read as much data as is available (up to the buffer - size). */ - checkInterrupt(); - ssize_t n = read(fd, (char *) buffer, bufSize); - if (n == -1) { - if (errno == EINTR) continue; - throw SysError("reading from file"); - } - if (n == 0) throw EndOfFile("unexpected end-of-file"); - bufPosIn = n; - } + if (!bufPosIn) bufPosIn = read(buffer, bufSize); /* Copy out the data in the buffer. */ size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; @@ -64,7 +74,20 @@ void FdSource::operator () (unsigned char * data, unsigned int len) } -void writePadding(unsigned int len, Sink & sink) +size_t FdSource::read(unsigned char * data, size_t len) +{ + ssize_t n; + do { + checkInterrupt(); + n = ::read(fd, (char *) data, bufSize); + } while (n == -1 && errno == EINTR); + if (n == -1) throw SysError("reading from file"); + if (n == 0) throw EndOfFile("unexpected end-of-file"); + return n; +} + + +void writePadding(size_t len, Sink & sink) { if (len % 8) { unsigned char zero[8]; @@ -103,7 +126,7 @@ void writeLongLong(unsigned long long n, Sink & sink) void writeString(const string & s, Sink & sink) { - unsigned int len = s.length(); + size_t len = s.length(); writeInt(len, sink); sink((const unsigned char *) s.c_str(), len); writePadding(len, sink); @@ -118,11 +141,11 @@ void writeStringSet(const StringSet & ss, Sink & sink) } -void readPadding(unsigned int len, Source & source) +void readPadding(size_t len, Source & source) { if (len % 8) { unsigned char zero[8]; - unsigned int n = 8 - (len % 8); + size_t n = 8 - (len % 8); source(zero, n); for (unsigned int i = 0; i < n; i++) if (zero[i]) throw SerialisationError("non-zero padding"); @@ -162,7 +185,7 @@ unsigned long long readLongLong(Source & source) string readString(Source & source) { - unsigned int len = readInt(source); + size_t len = readInt(source); unsigned char * buf = new unsigned char[len]; AutoDeleteArray d(buf); source(buf, len); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index b8d4d7a849d2..a0588668f037 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -11,7 +11,25 @@ namespace nix { struct Sink { virtual ~Sink() { } - virtual void operator () (const unsigned char * data, unsigned int len) = 0; + virtual void operator () (const unsigned char * data, size_t len) = 0; +}; + + +/* A buffered abstract sink. */ +struct BufferedSink : Sink +{ + size_t bufSize, bufPos; + unsigned char * buffer; + + BufferedSink(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPos(0), buffer(0) { } + ~BufferedSink(); + + void operator () (const unsigned char * data, size_t len); + + void flush(); + + virtual void write(const unsigned char * data, size_t len) = 0; }; @@ -20,56 +38,52 @@ struct Source { virtual ~Source() { } - /* The callee should store exactly *len bytes in the buffer - pointed to by data. It should block if that much data is not - yet available, or throw an error if it is not going to be - available. */ - virtual void operator () (unsigned char * data, unsigned int len) = 0; + /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. + It blocks if that much data is not yet available, or throws an + error if it is not going to be available. */ + virtual void operator () (unsigned char * data, size_t len) = 0; }; -/* A sink that writes data to a file descriptor (using a buffer). */ -struct FdSink : Sink +/* A buffered abstract source. */ +struct BufferedSource : Source { - int fd; - unsigned int bufSize, bufPos; + size_t bufSize, bufPosIn, bufPosOut; unsigned char * buffer; - FdSink() : fd(-1), bufSize(32 * 1024), bufPos(0), buffer(0) { } + BufferedSource(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { } + ~BufferedSource(); - FdSink(int fd, unsigned int bufSize = 32 * 1024) - : fd(fd), bufSize(bufSize), bufPos(0), buffer(0) { } - - ~FdSink() - { - flush(); - if (buffer) delete[] buffer; - } + void operator () (unsigned char * data, size_t len); - void operator () (const unsigned char * data, unsigned int len); - - void flush(); + /* Store up to ‘len’ in the buffer pointed to by ‘data’, and + return the number of bytes stored. If should block until at + least one byte is available. */ + virtual size_t read(unsigned char * data, size_t len) = 0; }; -/* A source that reads data from a file descriptor. */ -struct FdSource : Source +/* A sink that writes data to a file descriptor. */ +struct FdSink : BufferedSink { int fd; - unsigned int bufSize, bufPosIn, bufPosOut; - unsigned char * buffer; - FdSource() : fd(-1), bufSize(32 * 1024), bufPosIn(0), bufPosOut(0), buffer(0) { } - - FdSource(int fd, unsigned int bufSize = 32 * 1024) - : fd(fd), bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { } - - ~FdSource() - { - if (buffer) delete[] buffer; - } + FdSink() : fd(-1) { } + FdSink(int fd) : fd(fd) { } + ~FdSink() { flush(); } - void operator () (unsigned char * data, unsigned int len); + void write(const unsigned char * data, size_t len); +}; + + +/* A source that reads data from a file descriptor. */ +struct FdSource : BufferedSource +{ + int fd; + FdSource() : fd(-1) { } + FdSource(int fd) : fd(fd) { } + size_t read(unsigned char * data, size_t len); }; @@ -77,7 +91,7 @@ struct FdSource : Source struct StringSink : Sink { string s; - virtual void operator () (const unsigned char * data, unsigned int len) + void operator () (const unsigned char * data, size_t len) { s.append((const char *) data, len); } @@ -88,9 +102,9 @@ struct StringSink : Sink struct StringSource : Source { const string & s; - unsigned int pos; + size_t pos; StringSource(const string & _s) : s(_s), pos(0) { } - virtual void operator () (unsigned char * data, unsigned int len) + virtual void operator () (unsigned char * data, size_t len) { s.copy((char *) data, len, pos); pos += len; @@ -100,13 +114,13 @@ struct StringSource : Source }; -void writePadding(unsigned int len, Sink & sink); +void writePadding(size_t len, Sink & sink); void writeInt(unsigned int n, Sink & sink); void writeLongLong(unsigned long long n, Sink & sink); void writeString(const string & s, Sink & sink); void writeStringSet(const StringSet & ss, Sink & sink); -void readPadding(unsigned int len, Source & source); +void readPadding(size_t len, Source & source); unsigned int readInt(Source & source); unsigned long long readLongLong(Source & source); string readString(Source & source); diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 6c222420e0ce..a89852638214 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -202,8 +202,7 @@ struct TunnelSink : Sink { Sink & to; TunnelSink(Sink & to) : to(to) { } - virtual void operator () - (const unsigned char * data, unsigned int len) + virtual void operator () (const unsigned char * data, size_t len) { writeInt(STDERR_WRITE, to); writeString(string((const char *) data, len), to); @@ -215,8 +214,7 @@ struct TunnelSource : Source { Source & from; TunnelSource(Source & from) : from(from) { } - virtual void operator () - (unsigned char * data, unsigned int len) + virtual void operator () (unsigned char * data, size_t len) { /* Careful: we're going to receive data from the client now, so we have to disable the SIGPOLL handler. */ @@ -267,7 +265,7 @@ struct SavingSourceAdapter : Source Source & orig; string s; SavingSourceAdapter(Source & orig) : orig(orig) { } - void operator () (unsigned char * data, unsigned int len) + void operator () (unsigned char * data, size_t len) { orig(data, len); s.append((const char *) data, len); -- cgit 1.4.1 From e0bd307802d13476055f8ba99ab7808de0fd71e5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 16 Dec 2011 19:44:13 +0000 Subject: * Make the import operation through the daemon much more efficient (way fewer roundtrips) by allowing the client to send data in bigger chunks. * Some refactoring. --- src/libstore/local-store.cc | 7 ++++--- src/libstore/remote-store.cc | 6 +++--- src/libstore/worker-protocol.hh | 2 +- src/libutil/serialise.cc | 42 +++++++++++++++++++++++++++++------------ src/libutil/serialise.hh | 31 ++++++++++++++---------------- src/nix-worker/nix-worker.cc | 24 +++++++++++++++-------- 6 files changed, 68 insertions(+), 44 deletions(-) (limited to 'src/libstore/local-store.cc') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 525e5fc7bbe3..65b1cdbc87a3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1199,10 +1199,11 @@ struct HashAndReadSource : Source { hashing = true; } - virtual void operator () (unsigned char * data, size_t len) + size_t read(unsigned char * data, size_t len) { - readSource(data, len); - if (hashing) hashSink(data, len); + size_t n = readSource.read(data, len); + if (hashing) hashSink(data, n); + return n; } }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 7bf0ad7bd4c7..e976e8fa57ae 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -501,11 +501,11 @@ void RemoteStore::processStderr(Sink * sink, Source * source) } else if (msg == STDERR_READ) { if (!source) throw Error("no source"); - unsigned int len = readInt(from); + size_t len = readInt(from); unsigned char * buf = new unsigned char[len]; AutoDeleteArray d(buf); - (*source)(buf, len); - writeString(string((const char *) buf, len), to); + size_t n = source->read(buf, len); + writeString(string((const char *) buf, n), to); // !!! inefficient to.flush(); } else { diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index acb8bc8b2948..acabd6ca30d2 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -8,7 +8,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x108 +#define PROTOCOL_VERSION 0x109 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 76f2e721a535..640267a131bf 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -23,8 +23,9 @@ void BufferedSink::operator () (const unsigned char * data, size_t len) while (len) { /* Optimisation: bypass the buffer if the data exceeds the - buffer size and there is no unflushed data. */ - if (bufPos == 0 && len >= bufSize) { + buffer size. */ + if (bufPos + len >= bufSize) { + flush(); write(data, len); break; } @@ -59,29 +60,37 @@ void FdSink::write(const unsigned char * data, size_t len) } +void Source::operator () (unsigned char * data, size_t len) +{ + while (len) { + size_t n = read(data, len); + data += n; len -= n; + } +} + + BufferedSource::~BufferedSource() { if (buffer) delete[] buffer; } -void BufferedSource::operator () (unsigned char * data, size_t len) +size_t BufferedSource::read(unsigned char * data, size_t len) { if (!buffer) buffer = new unsigned char[bufSize]; - while (len) { - if (!bufPosIn) bufPosIn = read(buffer, bufSize); + if (!bufPosIn) bufPosIn = readUnbuffered(buffer, bufSize); - /* Copy out the data in the buffer. */ - size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; - memcpy(data, buffer + bufPosOut, n); - data += n; bufPosOut += n; len -= n; - if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0; - } + /* Copy out the data in the buffer. */ + size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; + memcpy(data, buffer + bufPosOut, n); + bufPosOut += n; + if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0; + return n; } -size_t FdSource::read(unsigned char * data, size_t len) +size_t FdSource::readUnbuffered(unsigned char * data, size_t len) { ssize_t n; do { @@ -94,6 +103,15 @@ size_t FdSource::read(unsigned char * data, size_t len) } +size_t StringSource::read(unsigned char * data, size_t len) +{ + if (pos == s.size()) throw EndOfFile("end of string reached"); + size_t n = s.copy((char *) data, len, pos); + pos += n; + return n; +} + + void writePadding(size_t len, Sink & sink) { if (len % 8) { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index a155f6681e2b..25398b09d7e6 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -24,7 +24,7 @@ struct BufferedSink : Sink BufferedSink(size_t bufSize = 32 * 1024) : bufSize(bufSize), bufPos(0), buffer(0) { } ~BufferedSink(); - + void operator () (const unsigned char * data, size_t len); void flush(); @@ -39,9 +39,14 @@ struct Source virtual ~Source() { } /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. - It blocks if that much data is not yet available, or throws an - error if it is not going to be available. */ - virtual void operator () (unsigned char * data, size_t len) = 0; + It blocks until all the requested data is available, or throws + an error if it is not going to be available. */ + void operator () (unsigned char * data, size_t len); + + /* Store up to ‘len’ in the buffer pointed to by ‘data’, and + return the number of bytes stored. If blocks until at least + one byte is available. */ + virtual size_t read(unsigned char * data, size_t len) = 0; }; @@ -55,12 +60,10 @@ struct BufferedSource : Source : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { } ~BufferedSource(); - void operator () (unsigned char * data, size_t len); + size_t read(unsigned char * data, size_t len); - /* Store up to ‘len’ in the buffer pointed to by ‘data’, and - return the number of bytes stored. If should block until at - least one byte is available. */ - virtual size_t read(unsigned char * data, size_t len) = 0; + /* Underlying read call, to be overriden. */ + virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; }; @@ -83,7 +86,7 @@ struct FdSource : BufferedSource int fd; FdSource() : fd(-1) { } FdSource(int fd) : fd(fd) { } - size_t read(unsigned char * data, size_t len); + size_t readUnbuffered(unsigned char * data, size_t len); }; @@ -104,13 +107,7 @@ struct StringSource : Source const string & s; size_t pos; StringSource(const string & _s) : s(_s), pos(0) { } - virtual void operator () (unsigned char * data, size_t len) - { - s.copy((char *) data, len, pos); - pos += len; - if (pos > s.size()) - throw Error("end of string reached"); - } + size_t read(unsigned char * data, size_t len); }; diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index a89852638214..695e4c38d5d1 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -210,11 +210,11 @@ struct TunnelSink : Sink }; -struct TunnelSource : Source +struct TunnelSource : BufferedSource { Source & from; TunnelSource(Source & from) : from(from) { } - virtual void operator () (unsigned char * data, size_t len) + size_t readUnbuffered(unsigned char * data, size_t len) { /* Careful: we're going to receive data from the client now, so we have to disable the SIGPOLL handler. */ @@ -224,11 +224,16 @@ struct TunnelSource : Source writeInt(STDERR_READ, to); writeInt(len, to); to.flush(); - string s = readString(from); - if (s.size() != len) throw Error("not enough data"); - memcpy(data, (const unsigned char *) s.c_str(), len); + string s = readString(from); // !!! inefficient startWork(); + + if (s.empty()) throw EndOfFile("unexpected end-of-file"); + if (s.size() > len) throw Error("client sent too much data"); + + memcpy(data, (const unsigned char *) s.c_str(), s.size()); + + return s.size(); } }; @@ -265,10 +270,11 @@ struct SavingSourceAdapter : Source Source & orig; string s; SavingSourceAdapter(Source & orig) : orig(orig) { } - void operator () (unsigned char * data, size_t len) + size_t read(unsigned char * data, size_t len) { - orig(data, len); - s.append((const char *) data, len); + size_t n = orig.read(data, len); + s.append((const char *) data, n); + return n; } }; @@ -397,6 +403,8 @@ static void performOp(unsigned int clientVersion, case wopImportPath: { startWork(); + if (GET_PROTOCOL_MINOR(clientVersion) < 9) + throw Error("import not supported; upgrade your client"); TunnelSource source(from); Path path = store->importPath(true, source); stopWork(); -- cgit 1.4.1 From 273b288a7e862ac1918064537ff130cc751fa9fd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 16 Dec 2011 22:31:25 +0000 Subject: * importPath() -> importPaths(). Because of buffering of the input stream it's now necessary for the daemon to process the entire sequence of exported paths, rather than letting the client do it. --- src/libstore/local-store.cc | 17 +++++++++++++++-- src/libstore/local-store.hh | 4 +++- src/libstore/remote-store.cc | 41 +++++++++++++++++++++-------------------- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.hh | 6 +++--- src/libstore/worker-protocol.hh | 4 ++-- src/libutil/serialise.cc | 16 +++++++++++----- src/libutil/serialise.hh | 4 ++-- src/nix-store/nix-store.cc | 10 ++++------ src/nix-worker/nix-worker.cc | 28 +++++++++++++--------------- 10 files changed, 75 insertions(+), 57 deletions(-) (limited to 'src/libstore/local-store.cc') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 65b1cdbc87a3..cf0e2ad1b13b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1156,7 +1156,7 @@ void LocalStore::exportPath(const Path & path, bool sign, PathSet references; queryReferences(path, references); - writeStringSet(references, hashAndWriteSink); + writeStrings(references, hashAndWriteSink); Path deriver = queryDeriver(path); writeString(deriver, hashAndWriteSink); @@ -1243,7 +1243,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) Path dstPath = readStorePath(hashAndReadSource); - PathSet references = readStorePaths(hashAndReadSource); + PathSet references = readStorePaths(hashAndReadSource); Path deriver = readString(hashAndReadSource); if (deriver != "") assertStorePath(deriver); @@ -1330,6 +1330,19 @@ Path LocalStore::importPath(bool requireSignature, Source & source) } +Paths LocalStore::importPaths(bool requireSignature, Source & source) +{ + Paths res; + while (true) { + unsigned long long n = readLongLong(source); + if (n == 0) break; + if (n != 1) throw Error("input doesn't look like something created by `nix-store --export'"); + res.push_back(importPath(requireSignature, source)); + } + return res; +} + + void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFreed, unsigned long long & blocksFreed) { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 7ef01b2644b4..4cb905f67231 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -146,7 +146,7 @@ public: void exportPath(const Path & path, bool sign, Sink & sink); - Path importPath(bool requireSignature, Source & source); + Paths importPaths(bool requireSignature, Source & source); void buildDerivations(const PathSet & drvPaths); @@ -259,6 +259,8 @@ private: Path createTempDirInStore(); + Path importPath(bool requireSignature, Source & source); + void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index e976e8fa57ae..942c5bcf1c7c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -27,13 +27,15 @@ Path readStorePath(Source & from) } -PathSet readStorePaths(Source & from) +template T readStorePaths(Source & from) { - PathSet paths = readStringSet(from); - foreach (PathSet::iterator, i, paths) assertStorePath(*i); + T paths = readStrings(from); + foreach (typename T::iterator, i, paths) assertStorePath(*i); return paths; } +template PathSet readStorePaths(Source & from); + RemoteStore::RemoteStore() { @@ -215,7 +217,7 @@ PathSet RemoteStore::queryValidPaths() openConnection(); writeInt(wopQueryValidPaths, to); processStderr(); - return readStorePaths(from); + return readStorePaths(from); } @@ -242,7 +244,7 @@ bool RemoteStore::querySubstitutablePathInfo(const Path & path, if (reply == 0) return false; info.deriver = readString(from); if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths(from); + info.references = readStorePaths(from); info.downloadSize = readLongLong(from); info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0; return true; @@ -260,7 +262,7 @@ ValidPathInfo RemoteStore::queryPathInfo(const Path & path) info.deriver = readString(from); if (info.deriver != "") assertStorePath(info.deriver); info.hash = parseHash(htSHA256, readString(from)); - info.references = readStorePaths(from); + info.references = readStorePaths(from); info.registrationTime = readInt(from); info.narSize = readLongLong(from); return info; @@ -285,7 +287,7 @@ void RemoteStore::queryReferences(const Path & path, writeInt(wopQueryReferences, to); writeString(path, to); processStderr(); - PathSet references2 = readStorePaths(from); + PathSet references2 = readStorePaths(from); references.insert(references2.begin(), references2.end()); } @@ -297,7 +299,7 @@ void RemoteStore::queryReferrers(const Path & path, writeInt(wopQueryReferrers, to); writeString(path, to); processStderr(); - PathSet referrers2 = readStorePaths(from); + PathSet referrers2 = readStorePaths(from); referrers.insert(referrers2.begin(), referrers2.end()); } @@ -320,7 +322,7 @@ PathSet RemoteStore::queryDerivationOutputs(const Path & path) writeInt(wopQueryDerivationOutputs, to); writeString(path, to); processStderr(); - return readStorePaths(from); + return readStorePaths(from); } @@ -350,7 +352,7 @@ Path RemoteStore::addTextToStore(const string & name, const string & s, writeInt(wopAddTextToStore, to); writeString(name, to); writeString(s, to); - writeStringSet(references, to); + writeStrings(references, to); processStderr(); return readStorePath(from); @@ -369,14 +371,14 @@ void RemoteStore::exportPath(const Path & path, bool sign, } -Path RemoteStore::importPath(bool requireSignature, Source & source) +Paths RemoteStore::importPaths(bool requireSignature, Source & source) { openConnection(); - writeInt(wopImportPath, to); + writeInt(wopImportPaths, to); /* We ignore requireSignature, since the worker forces it to true anyway. */ processStderr(0, &source); - return readStorePath(from); + return readStorePaths(from); } @@ -384,7 +386,7 @@ void RemoteStore::buildDerivations(const PathSet & drvPaths) { openConnection(); writeInt(wopBuildDerivations, to); - writeStringSet(drvPaths, to); + writeStrings(drvPaths, to); processStderr(); readInt(from); } @@ -451,7 +453,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) writeInt(wopCollectGarbage, to); writeInt(options.action, to); - writeStringSet(options.pathsToDelete, to); + writeStrings(options.pathsToDelete, to); writeInt(options.ignoreLiveness, to); writeLongLong(options.maxFreed, to); writeInt(options.maxLinks, to); @@ -463,7 +465,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) processStderr(); - results.paths = readStringSet(from); + results.paths = readStrings(from); results.bytesFreed = readLongLong(from); results.blocksFreed = readLongLong(from); } @@ -474,7 +476,7 @@ PathSet RemoteStore::queryFailedPaths() openConnection(); writeInt(wopQueryFailedPaths, to); processStderr(); - return readStorePaths(from); + return readStorePaths(from); } @@ -482,7 +484,7 @@ void RemoteStore::clearFailedPaths(const PathSet & paths) { openConnection(); writeInt(wopClearFailedPaths, to); - writeStringSet(paths, to); + writeStrings(paths, to); processStderr(); readInt(from); } @@ -504,8 +506,7 @@ void RemoteStore::processStderr(Sink * sink, Source * source) size_t len = readInt(from); unsigned char * buf = new unsigned char[len]; AutoDeleteArray d(buf); - size_t n = source->read(buf, len); - writeString(string((const char *) buf, n), to); // !!! inefficient + writeString(buf, source->read(buf, len), to); to.flush(); } else { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 519f46fd1a6d..34a2d91dfcd1 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -56,7 +56,7 @@ public: void exportPath(const Path & path, bool sign, Sink & sink); - Path importPath(bool requireSignature, Source & source); + Paths importPaths(bool requireSignature, Source & source); void buildDerivations(const PathSet & drvPaths); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e3a2c0daa20c..d4997c886207 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -169,9 +169,9 @@ public: virtual void exportPath(const Path & path, bool sign, Sink & sink) = 0; - /* Import a NAR dump created by exportPath() into the Nix - store. */ - virtual Path importPath(bool requireSignature, Source & source) = 0; + /* Import a sequence of NAR dumps created by exportPaths() into + the Nix store. */ + virtual Paths importPaths(bool requireSignature, Source & source) = 0; /* Ensure that the output paths of the derivation are valid. If they are already valid, this is a no-op. Otherwise, validity diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index acabd6ca30d2..760d08a747ae 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -29,7 +29,6 @@ typedef enum { wopSyncWithGC = 13, wopFindRoots = 14, wopExportPath = 16, - wopImportPath = 17, wopQueryDeriver = 18, wopSetOptions = 19, wopCollectGarbage = 20, @@ -39,6 +38,7 @@ typedef enum { wopQueryFailedPaths = 24, wopClearFailedPaths = 25, wopQueryPathInfo = 26, + wopImportPaths = 27, } WorkerOp; @@ -58,7 +58,7 @@ typedef enum { Path readStorePath(Source & from); -PathSet readStorePaths(Source & from); +template T readStorePaths(Source & from); } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index ba549c214f10..c4563ffd1212 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -163,13 +163,16 @@ void writeString(const string & s, Sink & sink) } -void writeStringSet(const StringSet & ss, Sink & sink) +template void writeStrings(const T & ss, Sink & sink) { writeInt(ss.size(), sink); - for (StringSet::iterator i = ss.begin(); i != ss.end(); ++i) + foreach (typename T::const_iterator, i, ss) writeString(*i, sink); } +template void writeStrings(const Paths & ss, Sink & sink); +template void writeStrings(const PathSet & ss, Sink & sink); + void readPadding(size_t len, Source & source) { @@ -234,14 +237,17 @@ string readString(Source & source) } -StringSet readStringSet(Source & source) +template T readStrings(Source & source) { unsigned int count = readInt(source); - StringSet ss; + T ss; while (count--) - ss.insert(readString(source)); + ss.insert(ss.end(), readString(source)); return ss; } +template Paths readStrings(Source & source); +template PathSet readStrings(Source & source); + } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index efd8e2a060eb..ded4b12a046e 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -116,14 +116,14 @@ void writeInt(unsigned int n, Sink & sink); void writeLongLong(unsigned long long n, Sink & sink); void writeString(const unsigned char * buf, size_t len, Sink & sink); void writeString(const string & s, Sink & sink); -void writeStringSet(const StringSet & ss, Sink & sink); +template void writeStrings(const T & ss, Sink & sink); void readPadding(size_t len, Source & source); unsigned int readInt(Source & source); unsigned long long readLongLong(Source & source); size_t readString(unsigned char * buf, size_t max, Source & source); string readString(Source & source); -StringSet readStringSet(Source & source); +template T readStrings(Source & source); MakeError(SerialisationError, Error) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 740033b453d7..e92ccb153d12 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -600,12 +600,10 @@ static void opImport(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); FdSource source(STDIN_FILENO); - while (true) { - unsigned long long n = readLongLong(source); - if (n == 0) break; - if (n != 1) throw Error("input doesn't look like something created by `nix-store --export'"); - cout << format("%1%\n") % store->importPath(requireSignature, source) << std::flush; - } + Paths paths = store->importPaths(requireSignature, source); + + foreach (Paths::iterator, i, paths) + cout << format("%1%\n") % *i << std::flush; } diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 85e2105b2757..5f57b2981d9a 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -327,7 +327,7 @@ static void performOp(unsigned int clientVersion, store->queryReferrers(path, paths); else paths = store->queryDerivationOutputs(path); stopWork(); - writeStringSet(paths, to); + writeStrings(paths, to); break; } @@ -377,7 +377,7 @@ static void performOp(unsigned int clientVersion, case wopAddTextToStore: { string suffix = readString(from); string s = readString(from); - PathSet refs = readStorePaths(from); + PathSet refs = readStorePaths(from); startWork(); Path path = store->addTextToStore(suffix, s, refs); stopWork(); @@ -396,19 +396,17 @@ static void performOp(unsigned int clientVersion, break; } - case wopImportPath: { + case wopImportPaths: { startWork(); - if (GET_PROTOCOL_MINOR(clientVersion) < 9) - throw Error("import not supported; upgrade your client"); TunnelSource source(from); - Path path = store->importPath(true, source); + Paths paths = store->importPaths(true, source); stopWork(); - writeString(path, to); + writeStrings(paths, to); break; } case wopBuildDerivations: { - PathSet drvs = readStorePaths(from); + PathSet drvs = readStorePaths(from); startWork(); store->buildDerivations(drvs); stopWork(); @@ -466,7 +464,7 @@ static void performOp(unsigned int clientVersion, case wopCollectGarbage: { GCOptions options; options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = readStorePaths(from); + options.pathsToDelete = readStorePaths(from); options.ignoreLiveness = readInt(from); options.maxFreed = readLongLong(from); options.maxLinks = readInt(from); @@ -484,7 +482,7 @@ static void performOp(unsigned int clientVersion, store->collectGarbage(options, results); stopWork(); - writeStringSet(results.paths, to); + writeStrings(results.paths, to); writeLongLong(results.bytesFreed, to); writeLongLong(results.blocksFreed, to); @@ -522,7 +520,7 @@ static void performOp(unsigned int clientVersion, writeInt(res ? 1 : 0, to); if (res) { writeString(info.deriver, to); - writeStringSet(info.references, to); + writeStrings(info.references, to); writeLongLong(info.downloadSize, to); if (GET_PROTOCOL_MINOR(clientVersion) >= 7) writeLongLong(info.narSize, to); @@ -534,7 +532,7 @@ static void performOp(unsigned int clientVersion, startWork(); PathSet paths = store->queryValidPaths(); stopWork(); - writeStringSet(paths, to); + writeStrings(paths, to); break; } @@ -542,12 +540,12 @@ static void performOp(unsigned int clientVersion, startWork(); PathSet paths = store->queryFailedPaths(); stopWork(); - writeStringSet(paths, to); + writeStrings(paths, to); break; } case wopClearFailedPaths: { - PathSet paths = readStringSet(from); + PathSet paths = readStrings(from); startWork(); store->clearFailedPaths(paths); stopWork(); @@ -562,7 +560,7 @@ static void performOp(unsigned int clientVersion, stopWork(); writeString(info.deriver, to); writeString(printHash(info.hash), to); - writeStringSet(info.references, to); + writeStrings(info.references, to); writeInt(info.registrationTime, to); writeLongLong(info.narSize, to); break; -- cgit 1.4.1