From c0b706213dad330bd51607ff73059c87f0ec5b93 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 7 Nov 2011 21:11:59 +0000 Subject: * Boldly make SQLite WAL mode the default again. Hopefully the intermittent problems are gone by now. WAL mode is preferrable because it does way fewer fsyncs. --- src/libstore/local-store.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 702ff67e793e..06cadcb0ff0e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -327,10 +327,9 @@ void LocalStore::openDB(bool create) if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) throwSQLiteError(db, "setting synchronous mode"); - /* Set the SQLite journal mode. WAL mode is fastest, but doesn't - seem entirely stable at the moment (Oct. 2010). Thus, use - truncate mode by default. */ - string mode = queryBoolSetting("use-sqlite-wal", false) ? "wal" : "truncate"; + /* Set the SQLite journal mode. WAL mode is fastest, so it's the + default. */ + string mode = queryBoolSetting("use-sqlite-wal", true) ? "wal" : "truncate"; string prevMode; { SQLiteStmt stmt; -- cgit 1.4.1 From 4e1ea17052b4cc2445bc2ece2136f248112b4e45 Mon Sep 17 00:00:00 2001 From: Rob Vermaas Date: Mon, 21 Nov 2011 15:19:51 +0000 Subject: nix: add /etc/hosts with localhost entry to chroot builds. --- src/libstore/build.cc | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index d12f41d66941..171c08913255 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1650,6 +1650,9 @@ void DerivationGoal::startBuilder() (format("nixbld:!:%1%:\n") % (buildUser.enabled() ? buildUser.getGID() : getgid())).str()); + /* Create /etc/hosts with localhost entry. */ + writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n"); + /* Bind-mount a user-configurable set of directories from the host file system. The `/dev/pts' directory must be mounted separately so that newly-created pseudo-terminals show -- cgit 1.4.1 From 993fa94fb489f46e127ef760bea8c65ef281ef7f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Nov 2011 17:28:41 +0000 Subject: * Move initialisation of variables like nixConfDir from libmain to libstore so that the Perl bindings can use it as well. It's vital that the Perl bindings use the configuration file, because otherwise nix-copy-closure will fail with a ‘database locked’ message if the value of ‘use-sqlite-wal’ is changed from the default. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- perl/lib/Nix/Store.xs | 4 +--- src/libmain/Makefile.am | 7 ------- src/libmain/shared.cc | 26 ++------------------------ src/libstore/Makefile.am | 11 ++++++++++- src/libstore/globals.cc | 30 ++++++++++++++++++++++++++++++ src/libstore/globals.hh | 4 +++- 6 files changed, 46 insertions(+), 36 deletions(-) (limited to 'src/libstore') diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index af71ad955c4f..9e51ea337c7e 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -18,10 +18,8 @@ using namespace nix; void doInit() { if (!store) { - nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", "/nix/store"))); - nixStateDir = canonPath(getEnv("NIX_STATE_DIR", "/nix/var/nix")); - nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db"); try { + setDefaultsFromEnvironment(); store = openStore(); } catch (Error & e) { croak(e.what()); diff --git a/src/libmain/Makefile.am b/src/libmain/Makefile.am index ababc3595968..404353c62922 100644 --- a/src/libmain/Makefile.am +++ b/src/libmain/Makefile.am @@ -7,13 +7,6 @@ libmain_la_LIBADD = ../libstore/libstore.la @BDW_GC_LIBS@ pkginclude_HEADERS = shared.hh AM_CXXFLAGS = \ - -DNIX_STORE_DIR=\"$(storedir)\" \ - -DNIX_DATA_DIR=\"$(datadir)\" \ - -DNIX_STATE_DIR=\"$(localstatedir)/nix\" \ - -DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \ - -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \ - -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ - -DNIX_BIN_DIR=\"$(bindir)\" \ -DNIX_VERSION=\"$(VERSION)\" \ -I$(srcdir)/.. -I$(srcdir)/../libutil \ -I$(srcdir)/../libstore diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 8532cdaadcab..f7c11ed0544e 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -120,30 +120,8 @@ static bool showTrace = false; processor. */ static void initAndRun(int argc, char * * argv) { - /* Setup Nix paths. */ - nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR))); - nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR)); - nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR)); - nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR)); - nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db"); - nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR)); - nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)); - nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)); - - string subs = getEnv("NIX_SUBSTITUTERS", "default"); - if (subs == "default") { - substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl"); - substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl"); - } else - substituters = tokenizeString(subs, ":"); - - /* Get some settings from the configuration file. */ - thisSystem = querySetting("system", SYSTEM); - maxBuildJobs = queryIntSetting("build-max-jobs", 1); - buildCores = queryIntSetting("build-cores", 1); - maxSilentTime = queryIntSetting("build-max-silent-time", 0); - buildTimeout = queryIntSetting("build-timeout", 0); - + setDefaultsFromEnvironment(); + /* Catch SIGINT. */ struct sigaction act; act.sa_handler = sigintHandler; diff --git a/src/libstore/Makefile.am b/src/libstore/Makefile.am index e19256b925ea..39a61233b4a7 100644 --- a/src/libstore/Makefile.am +++ b/src/libstore/Makefile.am @@ -15,7 +15,16 @@ libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la ${aterm_ EXTRA_DIST = schema.sql AM_CXXFLAGS = -Wall \ - ${sqlite_include} -I$(srcdir)/.. -I$(srcdir)/../libutil + ${sqlite_include} -I$(srcdir)/.. -I$(srcdir)/../libutil \ + -DNIX_STORE_DIR=\"$(storedir)\" \ + -DNIX_DATA_DIR=\"$(datadir)\" \ + -DNIX_STATE_DIR=\"$(localstatedir)/nix\" \ + -DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \ + -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \ + -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ + -DNIX_BIN_DIR=\"$(bindir)\" \ + -I$(srcdir)/.. -I$(srcdir)/../libutil \ + -I$(srcdir)/../libstore local-store.lo: schema.sql.hh diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 2e9dc88237d1..5c22f1406649 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,3 +1,5 @@ +#include "config.h" + #include "globals.hh" #include "util.hh" @@ -138,5 +140,33 @@ void reloadSettings() settings.clear(); } + +void setDefaultsFromEnvironment() +{ + /* Setup Nix paths. */ + nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR))); + nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR)); + nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR)); + nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR)); + nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db"); + nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR)); + nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)); + nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)); + + string subs = getEnv("NIX_SUBSTITUTERS", "default"); + if (subs == "default") { + substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl"); + substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl"); + } else + substituters = tokenizeString(subs, ":"); + + /* Get some settings from the configuration file. */ + thisSystem = querySetting("system", SYSTEM); + maxBuildJobs = queryIntSetting("build-max-jobs", 1); + buildCores = queryIntSetting("build-cores", 1); + maxSilentTime = queryIntSetting("build-max-silent-time", 0); + buildTimeout = queryIntSetting("build-timeout", 0); +} + } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 231c1f8508ac..12a9b9ca15c0 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -114,7 +114,9 @@ void overrideSetting(const string & name, const Strings & value); void reloadSettings(); - +void setDefaultsFromEnvironment(); + + } -- cgit 1.4.1 From 5bbd693caedd5d50994938555b3a4b535875347e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Nov 2011 15:13:37 +0000 Subject: * Add an API function exportPaths() that provides the functionality of ‘nix-store --export’. * Add a Perl module that provides the functionality of ‘nix-copy-closure --to’. This is used by build-remote.pl so it no longer needs to start a separate nix-copy-closure process. Also, it uses the Perl API to do the export, so it doesn't need to start a separate nix-store process either. As a result, nix-copy-closure and build-remote.pl should no longer fail on very large closures due to an "Argument list too long" error. (Note that having very many dependencies in a single derivation can still fail because the environment can become too large. Can't be helped though.) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- perl/Makefile.am | 2 +- perl/lib/Nix/CopyClosure.pm | 46 +++++++++++++++++++++++++++++++++++++++++++++ perl/lib/Nix/Store.pm | 2 +- perl/lib/Nix/Store.xs | 13 +++++++++++++ scripts/build-remote.pl.in | 4 ++-- scripts/nix-copy-closure.in | 36 ++++++++--------------------------- src/libstore/local-store.cc | 2 +- src/libstore/store-api.cc | 11 +++++++++++ src/libstore/store-api.hh | 6 ++++++ src/nix-store/nix-store.cc | 6 +----- 10 files changed, 90 insertions(+), 38 deletions(-) create mode 100644 perl/lib/Nix/CopyClosure.pm (limited to 'src/libstore') diff --git a/perl/Makefile.am b/perl/Makefile.am index eded469f924b..93f5415c8bb6 100644 --- a/perl/Makefile.am +++ b/perl/Makefile.am @@ -2,7 +2,7 @@ perlversion := $(shell perl -e 'use Config; print $$Config{version};') perlarchname := $(shell perl -e 'use Config; print $$Config{archname};') perllibdir = $(libdir)/perl5/site_perl/$(perlversion)/$(perlarchname) -PERL_MODULES = lib/Nix/Store.pm lib/Nix/Manifest.pm lib/Nix/GeneratePatches.pm lib/Nix/SSH.pm lib/Nix/Config.pm.in +PERL_MODULES = lib/Nix/Store.pm lib/Nix/Manifest.pm lib/Nix/GeneratePatches.pm lib/Nix/SSH.pm lib/Nix/CopyClosure.pm lib/Nix/Config.pm.in all: $(PERL_MODULES:.in=) ln -sfn $(abs_builddir)/.libs/libNixStore.so lib/Store.so diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm new file mode 100644 index 000000000000..045f6bfaf156 --- /dev/null +++ b/perl/lib/Nix/CopyClosure.pm @@ -0,0 +1,46 @@ +package Nix::CopyClosure; + +use strict; +use Nix::Config; +use Nix::Store; + + +sub copyTo { + my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor, $includeOutputs, $dryRun, $sign) = @_; + + $compressor = "$compressor |" if $compressor ne ""; + $decompressor = "$decompressor |" if $decompressor ne ""; + + # Get the closure of this path. + my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs, + map { followLinksToStorePath $_ } @{$storePaths}))); + + # Ask the remote host which paths are invalid. Because of limits + # to the command line length, do this in chunks. Eventually, + # we'll want to use ‘--from-stdin’, but we can't rely on the + # target having this option yet. + my @missing = (); + while (scalar(@closure) > 0) { + my @ps = splice(@closure, 0, 1500); + open(READ, "set -f; ssh $sshHost @{$sshOpts} nix-store --check-validity --print-invalid @ps|"); + while () { + chomp; + push @missing, $_; + } + close READ or die; + } + + # Export the store paths and import them on the remote machine. + if (scalar @missing > 0) { + print STDERR "copying ", scalar @missing, " missing paths to ‘$sshHost’...\n"; + #print STDERR " $_\n" foreach @missing; + unless ($dryRun) { + open SSH, "| $compressor ssh $sshHost @{$sshOpts} '$decompressor nix-store --import'" or die; + exportPaths(fileno(SSH), $sign, @missing); + close SSH or die "copying store paths to remote machine `$sshHost' failed: $?"; + } + } +} + + +1; diff --git a/perl/lib/Nix/Store.pm b/perl/lib/Nix/Store.pm index bef6e7460ca7..d96f8e9ab652 100644 --- a/perl/lib/Nix/Store.pm +++ b/perl/lib/Nix/Store.pm @@ -12,7 +12,7 @@ our %EXPORT_TAGS = ( 'all' => [ qw( ) ] ); our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } ); -our @EXPORT = qw(isValidPath topoSortPaths computeFSClosure followLinksToStorePath); +our @EXPORT = qw(isValidPath topoSortPaths computeFSClosure followLinksToStorePath exportPaths); our $VERSION = '0.15'; diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 9e51ea337c7e..b50451f45f10 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -146,3 +146,16 @@ SV * followLinksToStorePath(char * path) } OUTPUT: RETVAL + + +void exportPaths(int fd, int sign, ...) + PPCODE: + try { + doInit(); + Paths paths; + for (int n = 2; n < items; ++n) paths.push_back(SvPV_nolen(ST(n))); + FdSink sink(fd); + exportPaths(*store, paths, sign, sink); + } catch (Error & e) { + croak(e.what()); + } diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index 110c95f223a0..72d09565bc25 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -5,6 +5,7 @@ use English '-no_match_vars'; use IO::Handle; use Nix::Config; use Nix::SSH qw/sshOpts openSSHConnection/; +use Nix::CopyClosure; no warnings('once'); @@ -225,8 +226,7 @@ sub removeRoots { # Copy the derivation and its dependencies to the build machine. -system("NIX_SSHOPTS=\"@sshOpts\" @bindir@/nix-copy-closure $hostName $maybeSign $drvPath @inputs") == 0 - or die "cannot copy inputs to $hostName: $?"; +Nix::CopyClosure::copyTo($hostName, [ @sshOpts ], [ $drvPath, @inputs ], "", "", 0, 0, $maybeSign ne ""); # Perform the build. diff --git a/scripts/nix-copy-closure.in b/scripts/nix-copy-closure.in index 172acd9e7da2..2eac56e3f229 100755 --- a/scripts/nix-copy-closure.in +++ b/scripts/nix-copy-closure.in @@ -3,6 +3,7 @@ use Nix::SSH; use Nix::Config; use Nix::Store; +use Nix::CopyClosure; if (scalar @ARGV < 1) { @@ -39,8 +40,8 @@ while (@ARGV) { $sign = 1; } elsif ($arg eq "--gzip") { - $compressor = "| gzip"; - $decompressor = "gunzip |"; + $compressor = "gzip"; + $decompressor = "gunzip"; } elsif ($arg eq "--from") { $toMode = 0; @@ -67,30 +68,7 @@ openSSHConnection $sshHost or die "$0: unable to start SSH\n"; if ($toMode) { # Copy TO the remote machine. - - # Get the closure of this path. - my @allStorePaths = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs, map { followLinksToStorePath $_ } @storePaths))); - - # Ask the remote host which paths are invalid. - open(READ, "set -f; ssh $sshHost @sshOpts nix-store --check-validity --print-invalid @allStorePaths|"); - my @missing = (); - while () { - chomp; - push @missing, $_; - } - close READ or die; - - # Export the store paths and import them on the remote machine. - if (scalar @missing > 0) { - print STDERR "copying these missing paths:\n"; - print STDERR " $_\n" foreach @missing; - unless ($dryRun) { - my $extraOpts = $sign ? "--sign" : ""; - system("set -f; nix-store --export $extraOpts @missing $compressor | ssh $sshHost @sshOpts '$decompressor nix-store --import'") == 0 - or die "copying store paths to remote machine `$sshHost' failed: $?"; - } - } - + Nix::CopyClosure::copyTo($sshHost, [ @sshOpts ], [ @storePaths ], $compressor, $decompressor, $includeOutputs, $dryRun, $sign); } else { # Copy FROM the remote machine. @@ -112,8 +90,10 @@ else { # Copy FROM the remote machine. # Export the store paths on the remote machine and import them on locally. if (scalar @missing > 0) { - print STDERR "copying these missing paths:\n"; - print STDERR " $_\n" foreach @missing; + print STDERR "copying ", scalar @missing, " missing paths from ‘$sshHost’...\n"; + #print STDERR " $_\n" foreach @missing; + $compressor = "| $compressor" if $compressor ne ""; + $decompressor = "$decompressor |" if $decompressor ne ""; unless ($dryRun) { my $extraOpts = $sign ? "--sign" : ""; system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor $Nix::Config::binDir/nix-store --import") == 0 diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 06cadcb0ff0e..3c1f2ecacdcb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1255,7 +1255,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) bool haveSignature = readInt(hashAndReadSource) == 1; if (requireSignature && !haveSignature) - throw Error("imported archive lacks a signature"); + throw Error(format("imported archive of `%1%' lacks a signature") % dstPath); if (haveSignature) { string signature = readString(hashAndReadSource); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index d67ff2c77299..36ade2170876 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -298,6 +298,17 @@ string showPaths(const PathSet & paths) } +void exportPaths(StoreAPI & store, const Paths & paths, + bool sign, Sink & sink) +{ + foreach (Paths::const_iterator, i, paths) { + writeInt(1, sink); + store.exportPath(*i, sign, sink); + } + writeInt(0, sink); +} + + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b3e67436c6c3..8bfb09880edd 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -342,6 +342,12 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven = false); +/* Export multiple paths in the format expected by ‘nix-store + --import’. */ +void exportPaths(StoreAPI & store, const Paths & paths, + bool sign, Sink & sink); + + } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 371ca54af07c..84d3da032c2e 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -594,11 +594,7 @@ static void opExport(Strings opFlags, Strings opArgs) else throw UsageError(format("unknown flag `%1%'") % *i); FdSink sink(STDOUT_FILENO); - for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); ++i) { - writeInt(1, sink); - store->exportPath(*i, sign, sink); - } - writeInt(0, sink); + exportPaths(*store, opArgs, sign, sink); } -- cgit 1.4.1 From b1eb8f4249dbf666afa046c45e903566e9eb2df9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 29 Nov 2011 13:00:41 +0000 Subject: * Get rid of some superfluous error messages if a substituter fails. * Say "fetch" instead of "substitute". --- src/libmain/shared.cc | 2 +- src/libstore/build.cc | 21 ++++++--------------- 2 files changed, 7 insertions(+), 16 deletions(-) (limited to 'src/libstore') diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index f7c11ed0544e..9076e99947e1 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -65,7 +65,7 @@ void printMissing(StoreAPI & store, const PathSet & paths) } if (!willSubstitute.empty()) { - printMsg(lvlInfo, format("these paths will be downloaded/copied (%.2f MiB download, %.2f MiB unpacked):") + printMsg(lvlInfo, format("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):") % (downloadSize / (1024.0 * 1024.0)) % (narSize / (1024.0 * 1024.0))); foreach (PathSet::iterator, i, willSubstitute) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 171c08913255..a8ef9b23efaf 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2202,9 +2202,7 @@ void SubstitutionGoal::tryNext() if (subs.size() == 0) { /* None left. Terminate this goal and let someone else deal with it. */ - printMsg(lvlError, - format("path `%1%' is required, but there is no substituter that can build it") - % storePath); + debug(format("path `%1%' is required, but there is no substituter that can build it") % storePath); amDone(ecFailed); return; } @@ -2235,8 +2233,7 @@ void SubstitutionGoal::referencesValid() trace("all references realised"); if (nrFailed > 0) { - printMsg(lvlError, - format("some references of path `%1%' could not be realised") % storePath); + debug(format("some references of path `%1%' could not be realised") % storePath); amDone(ecFailed); return; } @@ -2289,9 +2286,7 @@ void SubstitutionGoal::tryToRun() return; } - printMsg(lvlInfo, - format("substituting path `%1%' using substituter `%2%'") - % storePath % sub); + printMsg(lvlInfo, format("fetching path `%1%'...") % storePath); logPipe.create(); @@ -2367,19 +2362,15 @@ void SubstitutionGoal::finished() try { if (!statusOk(status)) - throw SubstError(format("builder for `%1%' %2%") + throw SubstError(format("fetching path `%1%' %2%") % storePath % statusToString(status)); if (!pathExists(storePath)) - throw SubstError( - format("substitute did not produce path `%1%'") - % storePath); + throw SubstError(format("substitute did not produce path `%1%'") % storePath); } catch (SubstError & e) { - printMsg(lvlInfo, - format("substitution of path `%1%' using substituter `%2%' failed: %3%") - % storePath % sub % e.msg()); + printMsg(lvlInfo, e.msg()); if (printBuildTrace) { printMsg(lvlError, format("@ substituter-failed %1% %2% %3%") -- cgit 1.4.1 From 24f863d86b0316c736fe9e89998cd442b8a400dd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 Dec 2011 13:48:48 +0000 Subject: * When doing "nix-store --add-fixed" without "--recursive" via the Nix daemon (which is an error), print a nicer error message than "Connection reset by peer" or "broken pipe". * In the daemon, log errors that occur during request parameter processing. --- src/libstore/remote-store.cc | 2 +- src/libstore/store-api.hh | 4 +--- src/nix-worker/nix-worker.cc | 16 ++++++++++++++-- 3 files changed, 16 insertions(+), 6 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 568a6aa58a35..0c6a1c37d1ec 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -328,7 +328,7 @@ Path RemoteStore::addToStore(const Path & _srcPath, openConnection(); Path srcPath(absPath(_srcPath)); - + writeInt(wopAddToStore, to); writeString(baseNameOf(srcPath), to); /* backwards compatibility hack */ diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 8bfb09880edd..e3a2c0daa20c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -151,9 +151,7 @@ public: /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. - If `fixed' is true, then the output of a fixed-output - derivation is pre-loaded into the Nix store. The function - object `filter' can be used to exclude files (see + The function object `filter' can be used to exclude files (see libutil/archive.hh). */ virtual Path addToStore(const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 0fa1b40aede9..d74b82df4c23 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -241,11 +241,14 @@ struct TunnelSource : Source the contents of the file to `s'. Otherwise barf. */ struct RetrieveRegularNARSink : ParseSink { + bool regular; string s; + RetrieveRegularNARSink() : regular(true) { } + void createDirectory(const Path & path) { - throw Error("regular file expected"); + regular = false; } void receiveContents(unsigned char * data, unsigned int len) @@ -255,7 +258,7 @@ struct RetrieveRegularNARSink : ParseSink void createSymlink(const Path & path, const string & target) { - throw Error("regular file expected"); + regular = false; } }; @@ -363,6 +366,7 @@ static void performOp(unsigned int clientVersion, parseDump(sink, savedNAR); } else { parseDump(savedRegular, from); + if (!savedRegular.regular) throw Error("regular file expected"); } startWork(); @@ -638,7 +642,15 @@ static void processConnection() try { performOp(clientVersion, from, to, op); } catch (Error & e) { + /* If we're not in a state were we can send replies, then + something went wrong processing the input of the + client. This can happen especially if I/O errors occur + during addTextToStore() / importPath(). If that + happens, just send the error message and exit. */ + bool errorAllowed = canSendStderr; + if (!errorAllowed) printMsg(lvlError, format("error processing client input: %1%") % e.msg()); stopWork(false, e.msg(), GET_PROTOCOL_MINOR(clientVersion) >= 8 ? e.status : 0); + if (!errorAllowed) break; } assert(!canSendStderr); -- cgit 1.4.1 From 000160f5b915ce784e740c139f81e0cbeda751c4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 2 Dec 2011 17:52:18 +0000 Subject: * In ‘nix-store --verify --check-contents’, repair missing hashes rather than complain about them. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/libstore/local-store.cc | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 3c1f2ecacdcb..a353168ffb61 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -366,7 +366,7 @@ void LocalStore::openDB(bool create) stmtRegisterValidPath.create(db, "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);"); stmtUpdatePathInfo.create(db, - "update ValidPaths set narSize = ? where path = ?;"); + "update ValidPaths set narSize = ?, hash = ? where path = ?;"); stmtAddReference.create(db, "insert or replace into Refs (referrer, reference) values (?, ?);"); stmtQueryPathInfo.create(db, @@ -683,7 +683,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) } -/* Update path info in the database. Currently only updated the +/* Update path info in the database. Currently only updates the narSize field. */ void LocalStore::updatePathInfo(const ValidPathInfo & info) { @@ -692,6 +692,7 @@ void LocalStore::updatePathInfo(const ValidPathInfo & info) stmtUpdatePathInfo.bind64(info.narSize); else stmtUpdatePathInfo.bind(); // null + stmtUpdatePathInfo.bind("sha256:" + printHash(info.hash)); stmtUpdatePathInfo.bind(info.path); if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE) throwSQLiteError(db, format("updating info of path `%1%' in database") % info.path); @@ -1386,6 +1387,8 @@ void LocalStore::verifyStore(bool checkContents) if (checkContents) { printMsg(lvlInfo, "checking hashes..."); + Hash nullHash(htSHA256); + foreach (PathSet::iterator, i, validPaths) { try { ValidPathInfo info = queryPathInfo(*i); @@ -1394,17 +1397,30 @@ void LocalStore::verifyStore(bool checkContents) printMsg(lvlTalkative, format("checking contents of `%1%'") % *i); HashResult current = hashPath(info.hash.type, *i); - if (current.first != info.hash) { + if (info.hash != nullHash && info.hash != current.first) { printMsg(lvlError, format("path `%1%' was modified! " "expected hash `%2%', got `%3%'") % *i % printHash(info.hash) % printHash(current.first)); } else { + + bool update = false; + + /* Fill in missing hashes. */ + if (info.hash == nullHash) { + printMsg(lvlError, format("fixing missing hash on `%1%'") % *i); + info.hash = current.first; + update = true; + } + /* Fill in missing narSize fields (from old stores). */ if (info.narSize == 0) { printMsg(lvlError, format("updating size field on `%1%' to %2%") % *i % current.second); info.narSize = current.second; - updatePathInfo(info); + update = true; } + + if (update) updatePathInfo(info); + } } catch (Error & e) { -- cgit 1.4.1 From c8c0380744afd107611bba17127a182ecebd4e0b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 5 Dec 2011 21:04:20 +0000 Subject: * Remove unnecessary quotes. showPaths() already adds quotes. --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a353168ffb61..5107a93de299 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1347,7 +1347,7 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr PathSet referrers; queryReferrers(path, referrers); referrers.erase(path); /* ignore self-references */ if (!referrers.empty()) - throw PathInUse(format("cannot delete path `%1%' because it is in use by `%2%'") + throw PathInUse(format("cannot delete path `%1%' because it is in use by %2%") % path % showPaths(referrers)); invalidatePath(path); } -- cgit 1.4.1 From 3a48282b0681d68147e18f7464eaddf1d188c3be Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 Dec 2011 23:30:06 +0000 Subject: * Buffer writes in FdSink. This significantly reduces the number of system calls / context switches when dumping a NAR and in the worker protocol. --- src/libstore/remote-store.cc | 4 ++++ src/libutil/serialise.cc | 25 ++++++++++++++++++++++++- src/libutil/serialise.hh | 19 +++++++++++++------ src/nix-worker/nix-worker.cc | 16 +++++++++------- 4 files changed, 50 insertions(+), 14 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 0c6a1c37d1ec..8269b6a83168 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -65,6 +65,7 @@ void RemoteStore::openConnection() /* Send the magic greeting, check for the reply. */ try { writeInt(WORKER_MAGIC_1, to); + to.flush(); unsigned int magic = readInt(from); if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); @@ -166,6 +167,7 @@ void RemoteStore::connectToDaemon() RemoteStore::~RemoteStore() { try { + to.flush(); fdSocket.close(); if (child != -1) child.wait(true); @@ -488,6 +490,7 @@ void RemoteStore::clearFailedPaths(const PathSet & paths) void RemoteStore::processStderr(Sink * sink, Source * source) { + to.flush(); unsigned int msg; while ((msg = readInt(from)) == STDERR_NEXT || msg == STDERR_READ || msg == STDERR_WRITE) { @@ -503,6 +506,7 @@ void RemoteStore::processStderr(Sink * sink, Source * source) AutoDeleteArray d(buf); (*source)(buf, len); writeString(string((const char *) buf, len), to); + to.flush(); } else { string s = readString(from); diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 9b422271323f..66a64a6be4c8 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -9,7 +9,30 @@ namespace nix { void FdSink::operator () (const unsigned char * data, unsigned int len) { - writeFull(fd, data, len); + if (!buffer) buffer = new unsigned char[bufSize]; + + while (len) { + /* Optimisation: bypass the buffer if the data exceeds the + buffer size and there is no unflushed data. */ + if (bufPos == 0 && len >= bufSize) { + writeFull(fd, data, len); + break; + } + /* Otherwise, copy the bytes to the buffer. Flush the buffer + when it's full. */ + size_t n = bufPos + len > bufSize ? bufSize - bufPos : len; + memcpy(buffer + bufPos, data, n); + data += n; bufPos += n; len -= n; + if (bufPos == bufSize) flush(); + } +} + + +void FdSink::flush() +{ + if (fd == -1 || bufPos == 0) return; + writeFull(fd, buffer, bufPos); + bufPos = 0; } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 0e797d63bca9..711bd5e6c7ce 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -28,22 +28,29 @@ struct Source }; -/* A sink that writes data to a file descriptor. */ +/* A sink that writes data to a file descriptor (using a buffer). */ struct FdSink : Sink { int fd; + unsigned int bufSize, bufPos; + unsigned char * buffer; - FdSink() + FdSink() : fd(-1), bufSize(32 * 1024), bufPos(0), buffer(0) { } + + FdSink(int fd, unsigned int bufSize = 32 * 1024) + : fd(fd), bufSize(bufSize), bufPos(0), buffer(0) { - fd = -1; } - - FdSink(int fd) + + ~FdSink() { - this->fd = fd; + flush(); + if (buffer) delete[] buffer; } void operator () (const unsigned char * data, unsigned int len); + + void flush(); }; diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 8950f73ef25f..6c222420e0ce 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -57,6 +57,7 @@ static void tunnelStderr(const unsigned char * buf, size_t count) try { writeInt(STDERR_NEXT, to); writeString(string((char *) buf, count), to); + to.flush(); } catch (...) { /* Write failed; that means that the other side is gone. */ @@ -200,9 +201,7 @@ static void stopWork(bool success = true, const string & msg = "", unsigned int struct TunnelSink : Sink { Sink & to; - TunnelSink(Sink & to) : to(to) - { - } + TunnelSink(Sink & to) : to(to) { } virtual void operator () (const unsigned char * data, unsigned int len) { @@ -215,9 +214,7 @@ struct TunnelSink : Sink struct TunnelSource : Source { Source & from; - TunnelSource(Source & from) : from(from) - { - } + TunnelSource(Source & from) : from(from) { } virtual void operator () (unsigned char * data, unsigned int len) { @@ -228,6 +225,7 @@ struct TunnelSource : Source writeInt(STDERR_READ, to); writeInt(len, to); + to.flush(); string s = readString(from); if (s.size() != len) throw Error("not enough data"); memcpy(data, (const unsigned char *) s.c_str(), len); @@ -596,8 +594,8 @@ static void processConnection() unsigned int magic = readInt(from); if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); writeInt(WORKER_MAGIC_2, to); - writeInt(PROTOCOL_VERSION, to); + to.flush(); unsigned int clientVersion = readInt(from); /* Send startup error messages to the client. */ @@ -619,9 +617,11 @@ static void processConnection() store = boost::shared_ptr(new LocalStore()); stopWork(); + to.flush(); } catch (Error & e) { stopWork(false, e.msg()); + to.flush(); return; } @@ -652,6 +652,8 @@ static void processConnection() if (!errorAllowed) break; } + to.flush(); + assert(!canSendStderr); }; -- cgit 1.4.1 From 5a1b9ed0aa3a0c240b667dbe504b61b2b68e4d16 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 15 Dec 2011 16:19:53 +0000 Subject: * Refactoring: move sink/source buffering into separate classes. * Buffer the HashSink. This speeds up hashing a bit because it prevents lots of calls to the hash update functions (e.g. nix-hash went from 9.3s to 8.7s of user time on the closure of my /var/run/current-system). --- src/libstore/local-store.cc | 9 ++--- src/libstore/references.cc | 4 +- src/libstore/remote-store.cc | 2 +- src/libutil/hash.cc | 22 +++++----- src/libutil/hash.hh | 5 ++- src/libutil/serialise.cc | 69 ++++++++++++++++++++----------- src/libutil/serialise.hh | 96 +++++++++++++++++++++++++------------------- src/nix-worker/nix-worker.cc | 8 ++-- 8 files changed, 125 insertions(+), 90 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5107a93de299..525e5fc7bbe3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1103,16 +1103,14 @@ struct HashAndWriteSink : Sink HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) { } - virtual void operator () - (const unsigned char * data, unsigned int len) + virtual void operator () (const unsigned char * data, size_t len) { writeSink(data, len); hashSink(data, len); } Hash currentHash() { - HashSink hashSinkClone(hashSink); - return hashSinkClone.finish().first; + return hashSink.currentHash().first; } }; @@ -1201,8 +1199,7 @@ struct HashAndReadSource : Source { hashing = true; } - virtual void operator () - (unsigned char * data, unsigned int len) + virtual void operator () (unsigned char * data, size_t len) { readSource(data, len); if (hashing) hashSink(data, len); diff --git a/src/libstore/references.cc b/src/libstore/references.cc index ade9c9aa20e3..c1f9e3ba7711 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -57,11 +57,11 @@ struct RefScanSink : Sink RefScanSink() : hashSink(htSHA256) { } - void operator () (const unsigned char * data, unsigned int len); + void operator () (const unsigned char * data, size_t len); }; -void RefScanSink::operator () (const unsigned char * data, unsigned int len) +void RefScanSink::operator () (const unsigned char * data, size_t len) { hashSink(data, len); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8269b6a83168..7bf0ad7bd4c7 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -374,7 +374,7 @@ Path RemoteStore::importPath(bool requireSignature, Source & source) openConnection(); writeInt(wopImportPath, to); /* We ignore requireSignature, since the worker forces it to true - anyway. */ + anyway. */ processStderr(0, &source); return readStorePath(from); } diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 53342344107d..bbfe7847fd8a 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -306,21 +306,13 @@ HashSink::HashSink(HashType ht) : ht(ht) start(ht, *ctx); } -HashSink::HashSink(const HashSink & h) -{ - ht = h.ht; - bytes = h.bytes; - ctx = new Ctx; - *ctx = *h.ctx; -} - HashSink::~HashSink() { + bufPos = 0; delete ctx; } -void HashSink::operator () - (const unsigned char * data, unsigned int len) +void HashSink::write(const unsigned char * data, size_t len) { bytes += len; update(ht, *ctx, data, len); @@ -328,11 +320,21 @@ void HashSink::operator () HashResult HashSink::finish() { + flush(); Hash hash(ht); nix::finish(ht, *ctx, hash.hash); return HashResult(hash, bytes); } +HashResult HashSink::currentHash() +{ + flush(); + Ctx ctx2 = *ctx; + Hash hash(ht); + nix::finish(ht, ctx2, hash.hash); + return HashResult(hash, bytes); +} + HashResult hashPath( HashType ht, const Path & path, PathFilter & filter) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index cbdcf4c8d4ff..e0b6478cc418 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -91,7 +91,7 @@ string printHashType(HashType ht); union Ctx; -class HashSink : public Sink +class HashSink : public BufferedSink { private: HashType ht; @@ -102,8 +102,9 @@ public: HashSink(HashType ht); HashSink(const HashSink & h); ~HashSink(); - virtual void operator () (const unsigned char * data, unsigned int len); + void write(const unsigned char * data, size_t len); HashResult finish(); + HashResult currentHash(); }; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index e3a53c0d01dc..a82262704ea3 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -8,7 +8,16 @@ namespace nix { -void FdSink::operator () (const unsigned char * data, unsigned int len) +BufferedSink::~BufferedSink() +{ + /* We can't call flush() here, because C++ for some insane reason + doesn't allow you to call virtual methods from a destructor. */ + assert(!bufPos); + if (buffer) delete[] buffer; +} + + +void BufferedSink::operator () (const unsigned char * data, size_t len) { if (!buffer) buffer = new unsigned char[bufSize]; @@ -16,7 +25,7 @@ void FdSink::operator () (const unsigned char * data, unsigned int len) /* Optimisation: bypass the buffer if the data exceeds the buffer size and there is no unflushed data. */ if (bufPos == 0 && len >= bufSize) { - writeFull(fd, data, len); + write(data, len); break; } /* Otherwise, copy the bytes to the buffer. Flush the buffer @@ -29,31 +38,32 @@ void FdSink::operator () (const unsigned char * data, unsigned int len) } -void FdSink::flush() +void BufferedSink::flush() { - if (fd == -1 || bufPos == 0) return; - writeFull(fd, buffer, bufPos); + if (bufPos == 0) return; + write(buffer, bufPos); bufPos = 0; } -void FdSource::operator () (unsigned char * data, unsigned int len) +void FdSink::write(const unsigned char * data, size_t len) +{ + writeFull(fd, data, len); +} + + +BufferedSource::~BufferedSource() +{ + if (buffer) delete[] buffer; +} + + +void BufferedSource::operator () (unsigned char * data, size_t len) { if (!buffer) buffer = new unsigned char[bufSize]; while (len) { - if (!bufPosIn) { - /* Read as much data as is available (up to the buffer - size). */ - checkInterrupt(); - ssize_t n = read(fd, (char *) buffer, bufSize); - if (n == -1) { - if (errno == EINTR) continue; - throw SysError("reading from file"); - } - if (n == 0) throw EndOfFile("unexpected end-of-file"); - bufPosIn = n; - } + if (!bufPosIn) bufPosIn = read(buffer, bufSize); /* Copy out the data in the buffer. */ size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; @@ -64,7 +74,20 @@ void FdSource::operator () (unsigned char * data, unsigned int len) } -void writePadding(unsigned int len, Sink & sink) +size_t FdSource::read(unsigned char * data, size_t len) +{ + ssize_t n; + do { + checkInterrupt(); + n = ::read(fd, (char *) data, bufSize); + } while (n == -1 && errno == EINTR); + if (n == -1) throw SysError("reading from file"); + if (n == 0) throw EndOfFile("unexpected end-of-file"); + return n; +} + + +void writePadding(size_t len, Sink & sink) { if (len % 8) { unsigned char zero[8]; @@ -103,7 +126,7 @@ void writeLongLong(unsigned long long n, Sink & sink) void writeString(const string & s, Sink & sink) { - unsigned int len = s.length(); + size_t len = s.length(); writeInt(len, sink); sink((const unsigned char *) s.c_str(), len); writePadding(len, sink); @@ -118,11 +141,11 @@ void writeStringSet(const StringSet & ss, Sink & sink) } -void readPadding(unsigned int len, Source & source) +void readPadding(size_t len, Source & source) { if (len % 8) { unsigned char zero[8]; - unsigned int n = 8 - (len % 8); + size_t n = 8 - (len % 8); source(zero, n); for (unsigned int i = 0; i < n; i++) if (zero[i]) throw SerialisationError("non-zero padding"); @@ -162,7 +185,7 @@ unsigned long long readLongLong(Source & source) string readString(Source & source) { - unsigned int len = readInt(source); + size_t len = readInt(source); unsigned char * buf = new unsigned char[len]; AutoDeleteArray d(buf); source(buf, len); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index b8d4d7a849d2..a0588668f037 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -11,7 +11,25 @@ namespace nix { struct Sink { virtual ~Sink() { } - virtual void operator () (const unsigned char * data, unsigned int len) = 0; + virtual void operator () (const unsigned char * data, size_t len) = 0; +}; + + +/* A buffered abstract sink. */ +struct BufferedSink : Sink +{ + size_t bufSize, bufPos; + unsigned char * buffer; + + BufferedSink(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPos(0), buffer(0) { } + ~BufferedSink(); + + void operator () (const unsigned char * data, size_t len); + + void flush(); + + virtual void write(const unsigned char * data, size_t len) = 0; }; @@ -20,56 +38,52 @@ struct Source { virtual ~Source() { } - /* The callee should store exactly *len bytes in the buffer - pointed to by data. It should block if that much data is not - yet available, or throw an error if it is not going to be - available. */ - virtual void operator () (unsigned char * data, unsigned int len) = 0; + /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. + It blocks if that much data is not yet available, or throws an + error if it is not going to be available. */ + virtual void operator () (unsigned char * data, size_t len) = 0; }; -/* A sink that writes data to a file descriptor (using a buffer). */ -struct FdSink : Sink +/* A buffered abstract source. */ +struct BufferedSource : Source { - int fd; - unsigned int bufSize, bufPos; + size_t bufSize, bufPosIn, bufPosOut; unsigned char * buffer; - FdSink() : fd(-1), bufSize(32 * 1024), bufPos(0), buffer(0) { } + BufferedSource(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { } + ~BufferedSource(); - FdSink(int fd, unsigned int bufSize = 32 * 1024) - : fd(fd), bufSize(bufSize), bufPos(0), buffer(0) { } - - ~FdSink() - { - flush(); - if (buffer) delete[] buffer; - } + void operator () (unsigned char * data, size_t len); - void operator () (const unsigned char * data, unsigned int len); - - void flush(); + /* Store up to ‘len’ in the buffer pointed to by ‘data’, and + return the number of bytes stored. If should block until at + least one byte is available. */ + virtual size_t read(unsigned char * data, size_t len) = 0; }; -/* A source that reads data from a file descriptor. */ -struct FdSource : Source +/* A sink that writes data to a file descriptor. */ +struct FdSink : BufferedSink { int fd; - unsigned int bufSize, bufPosIn, bufPosOut; - unsigned char * buffer; - FdSource() : fd(-1), bufSize(32 * 1024), bufPosIn(0), bufPosOut(0), buffer(0) { } - - FdSource(int fd, unsigned int bufSize = 32 * 1024) - : fd(fd), bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { } - - ~FdSource() - { - if (buffer) delete[] buffer; - } + FdSink() : fd(-1) { } + FdSink(int fd) : fd(fd) { } + ~FdSink() { flush(); } - void operator () (unsigned char * data, unsigned int len); + void write(const unsigned char * data, size_t len); +}; + + +/* A source that reads data from a file descriptor. */ +struct FdSource : BufferedSource +{ + int fd; + FdSource() : fd(-1) { } + FdSource(int fd) : fd(fd) { } + size_t read(unsigned char * data, size_t len); }; @@ -77,7 +91,7 @@ struct FdSource : Source struct StringSink : Sink { string s; - virtual void operator () (const unsigned char * data, unsigned int len) + void operator () (const unsigned char * data, size_t len) { s.append((const char *) data, len); } @@ -88,9 +102,9 @@ struct StringSink : Sink struct StringSource : Source { const string & s; - unsigned int pos; + size_t pos; StringSource(const string & _s) : s(_s), pos(0) { } - virtual void operator () (unsigned char * data, unsigned int len) + virtual void operator () (unsigned char * data, size_t len) { s.copy((char *) data, len, pos); pos += len; @@ -100,13 +114,13 @@ struct StringSource : Source }; -void writePadding(unsigned int len, Sink & sink); +void writePadding(size_t len, Sink & sink); void writeInt(unsigned int n, Sink & sink); void writeLongLong(unsigned long long n, Sink & sink); void writeString(const string & s, Sink & sink); void writeStringSet(const StringSet & ss, Sink & sink); -void readPadding(unsigned int len, Source & source); +void readPadding(size_t len, Source & source); unsigned int readInt(Source & source); unsigned long long readLongLong(Source & source); string readString(Source & source); diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 6c222420e0ce..a89852638214 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -202,8 +202,7 @@ struct TunnelSink : Sink { Sink & to; TunnelSink(Sink & to) : to(to) { } - virtual void operator () - (const unsigned char * data, unsigned int len) + virtual void operator () (const unsigned char * data, size_t len) { writeInt(STDERR_WRITE, to); writeString(string((const char *) data, len), to); @@ -215,8 +214,7 @@ struct TunnelSource : Source { Source & from; TunnelSource(Source & from) : from(from) { } - virtual void operator () - (unsigned char * data, unsigned int len) + virtual void operator () (unsigned char * data, size_t len) { /* Careful: we're going to receive data from the client now, so we have to disable the SIGPOLL handler. */ @@ -267,7 +265,7 @@ struct SavingSourceAdapter : Source Source & orig; string s; SavingSourceAdapter(Source & orig) : orig(orig) { } - void operator () (unsigned char * data, unsigned int len) + void operator () (unsigned char * data, size_t len) { orig(data, len); s.append((const char *) data, len); -- cgit 1.4.1 From e0bd307802d13476055f8ba99ab7808de0fd71e5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 16 Dec 2011 19:44:13 +0000 Subject: * Make the import operation through the daemon much more efficient (way fewer roundtrips) by allowing the client to send data in bigger chunks. * Some refactoring. --- src/libstore/local-store.cc | 7 ++++--- src/libstore/remote-store.cc | 6 +++--- src/libstore/worker-protocol.hh | 2 +- src/libutil/serialise.cc | 42 +++++++++++++++++++++++++++++------------ src/libutil/serialise.hh | 31 ++++++++++++++---------------- src/nix-worker/nix-worker.cc | 24 +++++++++++++++-------- 6 files changed, 68 insertions(+), 44 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 525e5fc7bbe3..65b1cdbc87a3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1199,10 +1199,11 @@ struct HashAndReadSource : Source { hashing = true; } - virtual void operator () (unsigned char * data, size_t len) + size_t read(unsigned char * data, size_t len) { - readSource(data, len); - if (hashing) hashSink(data, len); + size_t n = readSource.read(data, len); + if (hashing) hashSink(data, n); + return n; } }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 7bf0ad7bd4c7..e976e8fa57ae 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -501,11 +501,11 @@ void RemoteStore::processStderr(Sink * sink, Source * source) } else if (msg == STDERR_READ) { if (!source) throw Error("no source"); - unsigned int len = readInt(from); + size_t len = readInt(from); unsigned char * buf = new unsigned char[len]; AutoDeleteArray d(buf); - (*source)(buf, len); - writeString(string((const char *) buf, len), to); + size_t n = source->read(buf, len); + writeString(string((const char *) buf, n), to); // !!! inefficient to.flush(); } else { diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index acb8bc8b2948..acabd6ca30d2 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -8,7 +8,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x108 +#define PROTOCOL_VERSION 0x109 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 76f2e721a535..640267a131bf 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -23,8 +23,9 @@ void BufferedSink::operator () (const unsigned char * data, size_t len) while (len) { /* Optimisation: bypass the buffer if the data exceeds the - buffer size and there is no unflushed data. */ - if (bufPos == 0 && len >= bufSize) { + buffer size. */ + if (bufPos + len >= bufSize) { + flush(); write(data, len); break; } @@ -59,29 +60,37 @@ void FdSink::write(const unsigned char * data, size_t len) } +void Source::operator () (unsigned char * data, size_t len) +{ + while (len) { + size_t n = read(data, len); + data += n; len -= n; + } +} + + BufferedSource::~BufferedSource() { if (buffer) delete[] buffer; } -void BufferedSource::operator () (unsigned char * data, size_t len) +size_t BufferedSource::read(unsigned char * data, size_t len) { if (!buffer) buffer = new unsigned char[bufSize]; - while (len) { - if (!bufPosIn) bufPosIn = read(buffer, bufSize); + if (!bufPosIn) bufPosIn = readUnbuffered(buffer, bufSize); - /* Copy out the data in the buffer. */ - size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; - memcpy(data, buffer + bufPosOut, n); - data += n; bufPosOut += n; len -= n; - if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0; - } + /* Copy out the data in the buffer. */ + size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; + memcpy(data, buffer + bufPosOut, n); + bufPosOut += n; + if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0; + return n; } -size_t FdSource::read(unsigned char * data, size_t len) +size_t FdSource::readUnbuffered(unsigned char * data, size_t len) { ssize_t n; do { @@ -94,6 +103,15 @@ size_t FdSource::read(unsigned char * data, size_t len) } +size_t StringSource::read(unsigned char * data, size_t len) +{ + if (pos == s.size()) throw EndOfFile("end of string reached"); + size_t n = s.copy((char *) data, len, pos); + pos += n; + return n; +} + + void writePadding(size_t len, Sink & sink) { if (len % 8) { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index a155f6681e2b..25398b09d7e6 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -24,7 +24,7 @@ struct BufferedSink : Sink BufferedSink(size_t bufSize = 32 * 1024) : bufSize(bufSize), bufPos(0), buffer(0) { } ~BufferedSink(); - + void operator () (const unsigned char * data, size_t len); void flush(); @@ -39,9 +39,14 @@ struct Source virtual ~Source() { } /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. - It blocks if that much data is not yet available, or throws an - error if it is not going to be available. */ - virtual void operator () (unsigned char * data, size_t len) = 0; + It blocks until all the requested data is available, or throws + an error if it is not going to be available. */ + void operator () (unsigned char * data, size_t len); + + /* Store up to ‘len’ in the buffer pointed to by ‘data’, and + return the number of bytes stored. If blocks until at least + one byte is available. */ + virtual size_t read(unsigned char * data, size_t len) = 0; }; @@ -55,12 +60,10 @@ struct BufferedSource : Source : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { } ~BufferedSource(); - void operator () (unsigned char * data, size_t len); + size_t read(unsigned char * data, size_t len); - /* Store up to ‘len’ in the buffer pointed to by ‘data’, and - return the number of bytes stored. If should block until at - least one byte is available. */ - virtual size_t read(unsigned char * data, size_t len) = 0; + /* Underlying read call, to be overriden. */ + virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; }; @@ -83,7 +86,7 @@ struct FdSource : BufferedSource int fd; FdSource() : fd(-1) { } FdSource(int fd) : fd(fd) { } - size_t read(unsigned char * data, size_t len); + size_t readUnbuffered(unsigned char * data, size_t len); }; @@ -104,13 +107,7 @@ struct StringSource : Source const string & s; size_t pos; StringSource(const string & _s) : s(_s), pos(0) { } - virtual void operator () (unsigned char * data, size_t len) - { - s.copy((char *) data, len, pos); - pos += len; - if (pos > s.size()) - throw Error("end of string reached"); - } + size_t read(unsigned char * data, size_t len); }; diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index a89852638214..695e4c38d5d1 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -210,11 +210,11 @@ struct TunnelSink : Sink }; -struct TunnelSource : Source +struct TunnelSource : BufferedSource { Source & from; TunnelSource(Source & from) : from(from) { } - virtual void operator () (unsigned char * data, size_t len) + size_t readUnbuffered(unsigned char * data, size_t len) { /* Careful: we're going to receive data from the client now, so we have to disable the SIGPOLL handler. */ @@ -224,11 +224,16 @@ struct TunnelSource : Source writeInt(STDERR_READ, to); writeInt(len, to); to.flush(); - string s = readString(from); - if (s.size() != len) throw Error("not enough data"); - memcpy(data, (const unsigned char *) s.c_str(), len); + string s = readString(from); // !!! inefficient startWork(); + + if (s.empty()) throw EndOfFile("unexpected end-of-file"); + if (s.size() > len) throw Error("client sent too much data"); + + memcpy(data, (const unsigned char *) s.c_str(), s.size()); + + return s.size(); } }; @@ -265,10 +270,11 @@ struct SavingSourceAdapter : Source Source & orig; string s; SavingSourceAdapter(Source & orig) : orig(orig) { } - void operator () (unsigned char * data, size_t len) + size_t read(unsigned char * data, size_t len) { - orig(data, len); - s.append((const char *) data, len); + size_t n = orig.read(data, len); + s.append((const char *) data, n); + return n; } }; @@ -397,6 +403,8 @@ static void performOp(unsigned int clientVersion, case wopImportPath: { startWork(); + if (GET_PROTOCOL_MINOR(clientVersion) < 9) + throw Error("import not supported; upgrade your client"); TunnelSource source(from); Path path = store->importPath(true, source); stopWork(); -- cgit 1.4.1 From 273b288a7e862ac1918064537ff130cc751fa9fd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 16 Dec 2011 22:31:25 +0000 Subject: * importPath() -> importPaths(). Because of buffering of the input stream it's now necessary for the daemon to process the entire sequence of exported paths, rather than letting the client do it. --- src/libstore/local-store.cc | 17 +++++++++++++++-- src/libstore/local-store.hh | 4 +++- src/libstore/remote-store.cc | 41 +++++++++++++++++++++-------------------- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.hh | 6 +++--- src/libstore/worker-protocol.hh | 4 ++-- src/libutil/serialise.cc | 16 +++++++++++----- src/libutil/serialise.hh | 4 ++-- src/nix-store/nix-store.cc | 10 ++++------ src/nix-worker/nix-worker.cc | 28 +++++++++++++--------------- 10 files changed, 75 insertions(+), 57 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 65b1cdbc87a3..cf0e2ad1b13b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1156,7 +1156,7 @@ void LocalStore::exportPath(const Path & path, bool sign, PathSet references; queryReferences(path, references); - writeStringSet(references, hashAndWriteSink); + writeStrings(references, hashAndWriteSink); Path deriver = queryDeriver(path); writeString(deriver, hashAndWriteSink); @@ -1243,7 +1243,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) Path dstPath = readStorePath(hashAndReadSource); - PathSet references = readStorePaths(hashAndReadSource); + PathSet references = readStorePaths(hashAndReadSource); Path deriver = readString(hashAndReadSource); if (deriver != "") assertStorePath(deriver); @@ -1330,6 +1330,19 @@ Path LocalStore::importPath(bool requireSignature, Source & source) } +Paths LocalStore::importPaths(bool requireSignature, Source & source) +{ + Paths res; + while (true) { + unsigned long long n = readLongLong(source); + if (n == 0) break; + if (n != 1) throw Error("input doesn't look like something created by `nix-store --export'"); + res.push_back(importPath(requireSignature, source)); + } + return res; +} + + void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFreed, unsigned long long & blocksFreed) { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 7ef01b2644b4..4cb905f67231 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -146,7 +146,7 @@ public: void exportPath(const Path & path, bool sign, Sink & sink); - Path importPath(bool requireSignature, Source & source); + Paths importPaths(bool requireSignature, Source & source); void buildDerivations(const PathSet & drvPaths); @@ -259,6 +259,8 @@ private: Path createTempDirInStore(); + Path importPath(bool requireSignature, Source & source); + void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index e976e8fa57ae..942c5bcf1c7c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -27,13 +27,15 @@ Path readStorePath(Source & from) } -PathSet readStorePaths(Source & from) +template T readStorePaths(Source & from) { - PathSet paths = readStringSet(from); - foreach (PathSet::iterator, i, paths) assertStorePath(*i); + T paths = readStrings(from); + foreach (typename T::iterator, i, paths) assertStorePath(*i); return paths; } +template PathSet readStorePaths(Source & from); + RemoteStore::RemoteStore() { @@ -215,7 +217,7 @@ PathSet RemoteStore::queryValidPaths() openConnection(); writeInt(wopQueryValidPaths, to); processStderr(); - return readStorePaths(from); + return readStorePaths(from); } @@ -242,7 +244,7 @@ bool RemoteStore::querySubstitutablePathInfo(const Path & path, if (reply == 0) return false; info.deriver = readString(from); if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths(from); + info.references = readStorePaths(from); info.downloadSize = readLongLong(from); info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0; return true; @@ -260,7 +262,7 @@ ValidPathInfo RemoteStore::queryPathInfo(const Path & path) info.deriver = readString(from); if (info.deriver != "") assertStorePath(info.deriver); info.hash = parseHash(htSHA256, readString(from)); - info.references = readStorePaths(from); + info.references = readStorePaths(from); info.registrationTime = readInt(from); info.narSize = readLongLong(from); return info; @@ -285,7 +287,7 @@ void RemoteStore::queryReferences(const Path & path, writeInt(wopQueryReferences, to); writeString(path, to); processStderr(); - PathSet references2 = readStorePaths(from); + PathSet references2 = readStorePaths(from); references.insert(references2.begin(), references2.end()); } @@ -297,7 +299,7 @@ void RemoteStore::queryReferrers(const Path & path, writeInt(wopQueryReferrers, to); writeString(path, to); processStderr(); - PathSet referrers2 = readStorePaths(from); + PathSet referrers2 = readStorePaths(from); referrers.insert(referrers2.begin(), referrers2.end()); } @@ -320,7 +322,7 @@ PathSet RemoteStore::queryDerivationOutputs(const Path & path) writeInt(wopQueryDerivationOutputs, to); writeString(path, to); processStderr(); - return readStorePaths(from); + return readStorePaths(from); } @@ -350,7 +352,7 @@ Path RemoteStore::addTextToStore(const string & name, const string & s, writeInt(wopAddTextToStore, to); writeString(name, to); writeString(s, to); - writeStringSet(references, to); + writeStrings(references, to); processStderr(); return readStorePath(from); @@ -369,14 +371,14 @@ void RemoteStore::exportPath(const Path & path, bool sign, } -Path RemoteStore::importPath(bool requireSignature, Source & source) +Paths RemoteStore::importPaths(bool requireSignature, Source & source) { openConnection(); - writeInt(wopImportPath, to); + writeInt(wopImportPaths, to); /* We ignore requireSignature, since the worker forces it to true anyway. */ processStderr(0, &source); - return readStorePath(from); + return readStorePaths(from); } @@ -384,7 +386,7 @@ void RemoteStore::buildDerivations(const PathSet & drvPaths) { openConnection(); writeInt(wopBuildDerivations, to); - writeStringSet(drvPaths, to); + writeStrings(drvPaths, to); processStderr(); readInt(from); } @@ -451,7 +453,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) writeInt(wopCollectGarbage, to); writeInt(options.action, to); - writeStringSet(options.pathsToDelete, to); + writeStrings(options.pathsToDelete, to); writeInt(options.ignoreLiveness, to); writeLongLong(options.maxFreed, to); writeInt(options.maxLinks, to); @@ -463,7 +465,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) processStderr(); - results.paths = readStringSet(from); + results.paths = readStrings(from); results.bytesFreed = readLongLong(from); results.blocksFreed = readLongLong(from); } @@ -474,7 +476,7 @@ PathSet RemoteStore::queryFailedPaths() openConnection(); writeInt(wopQueryFailedPaths, to); processStderr(); - return readStorePaths(from); + return readStorePaths(from); } @@ -482,7 +484,7 @@ void RemoteStore::clearFailedPaths(const PathSet & paths) { openConnection(); writeInt(wopClearFailedPaths, to); - writeStringSet(paths, to); + writeStrings(paths, to); processStderr(); readInt(from); } @@ -504,8 +506,7 @@ void RemoteStore::processStderr(Sink * sink, Source * source) size_t len = readInt(from); unsigned char * buf = new unsigned char[len]; AutoDeleteArray d(buf); - size_t n = source->read(buf, len); - writeString(string((const char *) buf, n), to); // !!! inefficient + writeString(buf, source->read(buf, len), to); to.flush(); } else { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 519f46fd1a6d..34a2d91dfcd1 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -56,7 +56,7 @@ public: void exportPath(const Path & path, bool sign, Sink & sink); - Path importPath(bool requireSignature, Source & source); + Paths importPaths(bool requireSignature, Source & source); void buildDerivations(const PathSet & drvPaths); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e3a2c0daa20c..d4997c886207 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -169,9 +169,9 @@ public: virtual void exportPath(const Path & path, bool sign, Sink & sink) = 0; - /* Import a NAR dump created by exportPath() into the Nix - store. */ - virtual Path importPath(bool requireSignature, Source & source) = 0; + /* Import a sequence of NAR dumps created by exportPaths() into + the Nix store. */ + virtual Paths importPaths(bool requireSignature, Source & source) = 0; /* Ensure that the output paths of the derivation are valid. If they are already valid, this is a no-op. Otherwise, validity diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index acabd6ca30d2..760d08a747ae 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -29,7 +29,6 @@ typedef enum { wopSyncWithGC = 13, wopFindRoots = 14, wopExportPath = 16, - wopImportPath = 17, wopQueryDeriver = 18, wopSetOptions = 19, wopCollectGarbage = 20, @@ -39,6 +38,7 @@ typedef enum { wopQueryFailedPaths = 24, wopClearFailedPaths = 25, wopQueryPathInfo = 26, + wopImportPaths = 27, } WorkerOp; @@ -58,7 +58,7 @@ typedef enum { Path readStorePath(Source & from); -PathSet readStorePaths(Source & from); +template T readStorePaths(Source & from); } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index ba549c214f10..c4563ffd1212 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -163,13 +163,16 @@ void writeString(const string & s, Sink & sink) } -void writeStringSet(const StringSet & ss, Sink & sink) +template void writeStrings(const T & ss, Sink & sink) { writeInt(ss.size(), sink); - for (StringSet::iterator i = ss.begin(); i != ss.end(); ++i) + foreach (typename T::const_iterator, i, ss) writeString(*i, sink); } +template void writeStrings(const Paths & ss, Sink & sink); +template void writeStrings(const PathSet & ss, Sink & sink); + void readPadding(size_t len, Source & source) { @@ -234,14 +237,17 @@ string readString(Source & source) } -StringSet readStringSet(Source & source) +template T readStrings(Source & source) { unsigned int count = readInt(source); - StringSet ss; + T ss; while (count--) - ss.insert(readString(source)); + ss.insert(ss.end(), readString(source)); return ss; } +template Paths readStrings(Source & source); +template PathSet readStrings(Source & source); + } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index efd8e2a060eb..ded4b12a046e 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -116,14 +116,14 @@ void writeInt(unsigned int n, Sink & sink); void writeLongLong(unsigned long long n, Sink & sink); void writeString(const unsigned char * buf, size_t len, Sink & sink); void writeString(const string & s, Sink & sink); -void writeStringSet(const StringSet & ss, Sink & sink); +template void writeStrings(const T & ss, Sink & sink); void readPadding(size_t len, Source & source); unsigned int readInt(Source & source); unsigned long long readLongLong(Source & source); size_t readString(unsigned char * buf, size_t max, Source & source); string readString(Source & source); -StringSet readStringSet(Source & source); +template T readStrings(Source & source); MakeError(SerialisationError, Error) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 740033b453d7..e92ccb153d12 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -600,12 +600,10 @@ static void opImport(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); FdSource source(STDIN_FILENO); - while (true) { - unsigned long long n = readLongLong(source); - if (n == 0) break; - if (n != 1) throw Error("input doesn't look like something created by `nix-store --export'"); - cout << format("%1%\n") % store->importPath(requireSignature, source) << std::flush; - } + Paths paths = store->importPaths(requireSignature, source); + + foreach (Paths::iterator, i, paths) + cout << format("%1%\n") % *i << std::flush; } diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 85e2105b2757..5f57b2981d9a 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -327,7 +327,7 @@ static void performOp(unsigned int clientVersion, store->queryReferrers(path, paths); else paths = store->queryDerivationOutputs(path); stopWork(); - writeStringSet(paths, to); + writeStrings(paths, to); break; } @@ -377,7 +377,7 @@ static void performOp(unsigned int clientVersion, case wopAddTextToStore: { string suffix = readString(from); string s = readString(from); - PathSet refs = readStorePaths(from); + PathSet refs = readStorePaths(from); startWork(); Path path = store->addTextToStore(suffix, s, refs); stopWork(); @@ -396,19 +396,17 @@ static void performOp(unsigned int clientVersion, break; } - case wopImportPath: { + case wopImportPaths: { startWork(); - if (GET_PROTOCOL_MINOR(clientVersion) < 9) - throw Error("import not supported; upgrade your client"); TunnelSource source(from); - Path path = store->importPath(true, source); + Paths paths = store->importPaths(true, source); stopWork(); - writeString(path, to); + writeStrings(paths, to); break; } case wopBuildDerivations: { - PathSet drvs = readStorePaths(from); + PathSet drvs = readStorePaths(from); startWork(); store->buildDerivations(drvs); stopWork(); @@ -466,7 +464,7 @@ static void performOp(unsigned int clientVersion, case wopCollectGarbage: { GCOptions options; options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = readStorePaths(from); + options.pathsToDelete = readStorePaths(from); options.ignoreLiveness = readInt(from); options.maxFreed = readLongLong(from); options.maxLinks = readInt(from); @@ -484,7 +482,7 @@ static void performOp(unsigned int clientVersion, store->collectGarbage(options, results); stopWork(); - writeStringSet(results.paths, to); + writeStrings(results.paths, to); writeLongLong(results.bytesFreed, to); writeLongLong(results.blocksFreed, to); @@ -522,7 +520,7 @@ static void performOp(unsigned int clientVersion, writeInt(res ? 1 : 0, to); if (res) { writeString(info.deriver, to); - writeStringSet(info.references, to); + writeStrings(info.references, to); writeLongLong(info.downloadSize, to); if (GET_PROTOCOL_MINOR(clientVersion) >= 7) writeLongLong(info.narSize, to); @@ -534,7 +532,7 @@ static void performOp(unsigned int clientVersion, startWork(); PathSet paths = store->queryValidPaths(); stopWork(); - writeStringSet(paths, to); + writeStrings(paths, to); break; } @@ -542,12 +540,12 @@ static void performOp(unsigned int clientVersion, startWork(); PathSet paths = store->queryFailedPaths(); stopWork(); - writeStringSet(paths, to); + writeStrings(paths, to); break; } case wopClearFailedPaths: { - PathSet paths = readStringSet(from); + PathSet paths = readStrings(from); startWork(); store->clearFailedPaths(paths); stopWork(); @@ -562,7 +560,7 @@ static void performOp(unsigned int clientVersion, stopWork(); writeString(info.deriver, to); writeString(printHash(info.hash), to); - writeStringSet(info.references, to); + writeStrings(info.references, to); writeInt(info.registrationTime, to); writeLongLong(info.narSize, to); break; -- cgit 1.4.1