From 48662d151bdf4a38670897beacea9d1bd750376a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 16 Mar 2018 20:22:34 +0100 Subject: Reduce substitution memory consumption copyStorePath() now pipes the output of srcStore->narFromPath() directly into dstStore->addToStore(). The sink used by the former is converted into a source usable by the latter using boost::coroutine2. This is based on [1]. This reduces the maximum resident size of $ nix build --store ~/my-nix/ /nix/store/b0zlxla7dmy1iwc3g459rjznx59797xy-binutils-2.28.1 --substituters file:///tmp/binary-cache-xz/ --no-require-sigs from 418592 KiB to 53416 KiB. (The previous commit also reduced the runtime from ~4.2s to ~3.4s, not sure why.) A further improvement will be to download files into a Sink. [1] https://github.com/NixOS/nix/compare/master...Mathnerd314:dump-fix-coroutine#diff-dcbcac55a634031f9cc73707da6e4b18 Issue #1969. --- src/libstore/local-store.cc | 34 ++++++++++++++++++++----------- src/libstore/local-store.hh | 2 +- src/libstore/store-api.cc | 49 ++++++++++++++++++++++++++------------------- src/libstore/store-api.hh | 7 ++++++- 4 files changed, 57 insertions(+), 35 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 3441b2472fcf..acc0002acee1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -964,20 +964,11 @@ void LocalStore::invalidatePath(State & state, const Path & path) } -void LocalStore::addToStore(const ValidPathInfo & info, const ref & nar, +void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) { assert(info.narHash); - Hash h = hashString(htSHA256, *nar); - if (h != info.narHash) - throw Error("hash mismatch importing path '%s'; expected hash '%s', got '%s'", - info.path, info.narHash.to_string(), h.to_string()); - - if (nar->size() != info.narSize) - throw Error("size mismatch importing path '%s'; expected %s, got %s", - info.path, info.narSize, nar->size()); - if (requireSigs && checkSigs && !info.checkSignatures(*this, publicKeys)) throw Error("cannot add path '%s' because it lacks a valid signature", info.path); @@ -999,8 +990,27 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref & deletePath(realPath); - StringSource source(*nar); - restorePath(realPath, source); + /* While restoring the path from the NAR, compute the hash + of the NAR. */ + HashSink hashSink(htSHA256); + + LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { + size_t n = source.read(data, len); + hashSink(data, n); + return n; + }); + + restorePath(realPath, wrapperSource); + + auto hashResult = hashSink.finish(); + + if (hashResult.first != info.narHash) + throw Error("hash mismatch importing path '%s'; expected hash '%s', got '%s'", + info.path, info.narHash.to_string(), hashResult.first.to_string()); + + if (hashResult.second != info.narSize) + throw Error("size mismatch importing path '%s'; expected %s, got %s", + info.path, info.narSize, hashResult.second); autoGC(); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index bbd50e1c1451..0d6c176595c8 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -143,7 +143,7 @@ public: void querySubstitutablePathInfos(const PathSet & paths, SubstitutablePathInfos & infos) override; - void addToStore(const ValidPathInfo & info, const ref & nar, + void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8830edcc3449..64f9b8d68b06 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -590,32 +590,15 @@ void copyStorePath(ref srcStore, ref dstStore, uint64_t total = 0; - auto progress = [&](size_t len) { - total += len; - act.progress(total, info->narSize); - }; - - struct MyStringSink : StringSink - { - typedef std::function Callback; - Callback callback; - MyStringSink(Callback callback) : callback(callback) { } - void operator () (const unsigned char * data, size_t len) override - { - StringSink::operator ()(data, len); - callback(len); - }; - }; - - MyStringSink sink(progress); - srcStore->narFromPath({storePath}, sink); - + // FIXME +#if 0 if (!info->narHash) { auto info2 = make_ref(*info); info2->narHash = hashString(htSHA256, *sink.s); if (!info->narSize) info2->narSize = sink.s->size(); info = info2; } +#endif if (info->ultimate) { auto info2 = make_ref(*info); @@ -623,7 +606,16 @@ void copyStorePath(ref srcStore, ref dstStore, info = info2; } - dstStore->addToStore(*info, sink.s, repair, checkSigs); + auto source = sinkToSource([&](Sink & sink) { + LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { + sink(data, len); + total += len; + act.progress(total, info->narSize); + }); + srcStore->narFromPath({storePath}, wrapperSink); + }); + + dstStore->addToStore(*info, *source, repair, checkSigs); } @@ -808,6 +800,21 @@ std::string makeFixedOutputCA(bool recursive, const Hash & hash) } +void Store::addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr accessor) +{ + addToStore(info, make_ref(narSource.drain()), repair, checkSigs, accessor); +} + +void Store::addToStore(const ValidPathInfo & info, const ref & nar, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr accessor) +{ + StringSource source(*nar); + addToStore(info, source, repair, checkSigs, accessor); +} + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 563aa566bd37..ea259f07e8ab 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -399,9 +399,14 @@ public: virtual bool wantMassQuery() { return false; } /* Import a path into the store. */ + virtual void addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, + std::shared_ptr accessor = 0); + + // FIXME: remove virtual void addToStore(const ValidPathInfo & info, const ref & nar, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr accessor = 0) = 0; + std::shared_ptr accessor = 0); /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. -- cgit 1.4.1