From 48662d151bdf4a38670897beacea9d1bd750376a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 16 Mar 2018 20:22:34 +0100 Subject: Reduce substitution memory consumption copyStorePath() now pipes the output of srcStore->narFromPath() directly into dstStore->addToStore(). The sink used by the former is converted into a source usable by the latter using boost::coroutine2. This is based on [1]. This reduces the maximum resident size of $ nix build --store ~/my-nix/ /nix/store/b0zlxla7dmy1iwc3g459rjznx59797xy-binutils-2.28.1 --substituters file:///tmp/binary-cache-xz/ --no-require-sigs from 418592 KiB to 53416 KiB. (The previous commit also reduced the runtime from ~4.2s to ~3.4s, not sure why.) A further improvement will be to download files into a Sink. [1] https://github.com/NixOS/nix/compare/master...Mathnerd314:dump-fix-coroutine#diff-dcbcac55a634031f9cc73707da6e4b18 Issue #1969. --- src/libutil/serialise.hh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'src/libutil/serialise.hh') diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 103b05767363..d0b4552e3399 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -61,6 +61,8 @@ struct Source virtual size_t read(unsigned char * data, size_t len) = 0; virtual bool good() { return true; } + + std::string drain(); }; @@ -191,6 +193,27 @@ struct LambdaSink : Sink }; +/* Convert a function into a source. */ +struct LambdaSource : Source +{ + typedef std::function lambda_t; + + lambda_t lambda; + + LambdaSource(const lambda_t & lambda) : lambda(lambda) { } + + size_t read(unsigned char * data, size_t len) override + { + return lambda(data, len); + } +}; + + +/* Convert a function that feeds data into a Sink into a Source. The + Source executes the function as a coroutine. */ +std::unique_ptr sinkToSource(std::function fun); + + void writePadding(size_t len, Sink & sink); void writeString(const unsigned char * buf, size_t len, Sink & sink); -- cgit 1.4.1