From bcc9943cee4d4f62e0bceb4245d390fb60744eae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2016 15:20:10 +0100 Subject: importPaths(): Optionally add NARs to binary cache accessor This enables an optimisation in hydra-queue-runner, preventing a download of a NAR it just uploaded to the cache when reading files like hydra-build-products. --- src/libstore/binary-cache-store.cc | 66 +++++++++++++++++++++----------------- src/libstore/binary-cache-store.hh | 5 +-- src/libstore/local-store.cc | 3 +- src/libstore/local-store.hh | 3 +- src/libstore/remote-store.cc | 3 +- src/libstore/remote-store.hh | 3 +- src/libstore/store-api.hh | 7 ++-- 7 files changed, 52 insertions(+), 38 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 6a4e3a560578..02e73d2ce1fa 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -183,7 +183,8 @@ void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink sink << exportMagic << storePath << res.references << res.deriver << 0; } -Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source) +Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) { assert(!requireSignature); Paths res; @@ -191,7 +192,7 @@ Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source) unsigned long long n = readLongLong(source); if (n == 0) break; if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’"); - res.push_back(importPath(source)); + res.push_back(importPath(source, accessor)); } return res; } @@ -215,34 +216,6 @@ struct NopSink : ParseSink { }; -Path BinaryCacheStore::importPath(Source & source) -{ - /* FIXME: some cut&paste of LocalStore::importPath(). */ - - /* Extract the NAR from the source. */ - TeeSource tee(source); - NopSink sink; - parseDump(sink, tee); - - uint32_t magic = readInt(source); - if (magic != exportMagic) - throw Error("Nix archive cannot be imported; wrong format"); - - ValidPathInfo info; - info.path = readStorePath(source); - - info.references = readStorePaths(source); - - readString(source); // deriver, don't care - - bool haveSignature = readInt(source) == 1; - assert(!haveSignature); - - addToCache(info, tee.data); - - return info.path; -} - ValidPathInfo BinaryCacheStore::queryPathInfo(const Path & storePath) { return ValidPathInfo(readNarInfo(storePath)); @@ -416,4 +389,37 @@ ref BinaryCacheStore::getFSAccessor() std::dynamic_pointer_cast(shared_from_this()))); } +Path BinaryCacheStore::importPath(Source & source, std::shared_ptr accessor) +{ + /* FIXME: some cut&paste of LocalStore::importPath(). */ + + /* Extract the NAR from the source. */ + TeeSource tee(source); + NopSink sink; + parseDump(sink, tee); + + uint32_t magic = readInt(source); + if (magic != exportMagic) + throw Error("Nix archive cannot be imported; wrong format"); + + ValidPathInfo info; + info.path = readStorePath(source); + + info.references = readStorePaths(source); + + readString(source); // deriver, don't care + + bool haveSignature = readInt(source) == 1; + assert(!haveSignature); + + addToCache(info, tee.data); + + auto accessor_ = std::dynamic_pointer_cast(accessor); + if (accessor_) + // FIXME: more gratuitous string copying + accessor_->nars.emplace(info.path, makeNarAccessor(make_ref(tee.data))); + + return info.path; +} + } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index c6f319d1bc82..6feb84cd2b10 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -127,9 +127,10 @@ public: void exportPath(const Path & path, bool sign, Sink & sink) override; - Paths importPaths(bool requireSignature, Source & source) override; + Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) override; - Path importPath(Source & source); + Path importPath(Source & source, std::shared_ptr accessor); void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 33f2569128aa..9a570668128e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1689,7 +1689,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source) } -Paths LocalStore::importPaths(bool requireSignature, Source & source) +Paths LocalStore::importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) { Paths res; while (true) { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index fded63ab38e1..c7ea9e5038b4 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -147,7 +147,8 @@ public: void exportPath(const Path & path, bool sign, Sink & sink) override; - Paths importPaths(bool requireSignature, Source & source) override; + Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) override; void buildPaths(const PathSet & paths, BuildMode buildMode) override; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index a1ac169c2b9e..82b7cfd7c17d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -406,7 +406,8 @@ void RemoteStore::exportPath(const Path & path, bool sign, } -Paths RemoteStore::importPaths(bool requireSignature, Source & source) +Paths RemoteStore::importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) { auto conn(connections->get()); conn->to << wopImportPaths; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 0019cd8f9f94..85c8292c7698 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -65,7 +65,8 @@ public: void exportPath(const Path & path, bool sign, Sink & sink) override; - Paths importPaths(bool requireSignature, Source & source) override; + Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) override; void buildPaths(const PathSet & paths, BuildMode buildMode) override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 347387f03403..488f32e16456 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -227,8 +227,11 @@ public: void exportPaths(const Paths & paths, bool sign, Sink & sink); /* Import a sequence of NAR dumps created by exportPaths() into - the Nix store. */ - virtual Paths importPaths(bool requireSignature, Source & source) = 0; + the Nix store. Optionally, the contents of the NARs are + preloaded into the specified FS accessor to speed up subsequent + access. */ + virtual Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) = 0; /* For each path, if it's a derivation, build it. Building a derivation means ensuring that the output paths are valid. If -- cgit 1.4.1