From 2df9cbeb47177d6a21606b4b509ebcf6bd0437a6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 Dec 2017 00:50:46 +0100 Subject: Provide random access to cached NARs E.g. $ time nix cat-store --store https://cache.nixos.org?local-nar-cache=/tmp/nars \ /nix/store/b0w2hafndl09h64fhb86kw6bmhbmnpm1-blender-2.79/share/icons/hicolor/scalable/apps/blender.svg > /dev/null real 0m4.139s $ time nix cat-store --store https://cache.nixos.org?local-nar-cache=/tmp/nars \ /nix/store/b0w2hafndl09h64fhb86kw6bmhbmnpm1-blender-2.79/share/icons/hicolor/scalable/apps/blender.svg > /dev/null real 0m0.024s (Before, the second call took ~0.220s.) This will use a NAR listing in /tmp/nars/b0w2hafndl09h64fhb86kw6bmhbmnpm1.ls containing all metadata, including the offsets of regular files inside the NAR. Thus, we don't need to read the entire NAR. (We do read the entire listing, but that's generally pretty small. We could use a SQLite DB by borrowing some more code from nixos-channel-scripts/file-cache.hh.) This is primarily useful when Hydra is serving files from an S3 binary cache, in particular when you have giant NARs. E.g. we had some 12 GiB NARs, so accessing individuals files was pretty slow. --- src/libstore/nar-accessor.hh | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'src/libstore/nar-accessor.hh') diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index ed8fe15cad23..1903355a236e 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -8,6 +8,16 @@ namespace nix { file. */ ref makeNarAccessor(ref nar); +/* Create a NAR accessor from a NAR listing (in the format produced by + listNar()). The callback getNarBytes(offset, length) is used by the + readFile() method of the accessor to get the contents of files + inside the NAR. */ +typedef std::function GetNarBytes; + +ref makeLazyNarAccessor( + const std::string & listing, + GetNarBytes getNarBytes); + class JSONPlaceholder; /* Write a JSON representation of the contents of a NAR (except file -- cgit 1.4.1