diff options
Diffstat (limited to 'third_party/nix/src/libstore/optimise-store.cc')
| -rw-r--r-- | third_party/nix/src/libstore/optimise-store.cc | 469 |
1 files changed, 226 insertions, 243 deletions
diff --git a/third_party/nix/src/libstore/optimise-store.cc b/third_party/nix/src/libstore/optimise-store.cc index 991512f21795..b43919a2bb40 100644 --- a/third_party/nix/src/libstore/optimise-store.cc +++ b/third_party/nix/src/libstore/optimise-store.cc @@ -1,302 +1,285 @@ -#include "util.hh" -#include "local-store.hh" -#include "globals.hh" - -#include <cstdlib> -#include <cstring> -#include <sys/types.h> -#include <sys/stat.h> -#include <unistd.h> #include <errno.h> #include <stdio.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> +#include <cstdlib> +#include <cstring> #include <regex> - +#include "globals.hh" +#include "local-store.hh" +#include "util.hh" namespace nix { - -static void makeWritable(const Path & path) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); - if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError(format("changing writability of '%1%'") % path); +static void makeWritable(const Path& path) { + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % path); + if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) + throw SysError(format("changing writability of '%1%'") % path); } - -struct MakeReadOnly -{ - Path path; - MakeReadOnly(const Path & path) : path(path) { } - ~MakeReadOnly() - { - try { - /* This will make the path read-only. */ - if (path != "") canonicaliseTimestampAndPermissions(path); - } catch (...) { - ignoreException(); - } +struct MakeReadOnly { + Path path; + MakeReadOnly(const Path& path) : path(path) {} + ~MakeReadOnly() { + try { + /* This will make the path read-only. */ + if (path != "") canonicaliseTimestampAndPermissions(path); + } catch (...) { + ignoreException(); } + } }; +LocalStore::InodeHash LocalStore::loadInodeHash() { + debug("loading hash inodes in memory"); + InodeHash inodeHash; -LocalStore::InodeHash LocalStore::loadInodeHash() -{ - debug("loading hash inodes in memory"); - InodeHash inodeHash; - - AutoCloseDir dir(opendir(linksDir.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); + AutoCloseDir dir(opendir(linksDir.c_str())); + if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { /* sic */ - checkInterrupt(); - // We don't care if we hit non-hash files, anything goes - inodeHash.insert(dirent->d_ino); - } - if (errno) throw SysError(format("reading directory '%1%'") % linksDir); + struct dirent* dirent; + while (errno = 0, dirent = readdir(dir.get())) { /* sic */ + checkInterrupt(); + // We don't care if we hit non-hash files, anything goes + inodeHash.insert(dirent->d_ino); + } + if (errno) throw SysError(format("reading directory '%1%'") % linksDir); - printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size()); + printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size()); - return inodeHash; + return inodeHash; } +Strings LocalStore::readDirectoryIgnoringInodes(const Path& path, + const InodeHash& inodeHash) { + Strings names; -Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash) -{ - Strings names; - - AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % path); + AutoCloseDir dir(opendir(path.c_str())); + if (!dir) throw SysError(format("opening directory '%1%'") % path); - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { /* sic */ - checkInterrupt(); - - if (inodeHash.count(dirent->d_ino)) { - debug(format("'%1%' is already linked") % dirent->d_name); - continue; - } + struct dirent* dirent; + while (errno = 0, dirent = readdir(dir.get())) { /* sic */ + checkInterrupt(); - string name = dirent->d_name; - if (name == "." || name == "..") continue; - names.push_back(name); + if (inodeHash.count(dirent->d_ino)) { + debug(format("'%1%' is already linked") % dirent->d_name); + continue; } - if (errno) throw SysError(format("reading directory '%1%'") % path); - return names; + string name = dirent->d_name; + if (name == "." || name == "..") continue; + names.push_back(name); + } + if (errno) throw SysError(format("reading directory '%1%'") % path); + + return names; } +void LocalStore::optimisePath_(Activity* act, OptimiseStats& stats, + const Path& path, InodeHash& inodeHash) { + checkInterrupt(); -void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, - const Path & path, InodeHash & inodeHash) -{ - checkInterrupt(); - - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % path); #if __APPLE__ - /* HFS/macOS has some undocumented security feature disabling hardlinking for - special files within .app dirs. *.app/Contents/PkgInfo and - *.app/Contents/Resources/\*.lproj seem to be the only paths affected. See - https://github.com/NixOS/nix/issues/1443 for more discussion. */ - - if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) - { - debug(format("'%1%' is not allowed to be linked in macOS") % path); - return; - } + /* HFS/macOS has some undocumented security feature disabling hardlinking for + special files within .app dirs. *.app/Contents/PkgInfo and + *.app/Contents/Resources/\*.lproj seem to be the only paths affected. See + https://github.com/NixOS/nix/issues/1443 for more discussion. */ + + if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) { + debug(format("'%1%' is not allowed to be linked in macOS") % path); + return; + } #endif - if (S_ISDIR(st.st_mode)) { - Strings names = readDirectoryIgnoringInodes(path, inodeHash); - for (auto & i : names) - optimisePath_(act, stats, path + "/" + i, inodeHash); - return; - } + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectoryIgnoringInodes(path, inodeHash); + for (auto& i : names) optimisePath_(act, stats, path + "/" + i, inodeHash); + return; + } - /* We can hard link regular files and maybe symlinks. */ - if (!S_ISREG(st.st_mode) + /* We can hard link regular files and maybe symlinks. */ + if (!S_ISREG(st.st_mode) #if CAN_LINK_SYMLINK - && !S_ISLNK(st.st_mode) + && !S_ISLNK(st.st_mode) #endif - ) return; - - /* Sometimes SNAFUs can cause files in the Nix store to be - modified, in particular when running programs as root under - NixOS (example: $fontconfig/var/cache being modified). Skip - those files. FIXME: check the modification time. */ - if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { - printError(format("skipping suspicious writable file '%1%'") % path); - return; + ) + return; + + /* Sometimes SNAFUs can cause files in the Nix store to be + modified, in particular when running programs as root under + NixOS (example: $fontconfig/var/cache being modified). Skip + those files. FIXME: check the modification time. */ + if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { + printError(format("skipping suspicious writable file '%1%'") % path); + return; + } + + /* This can still happen on top-level files. */ + if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) { + debug(format("'%1%' is already linked, with %2% other file(s)") % path % + (st.st_nlink - 2)); + return; + } + + /* Hash the file. Note that hashPath() returns the hash over the + NAR serialisation, which includes the execute bit on the file. + Thus, executable and non-executable files with the same + contents *won't* be linked (which is good because otherwise the + permissions would be screwed up). + + Also note that if `path' is a symlink, then we're hashing the + contents of the symlink (i.e. the result of readlink()), not + the contents of the target (which may not even exist). */ + Hash hash = hashPath(htSHA256, path).first; + debug(format("'%1%' has hash '%2%'") % path % hash.to_string()); + + /* Check if this is a known hash. */ + Path linkPath = linksDir + "/" + hash.to_string(Base32, false); + +retry: + if (!pathExists(linkPath)) { + /* Nope, create a hard link in the links directory. */ + if (link(path.c_str(), linkPath.c_str()) == 0) { + inodeHash.insert(st.st_ino); + return; } - /* This can still happen on top-level files. */ - if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) { - debug(format("'%1%' is already linked, with %2% other file(s)") % path % (st.st_nlink - 2)); + switch (errno) { + case EEXIST: + /* Fall through if another process created ‘linkPath’ before + we did. */ + break; + + case ENOSPC: + /* On ext4, that probably means the directory index is + full. When that happens, it's fine to ignore it: we + just effectively disable deduplication of this + file. */ + printInfo("cannot link '%s' to '%s': %s", linkPath, path, + strerror(errno)); return; - } - /* Hash the file. Note that hashPath() returns the hash over the - NAR serialisation, which includes the execute bit on the file. - Thus, executable and non-executable files with the same - contents *won't* be linked (which is good because otherwise the - permissions would be screwed up). - - Also note that if `path' is a symlink, then we're hashing the - contents of the symlink (i.e. the result of readlink()), not - the contents of the target (which may not even exist). */ - Hash hash = hashPath(htSHA256, path).first; - debug(format("'%1%' has hash '%2%'") % path % hash.to_string()); - - /* Check if this is a known hash. */ - Path linkPath = linksDir + "/" + hash.to_string(Base32, false); - - retry: - if (!pathExists(linkPath)) { - /* Nope, create a hard link in the links directory. */ - if (link(path.c_str(), linkPath.c_str()) == 0) { - inodeHash.insert(st.st_ino); - return; - } - - switch (errno) { - case EEXIST: - /* Fall through if another process created ‘linkPath’ before - we did. */ - break; - - case ENOSPC: - /* On ext4, that probably means the directory index is - full. When that happens, it's fine to ignore it: we - just effectively disable deduplication of this - file. */ - printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno)); - return; - - default: - throw SysError("cannot link '%1%' to '%2%'", linkPath, path); - } + default: + throw SysError("cannot link '%1%' to '%2%'", linkPath, path); } - - /* Yes! We've seen a file with the same contents. Replace the - current file with a hard link to that file. */ - struct stat stLink; - if (lstat(linkPath.c_str(), &stLink)) - throw SysError(format("getting attributes of path '%1%'") % linkPath); - - if (st.st_ino == stLink.st_ino) { - debug(format("'%1%' is already linked to '%2%'") % path % linkPath); - return; - } - - if (st.st_size != stLink.st_size) { - printError(format("removing corrupted link '%1%'") % linkPath); - unlink(linkPath.c_str()); - goto retry; - } - - printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath); - - /* Make the containing directory writable, but only if it's not - the store itself (we don't want or need to mess with its - permissions). */ - bool mustToggle = dirOf(path) != realStoreDir; - if (mustToggle) makeWritable(dirOf(path)); - - /* When we're done, make the directory read-only again and reset - its timestamp back to 0. */ - MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); - - Path tempLink = (format("%1%/.tmp-link-%2%-%3%") - % realStoreDir % getpid() % random()).str(); - - if (link(linkPath.c_str(), tempLink.c_str()) == -1) { - if (errno == EMLINK) { - /* Too many links to the same file (>= 32000 on most file - systems). This is likely to happen with empty files. - Just shrug and ignore. */ - if (st.st_size) - printInfo(format("'%1%' has maximum number of links") % linkPath); - return; - } - throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath); + } + + /* Yes! We've seen a file with the same contents. Replace the + current file with a hard link to that file. */ + struct stat stLink; + if (lstat(linkPath.c_str(), &stLink)) + throw SysError(format("getting attributes of path '%1%'") % linkPath); + + if (st.st_ino == stLink.st_ino) { + debug(format("'%1%' is already linked to '%2%'") % path % linkPath); + return; + } + + if (st.st_size != stLink.st_size) { + printError(format("removing corrupted link '%1%'") % linkPath); + unlink(linkPath.c_str()); + goto retry; + } + + printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath); + + /* Make the containing directory writable, but only if it's not + the store itself (we don't want or need to mess with its + permissions). */ + bool mustToggle = dirOf(path) != realStoreDir; + if (mustToggle) makeWritable(dirOf(path)); + + /* When we're done, make the directory read-only again and reset + its timestamp back to 0. */ + MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); + + Path tempLink = + (format("%1%/.tmp-link-%2%-%3%") % realStoreDir % getpid() % random()) + .str(); + + if (link(linkPath.c_str(), tempLink.c_str()) == -1) { + if (errno == EMLINK) { + /* Too many links to the same file (>= 32000 on most file + systems). This is likely to happen with empty files. + Just shrug and ignore. */ + if (st.st_size) + printInfo(format("'%1%' has maximum number of links") % linkPath); + return; } - - /* Atomically replace the old file with the new hard link. */ - if (rename(tempLink.c_str(), path.c_str()) == -1) { - if (unlink(tempLink.c_str()) == -1) - printError(format("unable to unlink '%1%'") % tempLink); - if (errno == EMLINK) { - /* Some filesystems generate too many links on the rename, - rather than on the original link. (Probably it - temporarily increases the st_nlink field before - decreasing it again.) */ - debug("'%s' has reached maximum number of links", linkPath); - return; - } - throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % path); + throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath); + } + + /* Atomically replace the old file with the new hard link. */ + if (rename(tempLink.c_str(), path.c_str()) == -1) { + if (unlink(tempLink.c_str()) == -1) + printError(format("unable to unlink '%1%'") % tempLink); + if (errno == EMLINK) { + /* Some filesystems generate too many links on the rename, + rather than on the original link. (Probably it + temporarily increases the st_nlink field before + decreasing it again.) */ + debug("'%s' has reached maximum number of links", linkPath); + return; } + throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % path); + } - stats.filesLinked++; - stats.bytesFreed += st.st_size; - stats.blocksFreed += st.st_blocks; + stats.filesLinked++; + stats.bytesFreed += st.st_size; + stats.blocksFreed += st.st_blocks; - if (act) - act->result(resFileLinked, st.st_size, st.st_blocks); + if (act) act->result(resFileLinked, st.st_size, st.st_blocks); } +void LocalStore::optimiseStore(OptimiseStats& stats) { + Activity act(*logger, actOptimiseStore); -void LocalStore::optimiseStore(OptimiseStats & stats) -{ - Activity act(*logger, actOptimiseStore); - - PathSet paths = queryAllValidPaths(); - InodeHash inodeHash = loadInodeHash(); + PathSet paths = queryAllValidPaths(); + InodeHash inodeHash = loadInodeHash(); - act.progress(0, paths.size()); + act.progress(0, paths.size()); - uint64_t done = 0; + uint64_t done = 0; - for (auto & i : paths) { - addTempRoot(i); - if (!isValidPath(i)) continue; /* path was GC'ed, probably */ - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", i)); - optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash); - } - done++; - act.progress(done, paths.size()); + for (auto& i : paths) { + addTempRoot(i); + if (!isValidPath(i)) continue; /* path was GC'ed, probably */ + { + Activity act(*logger, lvlTalkative, actUnknown, + fmt("optimising path '%s'", i)); + optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash); } + done++; + act.progress(done, paths.size()); + } } -static string showBytes(unsigned long long bytes) -{ - return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); +static string showBytes(unsigned long long bytes) { + return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); } -void LocalStore::optimiseStore() -{ - OptimiseStats stats; +void LocalStore::optimiseStore() { + OptimiseStats stats; - optimiseStore(stats); + optimiseStore(stats); - printInfo( - format("%1% freed by hard-linking %2% files") - % showBytes(stats.bytesFreed) - % stats.filesLinked); + printInfo(format("%1% freed by hard-linking %2% files") % + showBytes(stats.bytesFreed) % stats.filesLinked); } -void LocalStore::optimisePath(const Path & path) -{ - OptimiseStats stats; - InodeHash inodeHash; +void LocalStore::optimisePath(const Path& path) { + OptimiseStats stats; + InodeHash inodeHash; - if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash); + if (settings.autoOptimiseStore) + optimisePath_(nullptr, stats, path, inodeHash); } - -} +} // namespace nix |