diff options
Diffstat (limited to 'src/libstore')
-rw-r--r-- | src/libstore/build.cc | 7 | ||||
-rw-r--r-- | src/libstore/derivations.hh | 6 | ||||
-rw-r--r-- | src/libstore/gc.cc | 37 | ||||
-rw-r--r-- | src/libstore/globals.hh | 6 | ||||
-rw-r--r-- | src/libstore/local-store.cc | 32 | ||||
-rw-r--r-- | src/libstore/local-store.hh | 17 | ||||
-rw-r--r-- | src/libstore/misc.hh | 6 | ||||
-rw-r--r-- | src/libstore/optimise-store.cc | 218 | ||||
-rw-r--r-- | src/libstore/pathlocks.hh | 6 | ||||
-rw-r--r-- | src/libstore/references.hh | 5 | ||||
-rw-r--r-- | src/libstore/remote-store.hh | 6 | ||||
-rw-r--r-- | src/libstore/store-api.hh | 6 | ||||
-rw-r--r-- | src/libstore/worker-protocol.hh | 7 |
13 files changed, 199 insertions, 160 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 4bef6f02dc00..c57a63db69dd 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1533,7 +1533,7 @@ void DerivationGoal::startBuilder() /* Create a temporary directory where the build will take place. */ - tmpDir = createTempDir("", "nix-build-" + baseNameOf(drvPath), false, false); + tmpDir = createTempDir("", "nix-build-" + baseNameOf(drvPath), false, false, 0700); /* For convenience, set an environment pointing to the top build directory. */ @@ -2099,6 +2099,8 @@ void DerivationGoal::computeClosure() if (allowed.find(*i) == allowed.end()) throw BuildError(format("output is not allowed to refer to path `%1%'") % *i); } + + worker.store.optimisePath(path); // FIXME: combine with scanForReferences() } /* Register each output path as valid, and register the sets of @@ -2182,6 +2184,7 @@ void DerivationGoal::deleteTmpDir(bool force) % drvPath % tmpDir); if (buildUser.enabled() && !amPrivileged()) getOwnership(tmpDir); + chmod(tmpDir.c_str(), 0755); } else deletePathWrapped(tmpDir); @@ -2562,6 +2565,8 @@ void SubstitutionGoal::finished() HashResult hash = hashPath(htSHA256, storePath); + worker.store.optimisePath(storePath); // FIXME: combine with hashPath() + ValidPathInfo info2; info2.path = storePath; info2.hash = hash.first; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 27e471d885cc..8f22b4afa6a1 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -1,5 +1,4 @@ -#ifndef __DERIVATIONS_H -#define __DERIVATIONS_H +#pragma once #include <map> @@ -81,6 +80,3 @@ typedef std::map<Path, Hash> DrvHashes; extern DrvHashes drvHashes; } - - -#endif /* !__DERIVATIONS_H */ diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index f6ed7dd2264e..874efe4d32d9 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -436,6 +436,8 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path) { checkInterrupt(); + if (path == linksDir) return true; + struct stat st; if (lstat(path.c_str(), &st)) { if (errno == ENOENT) return true; @@ -569,6 +571,37 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path) } +/* Unlink all files in /nix/store/.links that have a link count of 1, + which indicates that there are no other links and so they can be + safely deleted. FIXME: race condition with optimisePath(): we + might see a link count of 1 just before optimisePath() increases + the link count. */ +void LocalStore::removeUnusedLinks() +{ + AutoCloseDir dir = opendir(linksDir.c_str()); + if (!dir) throw SysError(format("opening directory `%1%'") % linksDir); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + Path path = linksDir + "/" + name; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) + throw SysError(format("statting `%1%'") % path); + + if (st.st_nlink != 1) continue; + + printMsg(lvlTalkative, format("deleting unused link `%1%'") % path); + + if (unlink(path.c_str()) == -1) + throw SysError(format("deleting `%1%'") % path); + } +} + + void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) { GCState state(results); @@ -682,6 +715,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) released. */ foreach (PathSet::iterator, i, state.invalidated) deleteGarbage(state, *i); + + /* Clean up the links directory. */ + printMsg(lvlError, format("deleting unused links...")); + removeUnusedLinks(); } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 12a9b9ca15c0..1c0877a5e1e9 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1,5 +1,4 @@ -#ifndef __GLOBALS_H -#define __GLOBALS_H +#pragma once #include "types.hh" @@ -118,6 +117,3 @@ void setDefaultsFromEnvironment(); } - - -#endif /* !__GLOBALS_H */ diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index aa21774df4a7..ebfcc946716a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -12,6 +12,7 @@ #include <sys/types.h> #include <sys/stat.h> +#include <sys/time.h> #include <unistd.h> #include <utime.h> #include <fcntl.h> @@ -208,6 +209,7 @@ LocalStore::LocalStore(bool reserveSpace) /* Create missing state directories if they don't already exist. */ createDirs(nixStore); + createDirs(linksDir = nixStore + "/.links"); Path profilesDir = nixStateDir + "/profiles"; createDirs(nixStateDir + "/profiles"); createDirs(nixStateDir + "/temproots"); @@ -444,7 +446,7 @@ void canonicalisePathMetaData(const Path & path, bool recurse) throw SysError(format("changing owner of `%1%' to %2%") % path % geteuid()); } - + if (!S_ISLNK(st.st_mode)) { /* Mask out all type related bits. */ @@ -458,14 +460,20 @@ void canonicalisePathMetaData(const Path & path, bool recurse) throw SysError(format("changing mode of `%1%' to %2$o") % path % mode); } - if (st.st_mtime != mtimeStore) { - struct utimbuf utimbuf; - utimbuf.actime = st.st_atime; - utimbuf.modtime = mtimeStore; - if (utime(path.c_str(), &utimbuf) == -1) - throw SysError(format("changing modification time of `%1%'") % path); - } - + } + + if (st.st_mtime != mtimeStore) { + struct timeval times[2]; + times[0].tv_sec = st.st_atime; + times[0].tv_usec = 0; + times[1].tv_sec = mtimeStore; + times[1].tv_usec = 0; +#if HAVE_LUTIMES + if (lutimes(path.c_str(), times) == -1) +#else + if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) +#endif + throw SysError(format("changing modification time of `%1%'") % path); } if (recurse && S_ISDIR(st.st_mode)) { @@ -1134,6 +1142,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, hash.second = dump.size(); } else hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); // FIXME: combine with hashPath() ValidPathInfo info; info.path = dstPath; @@ -1188,6 +1198,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s, canonicalisePathMetaData(dstPath); HashResult hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); ValidPathInfo info; info.path = dstPath; @@ -1423,6 +1435,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source) /* !!! if we were clever, we could prevent the hashPath() here. */ HashResult hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); // FIXME: combine with hashPath() ValidPathInfo info; info.path = dstPath; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 07d8198ecaa5..15dff1d02052 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -1,5 +1,4 @@ -#ifndef __LOCAL_STORE_H -#define __LOCAL_STORE_H +#pragma once #include <string> @@ -86,6 +85,8 @@ private: typedef std::map<Path, RunningSubstituter> RunningSubstituters; RunningSubstituters runningSubstituters; + + Path linksDir; public: @@ -168,8 +169,11 @@ public: /* Optimise the disk space usage of the Nix store by hard-linking files with the same contents. */ - void optimiseStore(bool dryRun, OptimiseStats & stats); + void optimiseStore(OptimiseStats & stats); + /* Optimise a single store path. */ + void optimisePath(const Path & path); + /* Check the integrity of the Nix store. */ void verifyStore(bool checkContents); @@ -260,6 +264,8 @@ private: int openGCLock(LockType lockType); + void removeUnusedLinks(); + void startSubstituter(const Path & substituter, RunningSubstituter & runningSubstituter); @@ -268,6 +274,8 @@ private: Path importPath(bool requireSignature, Source & source); void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); + + void optimisePath_(OptimiseStats & stats, const Path & path); }; @@ -302,6 +310,3 @@ void deletePathWrapped(const Path & path, void deletePathWrapped(const Path & path); } - - -#endif /* !__LOCAL_STORE_H */ diff --git a/src/libstore/misc.hh b/src/libstore/misc.hh index 850c12af4ebc..fe0bbdd799d7 100644 --- a/src/libstore/misc.hh +++ b/src/libstore/misc.hh @@ -1,5 +1,4 @@ -#ifndef __MISC_H -#define __MISC_H +#pragma once #include "derivations.hh" @@ -35,6 +34,3 @@ void queryMissing(StoreAPI & store, const PathSet & targets, } - - -#endif /* !__MISC_H */ diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index a486e66ef59e..84a72604bba2 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -1,6 +1,7 @@ #include "util.hh" #include "local-store.hh" #include "immutable.hh" +#include "globals.hh" #include <sys/types.h> #include <sys/stat.h> @@ -12,9 +13,6 @@ namespace nix { -typedef std::map<Hash, std::pair<Path, ino_t> > HashToPath; - - static void makeWritable(const Path & path) { struct stat st; @@ -51,140 +49,152 @@ struct MakeImmutable }; -static void hashAndLink(bool dryRun, HashToPath & hashToPath, - OptimiseStats & stats, const Path & path) +void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) throw SysError(format("getting attributes of path `%1%'") % path); + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + foreach (Strings::iterator, i, names) + optimisePath_(stats, path + "/" + *i); + return; + } + + /* We can hard link regular files and maybe symlinks. */ + if (!S_ISREG(st.st_mode) +#if CAN_LINK_SYMLINK + x + && !S_ISLNK(st.st_mode) +#endif + ) return; + /* Sometimes SNAFUs can cause files in the Nix store to be modified, in particular when running programs as root under NixOS (example: $fontconfig/var/cache being modified). Skip - those files. */ + those files. FIXME: check the modification time. */ if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path); return; } - /* We can hard link regular files and symlinks. */ - if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) { + /* Hash the file. Note that hashPath() returns the hash over the + NAR serialisation, which includes the execute bit on the file. + Thus, executable and non-executable files with the same + contents *won't* be linked (which is good because otherwise the + permissions would be screwed up). - /* Hash the file. Note that hashPath() returns the hash over - the NAR serialisation, which includes the execute bit on - the file. Thus, executable and non-executable files with - the same contents *won't* be linked (which is good because - otherwise the permissions would be screwed up). + Also note that if `path' is a symlink, then we're hashing the + contents of the symlink (i.e. the result of readlink()), not + the contents of the target (which may not even exist). */ + Hash hash = hashPath(htSHA256, path).first; + stats.totalFiles++; + printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); - Also note that if `path' is a symlink, then we're hashing - the contents of the symlink (i.e. the result of - readlink()), not the contents of the target (which may not - even exist). */ - Hash hash = hashPath(htSHA256, path).first; - stats.totalFiles++; - printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); + /* Check if this is a known hash. */ + Path linkPath = linksDir + "/" + printHash32(hash); - std::pair<Path, ino_t> prevPath = hashToPath[hash]; - - if (prevPath.first == "") { - hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino); - return; - } - - /* Yes! We've seen a file with the same contents. Replace - the current file with a hard link to that file. */ - stats.sameContents++; - if (prevPath.second == st.st_ino) { - printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % prevPath.first); - return; - } + if (!pathExists(linkPath)) { + /* Nope, create a hard link in the links directory. */ + makeMutable(path); + MakeImmutable mk1(path); + + if (link(path.c_str(), linkPath.c_str()) == -1) + throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path); + + return; + } + + /* Yes! We've seen a file with the same contents. Replace the + current file with a hard link to that file. */ + struct stat stLink; + if (lstat(linkPath.c_str(), &stLink)) + throw SysError(format("getting attributes of path `%1%'") % linkPath); + + stats.sameContents++; + if (st.st_ino == stLink.st_ino) { + printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % linkPath); + return; + } - if (!dryRun) { - - printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % prevPath.first); + printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % linkPath); - Path tempLink = (format("%1%.tmp-%2%-%3%") - % path % getpid() % rand()).str(); + Path tempLink = (format("%1%/.tmp-link-%2%-%3%") + % nixStore % getpid() % rand()).str(); - /* Make the containing directory writable, but only if - it's not the store itself (we don't want or need to - mess with its permissions). */ - bool mustToggle = !isStorePath(path); - if (mustToggle) makeWritable(dirOf(path)); + /* Make the containing directory writable, but only if it's not + the store itself (we don't want or need to mess with its + permissions). */ + bool mustToggle = !isStorePath(path); + if (mustToggle) makeWritable(dirOf(path)); - /* When we're done, make the directory read-only again and - reset its timestamp back to 0. */ - MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); - - /* If ‘prevPath’ is immutable, we can't create hard links - to it, so make it mutable first (and make it immutable - again when we're done). We also have to make ‘path’ - mutable, otherwise rename() will fail to delete it. */ - makeMutable(prevPath.first); - MakeImmutable mk1(prevPath.first); - - makeMutable(path); - MakeImmutable mk2(path); - - if (link(prevPath.first.c_str(), tempLink.c_str()) == -1) { - if (errno == EMLINK) { - /* Too many links to the same file (>= 32000 on - most file systems). This is likely to happen - with empty files. Just start over, creating - links to the current file. */ - printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first); - hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino); - return; - } - throw SysError(format("cannot link `%1%' to `%2%'") - % tempLink % prevPath.first); - } - - /* Atomically replace the old file with the new hard link. */ - if (rename(tempLink.c_str(), path.c_str()) == -1) { - if (errno == EMLINK) { - /* Some filesystems generate too many links on the - rename, rather than on the original link. - (Probably it temporarily increases the st_nlink - field before decreasing it again.) */ - printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first); - hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino); - - /* Unlink the temp link. */ - if (unlink(tempLink.c_str()) == -1) - printMsg(lvlError, format("unable to unlink `%1%'") % tempLink); - return; - } - throw SysError(format("cannot rename `%1%' to `%2%'") - % tempLink % path); - } - } else - printMsg(lvlTalkative, format("would link `%1%' to `%2%'") % path % prevPath.first); - - stats.filesLinked++; - stats.bytesFreed += st.st_size; - stats.blocksFreed += st.st_blocks; + /* When we're done, make the directory read-only again and reset + its timestamp back to 0. */ + MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); + + /* If ‘linkPath’ is immutable, we can't create hard links to it, + so make it mutable first (and make it immutable again when + we're done). We also have to make ‘path’ mutable, otherwise + rename() will fail to delete it. */ + makeMutable(linkPath); + MakeImmutable mk1(linkPath); + + makeMutable(path); + MakeImmutable mk2(path); + + if (link(linkPath.c_str(), tempLink.c_str()) == -1) { + if (errno == EMLINK) { + /* Too many links to the same file (>= 32000 on most file + systems). This is likely to happen with empty files. + Just shrug and ignore. */ + printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); + return; + } + throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath); } - if (S_ISDIR(st.st_mode)) { - Strings names = readDirectory(path); - foreach (Strings::iterator, i, names) - hashAndLink(dryRun, hashToPath, stats, path + "/" + *i); + /* Atomically replace the old file with the new hard link. */ + if (rename(tempLink.c_str(), path.c_str()) == -1) { + if (errno == EMLINK) { + /* Some filesystems generate too many links on the rename, + rather than on the original link. (Probably it + temporarily increases the st_nlink field before + decreasing it again.) */ + printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); + + /* Unlink the temp link. */ + if (unlink(tempLink.c_str()) == -1) + printMsg(lvlError, format("unable to unlink `%1%'") % tempLink); + return; + } + throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path); } + + stats.filesLinked++; + stats.bytesFreed += st.st_size; + stats.blocksFreed += st.st_blocks; } -void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats) +void LocalStore::optimiseStore(OptimiseStats & stats) { - HashToPath hashToPath; - PathSet paths = queryAllValidPaths(); foreach (PathSet::iterator, i, paths) { addTempRoot(*i); if (!isValidPath(*i)) continue; /* path was GC'ed, probably */ startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i); - hashAndLink(dryRun, hashToPath, stats, *i); + optimisePath_(stats, *i); + } +} + + +void LocalStore::optimisePath(const Path & path) +{ + if (queryBoolSetting("auto-optimise-store", true)) { + OptimiseStats stats; + optimisePath_(stats, path); } } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 57ca1584a6d6..8a6b1450da2a 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,5 +1,4 @@ -#ifndef __PATHLOCKS_H -#define __PATHLOCKS_H +#pragma once #include "types.hh" @@ -44,6 +43,3 @@ bool pathIsLockedByMe(const Path & path); } - - -#endif /* !__PATHLOCKS_H */ diff --git a/src/libstore/references.hh b/src/libstore/references.hh index 158c08a77646..013809d122f3 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -1,5 +1,4 @@ -#ifndef __REFERENCES_H -#define __REFERENCES_H +#pragma once #include "types.hh" #include "hash.hh" @@ -10,5 +9,3 @@ PathSet scanForReferences(const Path & path, const PathSet & refs, HashResult & hash); } - -#endif /* !__REFERENCES_H */ diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 68db0640ae70..ae4c48dad6ab 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -1,5 +1,4 @@ -#ifndef __REMOTE_STORE_H -#define __REMOTE_STORE_H +#pragma once #include <string> @@ -103,6 +102,3 @@ private: } - - -#endif /* !__REMOTE_STORE_H */ diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 5d8c09f5a526..324d802dc450 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,5 +1,4 @@ -#ifndef __STOREAPI_H -#define __STOREAPI_H +#pragma once #include "hash.hh" #include "serialise.hh" @@ -362,6 +361,3 @@ MakeError(BuildError, Error) /* denotes a permanent build failure */ } - - -#endif /* !__STOREAPI_H */ diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index d4ac0ea16fed..9677a46c2896 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,6 +1,4 @@ -#ifndef __WORKER_PROTOCOL_H -#define __WORKER_PROTOCOL_H - +#pragma once namespace nix { @@ -67,6 +65,3 @@ template<class T> T readStorePaths(Source & from); } - - -#endif /* !__WORKER_PROTOCOL_H */ |