about summary refs log tree commit diff
path: root/src/libstore/optimise-store.cc
blob: a486e66ef59e73f42caa79f0a03e76afc1e00aa7 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
#include "util.hh"
#include "local-store.hh"
#include "immutable.hh"

#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <errno.h>
#include <stdio.h>


namespace nix {


typedef std::map<Hash, std::pair<Path, ino_t> > HashToPath;


static void makeWritable(const Path & path)
{
    struct stat st;
    if (lstat(path.c_str(), &st))
	throw SysError(format("getting attributes of path `%1%'") % path);
    if (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode)) makeMutable(path);
    if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
        throw SysError(format("changing writability of `%1%'") % path);
}


struct MakeReadOnly
{
    Path path;
    MakeReadOnly(const Path & path) : path(path) { }
    ~MakeReadOnly()
    {
        try {
            /* This will make the path read-only (and restore the
               immutable bit on platforms that support it). */
            if (path != "") canonicalisePathMetaData(path, false);
        } catch (...) {
            ignoreException();
        }
    }
};


struct MakeImmutable
{
    Path path;
    MakeImmutable(const Path & path) : path(path) { }
    ~MakeImmutable() { makeImmutable(path); }
};


static void hashAndLink(bool dryRun, HashToPath & hashToPath,
    OptimiseStats & stats, const Path & path)
{
    struct stat st;
    if (lstat(path.c_str(), &st))
	throw SysError(format("getting attributes of path `%1%'") % path);

    /* Sometimes SNAFUs can cause files in the Nix store to be
       modified, in particular when running programs as root under
       NixOS (example: $fontconfig/var/cache being modified).  Skip
       those files. */
    if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
        printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path);
        return;
    }

    /* We can hard link regular files and symlinks. */
    if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) {

        /* Hash the file.  Note that hashPath() returns the hash over
           the NAR serialisation, which includes the execute bit on
           the file.  Thus, executable and non-executable files with
           the same contents *won't* be linked (which is good because
           otherwise the permissions would be screwed up).

           Also note that if `path' is a symlink, then we're hashing
           the contents of the symlink (i.e. the result of
           readlink()), not the contents of the target (which may not
           even exist). */
        Hash hash = hashPath(htSHA256, path).first;
        stats.totalFiles++;
        printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));

        std::pair<Path, ino_t> prevPath = hashToPath[hash];
        
        if (prevPath.first == "") {
            hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);
            return;
        }
            
        /* Yes!  We've seen a file with the same contents.  Replace
           the current file with a hard link to that file. */
        stats.sameContents++;
        if (prevPath.second == st.st_ino) {
            printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % prevPath.first);
            return;
        }
        
        if (!dryRun) {
            
            printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % prevPath.first);

            Path tempLink = (format("%1%.tmp-%2%-%3%")
                % path % getpid() % rand()).str();

            /* Make the containing directory writable, but only if
               it's not the store itself (we don't want or need to
               mess with its permissions). */
            bool mustToggle = !isStorePath(path);
            if (mustToggle) makeWritable(dirOf(path));
            
            /* When we're done, make the directory read-only again and
               reset its timestamp back to 0. */
            MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");

            /* If ‘prevPath’ is immutable, we can't create hard links
               to it, so make it mutable first (and make it immutable
               again when we're done).  We also have to make ‘path’
               mutable, otherwise rename() will fail to delete it. */
            makeMutable(prevPath.first);
            MakeImmutable mk1(prevPath.first);
            
            makeMutable(path);
            MakeImmutable mk2(path);

            if (link(prevPath.first.c_str(), tempLink.c_str()) == -1) {
                if (errno == EMLINK) {
                    /* Too many links to the same file (>= 32000 on
                       most file systems).  This is likely to happen
                       with empty files.  Just start over, creating
                       links to the current file. */
                    printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first);
                    hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);
                    return;
                }
                throw SysError(format("cannot link `%1%' to `%2%'")
                    % tempLink % prevPath.first);
            }

            /* Atomically replace the old file with the new hard link. */
            if (rename(tempLink.c_str(), path.c_str()) == -1) {
                if (errno == EMLINK) {
                    /* Some filesystems generate too many links on the
                       rename, rather than on the original link.
                       (Probably it temporarily increases the st_nlink
                       field before decreasing it again.) */
                    printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first);
                    hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);

                    /* Unlink the temp link. */
                    if (unlink(tempLink.c_str()) == -1)
                        printMsg(lvlError, format("unable to unlink `%1%'") % tempLink);
                    return;
                }
                throw SysError(format("cannot rename `%1%' to `%2%'")
                    % tempLink % path);
            }
        } else
            printMsg(lvlTalkative, format("would link `%1%' to `%2%'") % path % prevPath.first);
        
        stats.filesLinked++;
        stats.bytesFreed += st.st_size;
        stats.blocksFreed += st.st_blocks;
    }

    if (S_ISDIR(st.st_mode)) {
        Strings names = readDirectory(path);
	foreach (Strings::iterator, i, names)
	    hashAndLink(dryRun, hashToPath, stats, path + "/" + *i);
    }
}


void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats)
{
    HashToPath hashToPath;

    PathSet paths = queryAllValidPaths();

    foreach (PathSet::iterator, i, paths) {
        addTempRoot(*i);
        if (!isValidPath(*i)) continue; /* path was GC'ed, probably */
        startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i);
        hashAndLink(dryRun, hashToPath, stats, *i);
    }
}


}