about summary refs log tree commit diff
path: root/tvix/glue/src/known_paths.rs
blob: 239acca9829a5d8d0ed7d66611806c82541039a7 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
//! This module implements logic required for persisting known paths
//! during an evaluation.
//!
//! Tvix needs to be able to keep track of each Nix store path that it
//! knows about during the scope of a single evaluation and its
//! related builds.
//!
//! This data is required to find the derivation needed to actually trigger the
//! build, if necessary.

use nix_compat::{
    derivation::Derivation,
    store_path::{BuildStorePathError, StorePath, StorePathRef},
};
use std::collections::HashMap;

use crate::fetchers::Fetch;

/// Struct keeping track of all known Derivations in the current evaluation.
/// This keeps both the Derivation struct, as well as the "Hash derivation
/// modulo".
#[derive(Debug, Default)]
pub struct KnownPaths {
    /// All known derivation or FOD hashes.
    ///
    /// Keys are derivation paths, values are a tuple of the "hash derivation
    /// modulo" and the Derivation struct itself.
    derivations: HashMap<StorePath<String>, ([u8; 32], Derivation)>,

    /// A map from output path to (one) drv path.
    /// Note that in the case of FODs, multiple drvs can produce the same output
    /// path. We use one of them.
    outputs_to_drvpath: HashMap<StorePath<String>, StorePath<String>>,

    /// A map from output path to fetches (and their names).
    outputs_to_fetches: HashMap<StorePath<String>, (String, Fetch)>,
}

impl KnownPaths {
    /// Fetch the opaque "hash derivation modulo" for a given derivation path.
    pub fn get_hash_derivation_modulo(&self, drv_path: &StorePath<String>) -> Option<&[u8; 32]> {
        self.derivations
            .get(drv_path)
            .map(|(hash_derivation_modulo, _derivation)| hash_derivation_modulo)
    }

    /// Return a reference to the Derivation for a given drv path.
    pub fn get_drv_by_drvpath(&self, drv_path: &StorePath<String>) -> Option<&Derivation> {
        self.derivations
            .get(drv_path)
            .map(|(_hash_derivation_modulo, derivation)| derivation)
    }

    /// Return the drv path of the derivation producing the passed output path.
    /// Note there can be multiple Derivations producing the same output path in
    /// flight; this function will only return one of them.
    pub fn get_drv_path_for_output_path(
        &self,
        output_path: &StorePath<String>,
    ) -> Option<&StorePath<String>> {
        self.outputs_to_drvpath.get(output_path)
    }

    /// Insert a new [Derivation] into this struct.
    /// The Derivation struct must pass validation, and its output paths need to
    /// be fully calculated.
    /// All input derivations this refers to must also be inserted to this
    /// struct.
    pub fn add_derivation(&mut self, drv_path: StorePath<String>, drv: Derivation) {
        // check input derivations to have been inserted.
        #[cfg(debug_assertions)]
        {
            for input_drv_path in drv.input_derivations.keys() {
                debug_assert!(self.derivations.contains_key(input_drv_path));
            }
        }

        // compute the hash derivation modulo
        let hash_derivation_modulo = drv.hash_derivation_modulo(|drv_path| {
            self.get_hash_derivation_modulo(&drv_path.to_owned())
                .unwrap_or_else(|| panic!("{} not found", drv_path))
                .to_owned()
        });

        // For all output paths, update our lookup table.
        // We only write into the lookup table once.
        for output in drv.outputs.values() {
            self.outputs_to_drvpath
                .entry(output.path.as_ref().expect("missing store path").clone())
                .or_insert(drv_path.to_owned());
        }

        // insert the derivation itself
        #[allow(unused_variables)] // assertions on this only compiled in debug builds
        let old = self
            .derivations
            .insert(drv_path.to_owned(), (hash_derivation_modulo, drv));

        #[cfg(debug_assertions)]
        {
            if let Some(old) = old {
                debug_assert!(
                    old.0 == hash_derivation_modulo,
                    "hash derivation modulo for a given derivation should always be calculated the same"
                );
            }
        }
    }

    /// Insert a new [Fetch] into this struct, which *must* have an expected
    /// hash (otherwise we wouldn't be able to calculate the store path).
    /// Fetches without a known hash need to be fetched inside builtins.
    pub fn add_fetch<'a>(
        &mut self,
        fetch: Fetch,
        name: &'a str,
    ) -> Result<StorePathRef<'a>, BuildStorePathError> {
        let store_path = fetch
            .store_path(name)?
            .expect("Tvix bug: fetch must have an expected hash");
        // insert the fetch.
        self.outputs_to_fetches
            .insert(store_path.to_owned(), (name.to_owned(), fetch));

        Ok(store_path)
    }

    /// Return the name and fetch producing the passed output path.
    /// Note there can also be (multiple) Derivations producing the same output path.
    pub fn get_fetch_for_output_path(
        &self,
        output_path: &StorePath<String>,
    ) -> Option<(String, Fetch)> {
        self.outputs_to_fetches
            .get(output_path)
            .map(|(name, fetch)| (name.to_owned(), fetch.to_owned()))
    }

    /// Returns an iterator over all known derivations and their store path.
    pub fn get_derivations(&self) -> impl Iterator<Item = (&StorePath<String>, &Derivation)> {
        self.derivations.iter().map(|(k, v)| (k, &v.1))
    }
}

#[cfg(test)]
mod tests {
    use nix_compat::{derivation::Derivation, nixbase32, nixhash, store_path::StorePath};
    use url::Url;

    use crate::fetchers::Fetch;

    use super::KnownPaths;
    use hex_literal::hex;
    use lazy_static::lazy_static;

    lazy_static! {
        static ref BAR_DRV: Derivation = Derivation::from_aterm_bytes(include_bytes!(
            "tests/ss2p4wmxijn652haqyd7dckxwl4c7hxx-bar.drv"
        ))
        .expect("must parse");
        static ref FOO_DRV: Derivation = Derivation::from_aterm_bytes(include_bytes!(
            "tests/ch49594n9avinrf8ip0aslidkc4lxkqv-foo.drv"
        ))
        .expect("must parse");
        static ref BAR_DRV_PATH: StorePath<String> =
            StorePath::from_bytes(b"ss2p4wmxijn652haqyd7dckxwl4c7hxx-bar.drv").expect("must parse");
        static ref FOO_DRV_PATH: StorePath<String> =
            StorePath::from_bytes(b"ch49594n9avinrf8ip0aslidkc4lxkqv-foo.drv").expect("must parse");
        static ref BAR_OUT_PATH: StorePath<String> =
            StorePath::from_bytes(b"mp57d33657rf34lzvlbpfa1gjfv5gmpg-bar").expect("must parse");
        static ref FOO_OUT_PATH: StorePath<String> =
            StorePath::from_bytes(b"fhaj6gmwns62s6ypkcldbaj2ybvkhx3p-foo").expect("must parse");

        static ref FETCH_URL : Fetch = Fetch::URL{
            url: Url::parse("https://raw.githubusercontent.com/aaptel/notmuch-extract-patch/f732a53e12a7c91a06755ebfab2007adc9b3063b/notmuch-extract-patch").unwrap(),
            exp_hash: Some(nixhash::from_sri_str("sha256-Xa1Jbl2Eq5+L0ww+Ph1osA3Z/Dxe/RkN1/dITQCdXFk=").unwrap())
        };
        static ref FETCH_URL_OUT_PATH: StorePath<String> = StorePath::from_bytes(b"06qi00hylriyfm0nl827crgjvbax84mz-notmuch-extract-patch").unwrap();

        static ref FETCH_TARBALL : Fetch = Fetch::Tarball{
            url: Url::parse("https://github.com/NixOS/nixpkgs/archive/91050ea1e57e50388fa87a3302ba12d188ef723a.tar.gz").unwrap(),
            exp_nar_sha256: Some(nixbase32::decode_fixed("1hf6cgaci1n186kkkjq106ryf8mmlq9vnwgfwh625wa8hfgdn4dm").unwrap())
        };
        static ref FETCH_TARBALL_OUT_PATH: StorePath<String> = StorePath::from_bytes(b"7adgvk5zdfq4pwrhsm3n9lzypb12gw0g-source").unwrap();
    }

    /// ensure we don't allow acdding a Derivation that depends on another,
    /// not-yet-added Derivation.
    #[test]
    #[should_panic]
    fn drv_reject_if_missing_input_drv() {
        let mut known_paths = KnownPaths::default();

        // FOO_DRV depends on BAR_DRV, which wasn't added.
        known_paths.add_derivation(FOO_DRV_PATH.clone(), FOO_DRV.clone());
    }

    #[test]
    fn drv_happy_path() {
        let mut known_paths = KnownPaths::default();

        // get_drv_by_drvpath should return None for non-existing Derivations,
        // same as get_hash_derivation_modulo and get_drv_path_for_output_path
        assert_eq!(None, known_paths.get_drv_by_drvpath(&BAR_DRV_PATH));
        assert_eq!(None, known_paths.get_hash_derivation_modulo(&BAR_DRV_PATH));
        assert_eq!(
            None,
            known_paths.get_drv_path_for_output_path(&BAR_OUT_PATH)
        );

        // Add BAR_DRV
        known_paths.add_derivation(BAR_DRV_PATH.clone(), BAR_DRV.clone());

        // We should get it back
        assert_eq!(
            Some(&BAR_DRV.clone()),
            known_paths.get_drv_by_drvpath(&BAR_DRV_PATH)
        );

        // Test get_drv_path_for_output_path
        assert_eq!(
            Some(&BAR_DRV_PATH.clone()),
            known_paths.get_drv_path_for_output_path(&BAR_OUT_PATH)
        );

        // It should be possible to get the hash derivation modulo.
        assert_eq!(
            Some(&hex!(
                "c79aebd0ce3269393d4a1fde2cbd1d975d879b40f0bf40a48f550edc107fd5df"
            )),
            known_paths.get_hash_derivation_modulo(&BAR_DRV_PATH.clone())
        );

        // Now insert FOO_DRV too. It shouldn't panic, as BAR_DRV is already
        // added.
        known_paths.add_derivation(FOO_DRV_PATH.clone(), FOO_DRV.clone());

        assert_eq!(
            Some(&FOO_DRV.clone()),
            known_paths.get_drv_by_drvpath(&FOO_DRV_PATH)
        );
        assert_eq!(
            Some(&hex!(
                "af030d36d63d3d7f56a71adaba26b36f5fa1f9847da5eed953ed62e18192762f"
            )),
            known_paths.get_hash_derivation_modulo(&FOO_DRV_PATH.clone())
        );

        // Test get_drv_path_for_output_path
        assert_eq!(
            Some(&FOO_DRV_PATH.clone()),
            known_paths.get_drv_path_for_output_path(&FOO_OUT_PATH)
        );
    }

    #[test]
    fn fetch_happy_path() {
        let mut known_paths = KnownPaths::default();

        // get_fetch_for_output_path should return None for new fetches.
        assert!(known_paths
            .get_fetch_for_output_path(&FETCH_TARBALL_OUT_PATH)
            .is_none());

        // add_fetch should return the properly calculated store paths.
        assert_eq!(
            *FETCH_TARBALL_OUT_PATH,
            known_paths
                .add_fetch(FETCH_TARBALL.clone(), "source")
                .unwrap()
                .to_owned()
        );

        assert_eq!(
            *FETCH_URL_OUT_PATH,
            known_paths
                .add_fetch(FETCH_URL.clone(), "notmuch-extract-patch")
                .unwrap()
                .to_owned()
        );
    }

    #[test]
    fn get_derivations_working() {
        let mut known_paths = KnownPaths::default();

        // Add BAR_DRV
        known_paths.add_derivation(BAR_DRV_PATH.clone(), BAR_DRV.clone());

        // We should be able to find BAR_DRV_PATH and BAR_DRV as a pair in get_derivations.
        assert_eq!(
            Some((&BAR_DRV_PATH.clone(), &BAR_DRV.clone())),
            known_paths
                .get_derivations()
                .find(|(s, d)| (*s, *d) == (&BAR_DRV_PATH, &BAR_DRV))
        );
    }

    // TODO: add test panicking about missing digest
}