about summary refs log tree commit diff
path: root/tvix/nix-compat/src/nar/reader/mod.rs
blob: eef3b10f3c28eef52e737b5cb921de420f4cc037 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
//! Parser for the Nix archive format, aka NAR.
//!
//! NAR files (and their hashed representations) are used in C++ Nix for
//! a variety of things, including addressing fixed-output derivations
//! and transferring store paths between Nix stores.

use std::io::{
    self, BufRead,
    ErrorKind::{InvalidData, UnexpectedEof},
    Read, Write,
};

#[cfg(not(debug_assertions))]
use std::marker::PhantomData;

// Required reading for understanding this module.
use crate::nar::wire;

#[cfg(all(feature = "async", feature = "wire"))]
pub mod r#async;

mod read;
#[cfg(test)]
mod test;

pub type Reader<'a> = dyn BufRead + Send + 'a;

struct ArchiveReader<'a, 'r> {
    inner: &'a mut Reader<'r>,

    /// In debug mode, also track when we need to abandon this archive reader.
    ///
    /// The archive reader must be abandoned when:
    ///   * An error is encountered at any point
    ///   * A file or directory reader is dropped before being read entirely.
    ///
    /// All of these checks vanish in release mode.
    status: ArchiveReaderStatus<'a>,
}

macro_rules! try_or_poison {
    ($it:expr, $ex:expr) => {
        match $ex {
            Ok(x) => x,
            Err(e) => {
                $it.status.poison();
                return Err(e.into());
            }
        }
    };
}
/// Start reading a NAR file from `reader`.
pub fn open<'a, 'r>(reader: &'a mut Reader<'r>) -> io::Result<Node<'a, 'r>> {
    read::token(reader, &wire::TOK_NAR)?;
    Node::new(ArchiveReader {
        inner: reader,
        status: ArchiveReaderStatus::top(),
    })
}

pub enum Node<'a, 'r> {
    Symlink {
        target: Vec<u8>,
    },
    File {
        executable: bool,
        reader: FileReader<'a, 'r>,
    },
    Directory(DirReader<'a, 'r>),
}

impl<'a, 'r> Node<'a, 'r> {
    /// Start reading a [Node], matching the next [wire::Node].
    ///
    /// Reading the terminating [wire::TOK_PAR] is done immediately for [Node::Symlink],
    /// but is otherwise left to [DirReader] or [FileReader].
    fn new(mut reader: ArchiveReader<'a, 'r>) -> io::Result<Self> {
        Ok(match read::tag(reader.inner)? {
            wire::Node::Sym => {
                let target =
                    try_or_poison!(reader, read::bytes(reader.inner, wire::MAX_TARGET_LEN));

                if target.is_empty() || target.contains(&0) {
                    reader.status.poison();
                    return Err(InvalidData.into());
                }

                try_or_poison!(reader, read::token(reader.inner, &wire::TOK_PAR));
                reader.status.ready_parent(); // Immediately allow reading from parent again

                Node::Symlink { target }
            }
            tag @ (wire::Node::Reg | wire::Node::Exe) => {
                let len = try_or_poison!(&mut reader, read::u64(reader.inner));

                Node::File {
                    executable: tag == wire::Node::Exe,
                    reader: FileReader::new(reader, len)?,
                }
            }
            wire::Node::Dir => Node::Directory(DirReader::new(reader)),
        })
    }
}

/// File contents, readable through the [Read] trait.
///
/// It comes with some caveats:
///  * You must always read the entire file, unless you intend to abandon the entire archive reader.
///  * You must abandon the entire archive reader upon the first error.
///
/// It's fine to read exactly `reader.len()` bytes without ever seeing an explicit EOF.
pub struct FileReader<'a, 'r> {
    reader: ArchiveReader<'a, 'r>,
    len: u64,
    /// Truncated original file length for padding computation.
    /// We only care about the 3 least significant bits; semantically, this is a u3.
    pad: u8,
}

impl<'a, 'r> FileReader<'a, 'r> {
    /// Instantiate a new reader, starting after [wire::TOK_REG] or [wire::TOK_EXE].
    /// We handle the terminating [wire::TOK_PAR] on semantic EOF.
    fn new(mut reader: ArchiveReader<'a, 'r>, len: u64) -> io::Result<Self> {
        // For zero-length files, we have to read the terminating TOK_PAR
        // immediately, since FileReader::read may never be called; we've
        // already reached semantic EOF by definition.
        if len == 0 {
            read::token(reader.inner, &wire::TOK_PAR)?;
            reader.status.ready_parent();
        }

        Ok(Self {
            reader,
            len,
            pad: len as u8,
        })
    }

    pub fn is_empty(&self) -> bool {
        self.len == 0
    }

    pub fn len(&self) -> u64 {
        self.len
    }
}

impl FileReader<'_, '_> {
    /// Equivalent to [BufRead::fill_buf]
    ///
    /// We can't directly implement [BufRead], because [FileReader::consume] needs
    /// to perform fallible I/O.
    pub fn fill_buf(&mut self) -> io::Result<&[u8]> {
        if self.is_empty() {
            return Ok(&[]);
        }

        self.reader.check_correct();

        let mut buf = try_or_poison!(self.reader, self.reader.inner.fill_buf());

        if buf.is_empty() {
            self.reader.status.poison();
            return Err(UnexpectedEof.into());
        }

        if buf.len() as u64 > self.len {
            buf = &buf[..self.len as usize];
        }

        Ok(buf)
    }

    /// Analogous to [BufRead::consume], differing only in that it needs
    /// to perform I/O in order to read padding and terminators.
    pub fn consume(&mut self, n: usize) -> io::Result<()> {
        if n == 0 {
            return Ok(());
        }

        self.reader.check_correct();

        self.len = self
            .len
            .checked_sub(n as u64)
            .expect("consumed bytes past EOF");

        self.reader.inner.consume(n);

        if self.is_empty() {
            self.finish()?;
        }

        Ok(())
    }

    /// Copy the (remaining) contents of the file into `dst`.
    pub fn copy(&mut self, mut dst: impl Write) -> io::Result<()> {
        while !self.is_empty() {
            let buf = self.fill_buf()?;
            let n = try_or_poison!(self.reader, dst.write(buf));
            self.consume(n)?;
        }

        Ok(())
    }
}

impl Read for FileReader<'_, '_> {
    fn read(&mut self, mut buf: &mut [u8]) -> io::Result<usize> {
        if buf.is_empty() || self.is_empty() {
            return Ok(0);
        }

        self.reader.check_correct();

        if buf.len() as u64 > self.len {
            buf = &mut buf[..self.len as usize];
        }

        let n = try_or_poison!(self.reader, self.reader.inner.read(buf));
        self.len -= n as u64;

        if n == 0 {
            self.reader.status.poison();
            return Err(UnexpectedEof.into());
        }

        if self.is_empty() {
            self.finish()?;
        }

        Ok(n)
    }
}

impl FileReader<'_, '_> {
    /// We've reached semantic EOF, consume and verify the padding and terminating TOK_PAR.
    /// Files are padded to 64 bits (8 bytes), just like any other byte string in the wire format.
    fn finish(&mut self) -> io::Result<()> {
        let pad = (self.pad & 7) as usize;

        if pad != 0 {
            let mut buf = [0; 8];
            try_or_poison!(self.reader, self.reader.inner.read_exact(&mut buf[pad..]));

            if buf != [0; 8] {
                self.reader.status.poison();
                return Err(InvalidData.into());
            }
        }

        try_or_poison!(self.reader, read::token(self.reader.inner, &wire::TOK_PAR));

        // Done with reading this file, allow going back up the chain of readers
        self.reader.status.ready_parent();

        Ok(())
    }
}

/// A directory iterator, yielding a sequence of [Node]s.
/// It must be fully consumed before reading further from the [DirReader] that produced it, if any.
pub struct DirReader<'a, 'r> {
    reader: ArchiveReader<'a, 'r>,
    /// Previous directory entry name.
    /// We have to hang onto this to enforce name monotonicity.
    prev_name: Vec<u8>,
}

pub struct Entry<'a, 'r> {
    pub name: &'a [u8],
    pub node: Node<'a, 'r>,
}

impl<'a, 'r> DirReader<'a, 'r> {
    fn new(reader: ArchiveReader<'a, 'r>) -> Self {
        Self {
            reader,
            prev_name: vec![],
        }
    }

    /// Read the next [Entry] from the directory.
    ///
    /// We explicitly don't implement [Iterator], since treating this as
    /// a regular Rust iterator will surely lead you astray.
    ///
    ///  * You must always consume the entire iterator, unless you abandon the entire archive reader.
    ///  * You must abandon the entire archive reader on the first error.
    ///  * You must abandon the directory reader upon the first [None].
    ///  * Even if you know the amount of elements up front, you must keep reading until you encounter [None].
    #[allow(clippy::should_implement_trait)]
    pub fn next(&mut self) -> io::Result<Option<Entry<'_, 'r>>> {
        self.reader.check_correct();

        // COME FROM the previous iteration: if we've already read an entry,
        // read its terminating TOK_PAR here.
        if !self.prev_name.is_empty() {
            try_or_poison!(self.reader, read::token(self.reader.inner, &wire::TOK_PAR));
        }

        // Determine if there are more entries to follow
        if let wire::Entry::None = try_or_poison!(self.reader, read::tag(self.reader.inner)) {
            // We've reached the end of this directory.
            self.reader.status.ready_parent();
            return Ok(None);
        }

        let mut name = [0; wire::MAX_NAME_LEN + 1];
        let name = try_or_poison!(
            self.reader,
            read::bytes_buf(self.reader.inner, &mut name, wire::MAX_NAME_LEN)
        );

        if name.is_empty()
            || name.contains(&0)
            || name.contains(&b'/')
            || name == b"."
            || name == b".."
        {
            self.reader.status.poison();
            return Err(InvalidData.into());
        }

        // Enforce strict monotonicity of directory entry names.
        if &self.prev_name[..] >= name {
            self.reader.status.poison();
            return Err(InvalidData.into());
        }

        self.prev_name.clear();
        self.prev_name.extend_from_slice(name);

        try_or_poison!(self.reader, read::token(self.reader.inner, &wire::TOK_NOD));

        Ok(Some(Entry {
            name: &self.prev_name,
            // Don't need to worry about poisoning here: Node::new will do it for us if needed
            node: Node::new(self.reader.child())?,
        }))
    }
}

/// We use a stack of statuses to:
///   * Share poisoned state across all objects from the same underlying reader,
///     so we can check they are abandoned when an error occurs
///   * Make sure only the most recently created object is read from, and is fully exhausted
///     before anything it was created from is used again.
enum ArchiveReaderStatus<'a> {
    #[cfg(not(debug_assertions))]
    None(PhantomData<&'a ()>),
    #[cfg(debug_assertions)]
    StackTop { poisoned: bool, ready: bool },
    #[cfg(debug_assertions)]
    StackChild {
        poisoned: &'a mut bool,
        parent_ready: &'a mut bool,
        ready: bool,
    },
}

impl ArchiveReaderStatus<'_> {
    fn top() -> Self {
        #[cfg(debug_assertions)]
        {
            ArchiveReaderStatus::StackTop {
                poisoned: false,
                ready: true,
            }
        }

        #[cfg(not(debug_assertions))]
        ArchiveReaderStatus::None(PhantomData)
    }

    /// Poison all the objects sharing the same reader, to be used when an error occurs
    fn poison(&mut self) {
        match self {
            #[cfg(not(debug_assertions))]
            ArchiveReaderStatus::None(_) => {}
            #[cfg(debug_assertions)]
            ArchiveReaderStatus::StackTop { poisoned: x, .. } => *x = true,
            #[cfg(debug_assertions)]
            ArchiveReaderStatus::StackChild { poisoned: x, .. } => **x = true,
        }
    }

    /// Mark the parent as ready, allowing it to be used again and preventing this reference to the reader being used again.
    fn ready_parent(&mut self) {
        match self {
            #[cfg(not(debug_assertions))]
            ArchiveReaderStatus::None(_) => {}
            #[cfg(debug_assertions)]
            ArchiveReaderStatus::StackTop { ready, .. } => {
                *ready = false;
            }
            #[cfg(debug_assertions)]
            ArchiveReaderStatus::StackChild {
                ready,
                parent_ready,
                ..
            } => {
                *ready = false;
                **parent_ready = true;
            }
        };
    }

    fn poisoned(&self) -> bool {
        match self {
            #[cfg(not(debug_assertions))]
            ArchiveReaderStatus::None(_) => false,
            #[cfg(debug_assertions)]
            ArchiveReaderStatus::StackTop { poisoned, .. } => *poisoned,
            #[cfg(debug_assertions)]
            ArchiveReaderStatus::StackChild { poisoned, .. } => **poisoned,
        }
    }

    fn ready(&self) -> bool {
        match self {
            #[cfg(not(debug_assertions))]
            ArchiveReaderStatus::None(_) => true,
            #[cfg(debug_assertions)]
            ArchiveReaderStatus::StackTop { ready, .. } => *ready,
            #[cfg(debug_assertions)]
            ArchiveReaderStatus::StackChild { ready, .. } => *ready,
        }
    }
}

impl<'a, 'r> ArchiveReader<'a, 'r> {
    /// Create a new child reader from this one.
    /// In debug mode, this reader will panic if called before the new child is exhausted / calls `ready_parent`
    fn child(&mut self) -> ArchiveReader<'_, 'r> {
        ArchiveReader {
            inner: self.inner,
            #[cfg(not(debug_assertions))]
            status: ArchiveReaderStatus::None(PhantomData),
            #[cfg(debug_assertions)]
            status: match &mut self.status {
                ArchiveReaderStatus::StackTop { poisoned, ready } => {
                    *ready = false;
                    ArchiveReaderStatus::StackChild {
                        poisoned,
                        parent_ready: ready,
                        ready: true,
                    }
                }
                ArchiveReaderStatus::StackChild {
                    poisoned, ready, ..
                } => {
                    *ready = false;
                    ArchiveReaderStatus::StackChild {
                        poisoned,
                        parent_ready: ready,
                        ready: true,
                    }
                }
            },
        }
    }

    /// Check the reader is in the correct status.
    /// Only does anything when debug assertions are on.
    #[inline(always)]
    fn check_correct(&self) {
        assert!(
            !self.status.poisoned(),
            "Archive reader used after it was meant to be abandoned!"
        );
        assert!(
            self.status.ready(),
            "Non-ready archive reader used! (Should've been reading from something else)"
        );
    }
}