diff options
Diffstat (limited to 'tvix/nix-compat/src/nar/writer')
| -rw-r--r-- | tvix/nix-compat/src/nar/writer/async.rs | 235 | ||||
| -rw-r--r-- | tvix/nix-compat/src/nar/writer/mod.rs | 9 | ||||
| -rw-r--r-- | tvix/nix-compat/src/nar/writer/sync.rs | 224 | ||||
| -rw-r--r-- | tvix/nix-compat/src/nar/writer/test.rs | 128 |
4 files changed, 596 insertions, 0 deletions
diff --git a/tvix/nix-compat/src/nar/writer/async.rs b/tvix/nix-compat/src/nar/writer/async.rs new file mode 100644 index 000000000000..a2ce68fc3c9e --- /dev/null +++ b/tvix/nix-compat/src/nar/writer/async.rs @@ -0,0 +1,235 @@ +//! Implements an interface for writing the Nix archive format (NAR). +//! +//! NAR files (and their hashed representations) are used in C++ Nix for +//! addressing fixed-output derivations and a variety of other things. +//! +//! NAR files can be output to any type that implements [`AsyncWrite`], and content +//! can be read from any type that implementes [`AsyncBufRead`]. +//! +//! Writing a single file might look like this: +//! +//! ```rust +//! # futures::executor::block_on(async { +//! # use tokio::io::BufReader; +//! # let some_file: Vec<u8> = vec![0, 1, 2, 3, 4]; +//! +//! // Output location to write the NAR to. +//! let mut sink: Vec<u8> = Vec::new(); +//! +//! // Instantiate writer for this output location. +//! let mut nar = nix_compat::nar::writer::r#async::open(&mut sink).await?; +//! +//! // Acquire metadata for the single file to output, and pass it in a +//! // `BufRead`-implementing type. +//! +//! let executable = false; +//! let size = some_file.len() as u64; +//! let mut reader = BufReader::new(some_file.as_slice()); +//! nar.file(executable, size, &mut reader).await?; +//! # Ok::<(), std::io::Error>(()) +//! # }); +//! ``` + +use crate::nar::wire; +use std::{ + io::{ + self, + ErrorKind::{InvalidInput, UnexpectedEof}, + }, + pin::Pin, +}; +use tokio::io::{AsyncBufRead, AsyncBufReadExt, AsyncWrite, AsyncWriteExt}; + +/// Convenience type alias for types implementing [`AsyncWrite`]. +pub type Writer<'a> = dyn AsyncWrite + Unpin + Send + 'a; + +/// Create a new NAR, writing the output to the specified writer. +pub async fn open<'a, 'w: 'a>(writer: &'a mut Writer<'w>) -> io::Result<Node<'a, 'w>> { + let mut node = Node { writer }; + node.write(&wire::TOK_NAR).await?; + Ok(node) +} + +/// Single node in a NAR file. +/// +/// A NAR can be thought of as a tree of nodes represented by this type. Each +/// node can be a file, a symlink or a directory containing other nodes. +pub struct Node<'a, 'w: 'a> { + writer: &'a mut Writer<'w>, +} + +impl<'a, 'w> Node<'a, 'w> { + async fn write(&mut self, data: &[u8]) -> io::Result<()> { + self.writer.write_all(data).await + } + + async fn pad(&mut self, n: u64) -> io::Result<()> { + match (n & 7) as usize { + 0 => Ok(()), + n => self.write(&[0; 8][n..]).await, + } + } + + /// Make this node a symlink. + pub async fn symlink(mut self, target: &[u8]) -> io::Result<()> { + debug_assert!( + target.len() <= wire::MAX_TARGET_LEN, + "target.len() > {}", + wire::MAX_TARGET_LEN + ); + debug_assert!(!target.is_empty(), "target is empty"); + debug_assert!(!target.contains(&0), "target contains null byte"); + + self.write(&wire::TOK_SYM).await?; + self.write(&target.len().to_le_bytes()).await?; + self.write(target).await?; + self.pad(target.len() as u64).await?; + self.write(&wire::TOK_PAR).await?; + Ok(()) + } + + /// Make this node a single file. + pub async fn file( + mut self, + executable: bool, + size: u64, + reader: &mut (dyn AsyncBufRead + Unpin + Send), + ) -> io::Result<()> { + self.write(if executable { + &wire::TOK_EXE + } else { + &wire::TOK_REG + }) + .await?; + + self.write(&size.to_le_bytes()).await?; + + let mut need = size; + while need != 0 { + let data = reader.fill_buf().await?; + + if data.is_empty() { + return Err(UnexpectedEof.into()); + } + + let n = need.min(data.len() as u64) as usize; + self.write(&data[..n]).await?; + + need -= n as u64; + Pin::new(&mut *reader).consume(n); + } + + // bail if there's still data left in the passed reader. + // This uses the same code as [BufRead::has_data_left] (unstable). + if reader.fill_buf().await.map(|b| !b.is_empty())? { + return Err(io::Error::new( + InvalidInput, + "reader contained more data than specified size", + )); + } + + self.pad(size).await?; + self.write(&wire::TOK_PAR).await?; + + Ok(()) + } + + /// Make this node a directory, the content of which is set using the + /// resulting [`Directory`] value. + /// + /// It is the caller's responsibility to invoke [`Directory::close`], + /// or invalid archives will be produced silently. + pub async fn directory(mut self) -> io::Result<Directory<'a, 'w>> { + self.write(&wire::TOK_DIR).await?; + Ok(Directory::new(self)) + } +} + +#[cfg(debug_assertions)] +type Name = Vec<u8>; +#[cfg(not(debug_assertions))] +type Name = (); + +fn into_name(_name: &[u8]) -> Name { + #[cfg(debug_assertions)] + _name.to_owned() +} + +/// Content of a NAR node that represents a directory. +pub struct Directory<'a, 'w> { + node: Node<'a, 'w>, + prev_name: Option<Name>, +} + +impl<'a, 'w> Directory<'a, 'w> { + fn new(node: Node<'a, 'w>) -> Self { + Self { + node, + prev_name: None, + } + } + + /// Add an entry to the directory. + /// + /// The entry is simply another [`Node`], which can then be filled like the + /// root of a NAR (including, of course, by nesting directories). + /// + /// It is the caller's responsibility to ensure that directory entries are + /// written in order of ascending name. If this is not ensured, this method + /// may panic or silently produce invalid archives. + pub async fn entry(&mut self, name: &[u8]) -> io::Result<Node<'_, 'w>> { + debug_assert!( + name.len() <= wire::MAX_NAME_LEN, + "name.len() > {}", + wire::MAX_NAME_LEN + ); + debug_assert!(!name.is_empty(), "name is empty"); + debug_assert!(!name.contains(&0), "name contains null byte"); + debug_assert!(!name.contains(&b'/'), "name contains {:?}", '/'); + debug_assert!(name != b".", "name == {:?}", "."); + debug_assert!(name != b"..", "name == {:?}", ".."); + + match self.prev_name { + None => { + self.prev_name = Some(into_name(name)); + } + Some(ref mut _prev_name) => { + #[cfg(debug_assertions)] + { + use bstr::ByteSlice; + assert!( + &**_prev_name < name, + "misordered names: {:?} >= {:?}", + _prev_name.as_bstr(), + name.as_bstr() + ); + name.clone_into(_prev_name); + } + self.node.write(&wire::TOK_PAR).await?; + } + } + + self.node.write(&wire::TOK_ENT).await?; + self.node.write(&name.len().to_le_bytes()).await?; + self.node.write(name).await?; + self.node.pad(name.len() as u64).await?; + self.node.write(&wire::TOK_NOD).await?; + + Ok(Node { + writer: &mut *self.node.writer, + }) + } + + /// Close a directory and write terminators for the directory to the NAR. + /// + /// **Important:** This *must* be called when all entries have been written + /// in a directory, otherwise the resulting NAR file will be invalid. + pub async fn close(mut self) -> io::Result<()> { + if self.prev_name.is_some() { + self.node.write(&wire::TOK_PAR).await?; + } + + self.node.write(&wire::TOK_PAR).await?; + Ok(()) + } +} diff --git a/tvix/nix-compat/src/nar/writer/mod.rs b/tvix/nix-compat/src/nar/writer/mod.rs new file mode 100644 index 000000000000..fe8ccccb3787 --- /dev/null +++ b/tvix/nix-compat/src/nar/writer/mod.rs @@ -0,0 +1,9 @@ +pub use sync::*; + +pub mod sync; + +#[cfg(test)] +mod test; + +#[cfg(feature = "async")] +pub mod r#async; diff --git a/tvix/nix-compat/src/nar/writer/sync.rs b/tvix/nix-compat/src/nar/writer/sync.rs new file mode 100644 index 000000000000..6270129028fa --- /dev/null +++ b/tvix/nix-compat/src/nar/writer/sync.rs @@ -0,0 +1,224 @@ +//! Implements an interface for writing the Nix archive format (NAR). +//! +//! NAR files (and their hashed representations) are used in C++ Nix for +//! addressing fixed-output derivations and a variety of other things. +//! +//! NAR files can be output to any type that implements [`Write`], and content +//! can be read from any type that implementes [`BufRead`]. +//! +//! Writing a single file might look like this: +//! +//! ```rust +//! # use std::io::BufReader; +//! # let some_file: Vec<u8> = vec![0, 1, 2, 3, 4]; +//! +//! // Output location to write the NAR to. +//! let mut sink: Vec<u8> = Vec::new(); +//! +//! // Instantiate writer for this output location. +//! let mut nar = nix_compat::nar::writer::open(&mut sink)?; +//! +//! // Acquire metadata for the single file to output, and pass it in a +//! // `BufRead`-implementing type. +//! +//! let executable = false; +//! let size = some_file.len() as u64; +//! let mut reader = BufReader::new(some_file.as_slice()); +//! nar.file(executable, size, &mut reader)?; +//! # Ok::<(), std::io::Error>(()) +//! ``` + +use crate::nar::wire; +use std::io::{ + self, BufRead, + ErrorKind::{InvalidInput, UnexpectedEof}, + Write, +}; + +/// Convenience type alias for types implementing [`Write`]. +pub type Writer<'a> = dyn Write + Send + 'a; + +/// Create a new NAR, writing the output to the specified writer. +pub fn open<'a, 'w: 'a>(writer: &'a mut Writer<'w>) -> io::Result<Node<'a, 'w>> { + let mut node = Node { writer }; + node.write(&wire::TOK_NAR)?; + Ok(node) +} + +/// Single node in a NAR file. +/// +/// A NAR can be thought of as a tree of nodes represented by this type. Each +/// node can be a file, a symlink or a directory containing other nodes. +pub struct Node<'a, 'w: 'a> { + writer: &'a mut Writer<'w>, +} + +impl<'a, 'w> Node<'a, 'w> { + fn write(&mut self, data: &[u8]) -> io::Result<()> { + self.writer.write_all(data) + } + + fn pad(&mut self, n: u64) -> io::Result<()> { + match (n & 7) as usize { + 0 => Ok(()), + n => self.write(&[0; 8][n..]), + } + } + + /// Make this node a symlink. + pub fn symlink(mut self, target: &[u8]) -> io::Result<()> { + debug_assert!( + target.len() <= wire::MAX_TARGET_LEN, + "target.len() > {}", + wire::MAX_TARGET_LEN + ); + debug_assert!(!target.is_empty(), "target is empty"); + debug_assert!(!target.contains(&0), "target contains null byte"); + + self.write(&wire::TOK_SYM)?; + self.write(&target.len().to_le_bytes())?; + self.write(target)?; + self.pad(target.len() as u64)?; + self.write(&wire::TOK_PAR)?; + Ok(()) + } + + /// Make this node a single file. + pub fn file(mut self, executable: bool, size: u64, reader: &mut dyn BufRead) -> io::Result<()> { + self.write(if executable { + &wire::TOK_EXE + } else { + &wire::TOK_REG + })?; + + self.write(&size.to_le_bytes())?; + + let mut need = size; + while need != 0 { + let data = reader.fill_buf()?; + + if data.is_empty() { + return Err(UnexpectedEof.into()); + } + + let n = need.min(data.len() as u64) as usize; + self.write(&data[..n])?; + + need -= n as u64; + reader.consume(n); + } + + // bail if there's still data left in the passed reader. + // This uses the same code as [BufRead::has_data_left] (unstable). + if reader.fill_buf().map(|b| !b.is_empty())? { + return Err(io::Error::new( + InvalidInput, + "reader contained more data than specified size", + )); + } + + self.pad(size)?; + self.write(&wire::TOK_PAR)?; + + Ok(()) + } + + /// Make this node a directory, the content of which is set using the + /// resulting [`Directory`] value. + /// + /// It is the caller's responsibility to invoke [`Directory::close`], + /// or invalid archives will be produced silently. + pub fn directory(mut self) -> io::Result<Directory<'a, 'w>> { + self.write(&wire::TOK_DIR)?; + Ok(Directory::new(self)) + } +} + +#[cfg(debug_assertions)] +type Name = Vec<u8>; +#[cfg(not(debug_assertions))] +type Name = (); + +fn into_name(_name: &[u8]) -> Name { + #[cfg(debug_assertions)] + _name.to_owned() +} + +/// Content of a NAR node that represents a directory. +pub struct Directory<'a, 'w> { + node: Node<'a, 'w>, + prev_name: Option<Name>, +} + +impl<'a, 'w> Directory<'a, 'w> { + fn new(node: Node<'a, 'w>) -> Self { + Self { + node, + prev_name: None, + } + } + + /// Add an entry to the directory. + /// + /// The entry is simply another [`Node`], which can then be filled like the + /// root of a NAR (including, of course, by nesting directories). + /// + /// It is the caller's responsibility to ensure that directory entries are + /// written in order of ascending name. If this is not ensured, this method + /// may panic or silently produce invalid archives. + pub fn entry(&mut self, name: &[u8]) -> io::Result<Node<'_, 'w>> { + debug_assert!( + name.len() <= wire::MAX_NAME_LEN, + "name.len() > {}", + wire::MAX_NAME_LEN + ); + debug_assert!(!name.is_empty(), "name is empty"); + debug_assert!(!name.contains(&0), "name contains null byte"); + debug_assert!(!name.contains(&b'/'), "name contains {:?}", '/'); + debug_assert!(name != b".", "name == {:?}", "."); + debug_assert!(name != b"..", "name == {:?}", ".."); + + match self.prev_name { + None => { + self.prev_name = Some(into_name(name)); + } + Some(ref mut _prev_name) => { + #[cfg(debug_assertions)] + { + use bstr::ByteSlice; + assert!( + &**_prev_name < name, + "misordered names: {:?} >= {:?}", + _prev_name.as_bstr(), + name.as_bstr() + ); + name.clone_into(_prev_name); + } + self.node.write(&wire::TOK_PAR)?; + } + } + + self.node.write(&wire::TOK_ENT)?; + self.node.write(&name.len().to_le_bytes())?; + self.node.write(name)?; + self.node.pad(name.len() as u64)?; + self.node.write(&wire::TOK_NOD)?; + + Ok(Node { + writer: &mut *self.node.writer, + }) + } + + /// Close a directory and write terminators for the directory to the NAR. + /// + /// **Important:** This *must* be called when all entries have been written + /// in a directory, otherwise the resulting NAR file will be invalid. + pub fn close(mut self) -> io::Result<()> { + if self.prev_name.is_some() { + self.node.write(&wire::TOK_PAR)?; + } + + self.node.write(&wire::TOK_PAR)?; + Ok(()) + } +} diff --git a/tvix/nix-compat/src/nar/writer/test.rs b/tvix/nix-compat/src/nar/writer/test.rs new file mode 100644 index 000000000000..d7f18a49af34 --- /dev/null +++ b/tvix/nix-compat/src/nar/writer/test.rs @@ -0,0 +1,128 @@ +use crate::nar; + +#[test] +fn symlink() { + let mut buf = vec![]; + let node = nar::writer::open(&mut buf).unwrap(); + + node.symlink("/nix/store/somewhereelse".as_bytes()).unwrap(); + + assert_eq!(include_bytes!("../tests/symlink.nar"), buf.as_slice()); +} + +#[cfg(feature = "async")] +#[tokio::test] +async fn symlink_async() { + let mut buf = vec![]; + + let node = nar::writer::r#async::open(&mut buf).await.unwrap(); + node.symlink("/nix/store/somewhereelse".as_bytes()) + .await + .unwrap(); + + assert_eq!(include_bytes!("../tests/symlink.nar"), buf.as_slice()); +} + +#[test] +fn file() { + let mut buf = vec![]; + let node = nar::writer::open(&mut buf).unwrap(); + + let file_contents = "Hello World!".to_string(); + node.file( + false, + file_contents.len() as u64, + &mut std::io::Cursor::new(file_contents), + ) + .unwrap(); + + assert_eq!(include_bytes!("../tests/helloworld.nar"), buf.as_slice()); +} + +#[cfg(feature = "async")] +#[tokio::test] +async fn file_async() { + use std::io::Cursor; + + let mut buf = vec![]; + + let node = nar::writer::r#async::open(&mut buf).await.unwrap(); + + let file_contents = "Hello World!".to_string(); + node.file( + false, + file_contents.len() as u64, + &mut Cursor::new(file_contents), + ) + .await + .unwrap(); + + assert_eq!(include_bytes!("../tests/helloworld.nar"), buf.as_slice()); +} + +#[test] +fn complicated() { + let mut buf = vec![]; + let node = nar::writer::open(&mut buf).unwrap(); + + let mut dir_node = node.directory().unwrap(); + + let e = dir_node.entry(".keep".as_bytes()).unwrap(); + e.file(false, 0, &mut std::io::Cursor::new([])) + .expect("read .keep must succeed"); + + let e = dir_node.entry("aa".as_bytes()).unwrap(); + e.symlink("/nix/store/somewhereelse".as_bytes()) + .expect("symlink must succeed"); + + let e = dir_node.entry("keep".as_bytes()).unwrap(); + let mut subdir_node = e.directory().expect("directory must succeed"); + + let e_sub = subdir_node + .entry(".keep".as_bytes()) + .expect("subdir entry must succeed"); + e_sub.file(false, 0, &mut std::io::Cursor::new([])).unwrap(); + + // close the subdir, and then the dir, which is required. + subdir_node.close().unwrap(); + dir_node.close().unwrap(); + + assert_eq!(include_bytes!("../tests/complicated.nar"), buf.as_slice()); +} + +#[cfg(feature = "async")] +#[tokio::test] +async fn complicated_async() { + use std::io::Cursor; + + let mut buf = vec![]; + + let node = nar::writer::r#async::open(&mut buf).await.unwrap(); + + let mut dir_node = node.directory().await.unwrap(); + + let e = dir_node.entry(".keep".as_bytes()).await.unwrap(); + e.file(false, 0, &mut Cursor::new([])) + .await + .expect("read .keep must succeed"); + + let e = dir_node.entry("aa".as_bytes()).await.unwrap(); + e.symlink("/nix/store/somewhereelse".as_bytes()) + .await + .expect("symlink must succeed"); + + let e = dir_node.entry("keep".as_bytes()).await.unwrap(); + let mut subdir_node = e.directory().await.expect("directory must succeed"); + + let e_sub = subdir_node + .entry(".keep".as_bytes()) + .await + .expect("subdir entry must succeed"); + e_sub.file(false, 0, &mut Cursor::new([])).await.unwrap(); + + // close the subdir, and then the dir, which is required. + subdir_node.close().await.unwrap(); + dir_node.close().await.unwrap(); + + assert_eq!(include_bytes!("../tests/complicated.nar"), buf.as_slice()); +} |