From df3223fd681f64d54f6f393e53647d3a487eff25 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Mon, 13 Feb 2023 16:44:26 +0100 Subject: chore(tvix/store): move NAR rendering logic into Renderer struct This moves the logic rendering NARs to a struct using the previously introduced, more granular BlobService, ChunkService and DirectoryService. Instead of passing them around to the helper functions, they're kept as members of a struct. Remove the async invocations in the nar_renderer tests, there's nothing async in here. Change-Id: Ic6d24aaad68a1fda46ce29f2cdb5f7b87f481d5c Reviewed-on: https://cl.tvl.fyi/c/depot/+/8095 Reviewed-by: raitobezarius Tested-by: BuildkiteCI --- tvix/store/src/nar.rs | 70 --------- tvix/store/src/nar/mod.rs | 22 +++ tvix/store/src/nar/renderer.rs | 136 +++++++++++++++++ tvix/store/src/tests/mod.rs | 2 +- tvix/store/src/tests/nar.rs | 267 ---------------------------------- tvix/store/src/tests/nar_renderer.rs | 273 +++++++++++++++++++++++++++++++++++ 6 files changed, 432 insertions(+), 338 deletions(-) delete mode 100644 tvix/store/src/nar.rs create mode 100644 tvix/store/src/nar/mod.rs create mode 100644 tvix/store/src/nar/renderer.rs delete mode 100644 tvix/store/src/tests/nar.rs create mode 100644 tvix/store/src/tests/nar_renderer.rs (limited to 'tvix/store/src') diff --git a/tvix/store/src/nar.rs b/tvix/store/src/nar.rs deleted file mode 100644 index efcaf652e138..000000000000 --- a/tvix/store/src/nar.rs +++ /dev/null @@ -1,70 +0,0 @@ -//! This provides some common "client-side" libraries to interact with a tvix- -//! store, in this case to render NAR. -use crate::{ - client::StoreClient, - proto::{self, NamedNode}, -}; -use anyhow::Result; -use nix_compat::nar; - -/// Consumes a [proto::node::Node] pointing to the root of a (store) path, -/// and writes the contents in NAR serialization to the passed -/// [std::io::Write]. -/// -/// It uses a [StoreClient] to do the necessary lookups as it traverses the -/// structure. -pub fn write_nar( - w: &mut W, - proto_root_node: proto::node::Node, - store_client: &mut SC, -) -> Result<()> { - // Initialize NAR writer - let nar_root_node = nar::writer::open(w)?; - - walk_node(nar_root_node, proto_root_node, store_client) -} - -/// Process an intermediate node in the structure. -/// This consumes the node. -fn walk_node( - nar_node: nar::writer::Node, - proto_node: proto::node::Node, - store_client: &mut SC, -) -> Result<()> { - match proto_node { - proto::node::Node::Symlink(proto_symlink_node) => { - nar_node.symlink(&proto_symlink_node.target)?; - } - proto::node::Node::File(proto_file_node) => { - nar_node.file( - proto_file_node.executable, - proto_file_node.size.into(), - &mut store_client.open_blob(proto_file_node.digest)?, - )?; - } - proto::node::Node::Directory(proto_directory_node) => { - // look up that node from the store client - let proto_directory = store_client.get_directory(proto_directory_node.digest)?; - - // if it's None, that's an error! - if proto_directory.is_none() { - // TODO: proper error handling - panic!("not found!") - } - - // start a directory node - let mut nar_node_directory = nar_node.directory()?; - - // for each node in the directory, create a new entry with its name, - // and then invoke walk_node on that entry. - for proto_node in proto_directory.unwrap().nodes() { - let child_node = nar_node_directory.entry(proto_node.get_name())?; - walk_node(child_node, proto_node, store_client)?; - } - - // close the directory - nar_node_directory.close()?; - } - } - Ok(()) -} diff --git a/tvix/store/src/nar/mod.rs b/tvix/store/src/nar/mod.rs new file mode 100644 index 000000000000..d7d2cec4d803 --- /dev/null +++ b/tvix/store/src/nar/mod.rs @@ -0,0 +1,22 @@ +use data_encoding::BASE64; +use thiserror::Error; + +mod renderer; + +pub use renderer::NARRenderer; + +/// Errors that can encounter while rendering NARs. +#[derive(Debug, Error)] +pub enum RenderError { + #[error("failure talking to a backing store client: {0}")] + StoreError(crate::Error), + + #[error("unable to find directory {}, referred from {}", BASE64.encode(.0), .1)] + DirectoryNotFound(Vec, String), + + #[error("unable to find blob {}, referred from {}", BASE64.encode(.0), .1)] + BlobNotFound(Vec, String), + + #[error("failure using the NAR writer: {0}")] + NARWriterError(std::io::Error), +} diff --git a/tvix/store/src/nar/renderer.rs b/tvix/store/src/nar/renderer.rs new file mode 100644 index 000000000000..d8d9886b315d --- /dev/null +++ b/tvix/store/src/nar/renderer.rs @@ -0,0 +1,136 @@ +use crate::{ + blobservice::BlobService, + chunkservice::ChunkService, + directoryservice::DirectoryService, + proto::{self, NamedNode}, + BlobReader, +}; +use nix_compat::nar; + +use super::RenderError; + +/// A NAR renderer, using a blob_service, chunk_service and directory_service +/// to render a NAR to a writer. +#[derive(Clone)] +pub struct NARRenderer { + blob_service: BS, + chunk_service: CS, + directory_service: DS, +} + +impl NARRenderer { + pub fn new(blob_service: BS, chunk_service: CS, directory_service: DS) -> Self { + Self { + blob_service, + chunk_service, + directory_service, + } + } + + /// Consumes a [proto::node::Node] pointing to the root of a (store) path, + /// and writes the contents in NAR serialization to the passed + /// [std::io::Write]. + /// + /// It uses the different clients in the struct to perform the necessary + /// lookups as it traverses the structure. + pub fn write_nar( + &self, + w: &mut W, + proto_root_node: proto::node::Node, + ) -> Result<(), RenderError> { + // Initialize NAR writer + let nar_root_node = nar::writer::open(w).map_err(RenderError::NARWriterError)?; + + self.walk_node(nar_root_node, proto_root_node) + } + + /// Process an intermediate node in the structure. + /// This consumes the node. + fn walk_node( + &self, + nar_node: nar::writer::Node, + proto_node: proto::node::Node, + ) -> Result<(), RenderError> { + match proto_node { + proto::node::Node::Symlink(proto_symlink_node) => { + nar_node + .symlink(&proto_symlink_node.target) + .map_err(RenderError::NARWriterError)?; + } + proto::node::Node::File(proto_file_node) => { + // get the digest we're referring to + let digest = proto_file_node.digest; + // query blob_service for blob_meta + let resp = self + .blob_service + .stat(&proto::StatBlobRequest { + digest: digest.to_vec(), + include_chunks: true, + ..Default::default() + }) + .map_err(RenderError::StoreError)?; + + match resp { + // if it's None, that's an error! + None => { + return Err(RenderError::BlobNotFound(digest, proto_file_node.name)); + } + Some(blob_meta) => { + let mut blob_reader = std::io::BufReader::new(BlobReader::open( + &self.chunk_service, + blob_meta, + )); + nar_node + .file( + proto_file_node.executable, + proto_file_node.size.into(), + &mut blob_reader, + ) + .map_err(RenderError::NARWriterError)?; + } + } + } + proto::node::Node::Directory(proto_directory_node) => { + // get the digest we're referring to + let digest = proto_directory_node.digest; + // look it up with the directory service + let resp = self + .directory_service + .get(&proto::get_directory_request::ByWhat::Digest( + digest.to_vec(), + )) + .map_err(RenderError::StoreError)?; + + match resp { + // if it's None, that's an error! + None => { + return Err(RenderError::DirectoryNotFound( + digest, + proto_directory_node.name, + )) + } + Some(proto_directory) => { + // start a directory node + let mut nar_node_directory = + nar_node.directory().map_err(RenderError::NARWriterError)?; + + // for each node in the directory, create a new entry with its name, + // and then invoke walk_node on that entry. + for proto_node in proto_directory.nodes() { + let child_node = nar_node_directory + .entry(proto_node.get_name()) + .map_err(RenderError::NARWriterError)?; + self.walk_node(child_node, proto_node)?; + } + + // close the directory + nar_node_directory + .close() + .map_err(RenderError::NARWriterError)?; + } + } + } + } + Ok(()) + } +} diff --git a/tvix/store/src/tests/mod.rs b/tvix/store/src/tests/mod.rs index b945763f3686..6947b277d481 100644 --- a/tvix/store/src/tests/mod.rs +++ b/tvix/store/src/tests/mod.rs @@ -1,3 +1,3 @@ mod directory_service; -mod nar; +mod nar_renderer; mod path_info_service; diff --git a/tvix/store/src/tests/nar.rs b/tvix/store/src/tests/nar.rs deleted file mode 100644 index 5a865f52b492..000000000000 --- a/tvix/store/src/tests/nar.rs +++ /dev/null @@ -1,267 +0,0 @@ -use data_encoding::BASE64; - -use crate::client::StoreClient; -use crate::nar::write_nar; -use crate::proto; -use crate::proto::DirectoryNode; -use crate::proto::FileNode; -use crate::proto::SymlinkNode; -use lazy_static::lazy_static; - -const HELLOWORLD_BLOB_CONTENTS: &[u8] = b"Hello World!"; -const EMPTY_BLOB_CONTENTS: &[u8] = b""; - -lazy_static! { - static ref HELLOWORLD_BLOB_DIGEST: Vec = - blake3::hash(HELLOWORLD_BLOB_CONTENTS).as_bytes().to_vec(); - static ref EMPTY_BLOB_DIGEST: Vec = blake3::hash(EMPTY_BLOB_CONTENTS).as_bytes().to_vec(); - static ref DIRECTORY_WITH_KEEP: proto::Directory = proto::Directory { - directories: vec![], - files: vec![FileNode { - name: ".keep".to_string(), - digest: EMPTY_BLOB_DIGEST.to_vec(), - size: 0, - executable: false, - }], - symlinks: vec![], - }; - static ref DIRECTORY_COMPLICATED: proto::Directory = proto::Directory { - directories: vec![DirectoryNode { - name: "keep".to_string(), - digest: DIRECTORY_WITH_KEEP.digest(), - size: DIRECTORY_WITH_KEEP.size(), - }], - files: vec![FileNode { - name: ".keep".to_string(), - digest: EMPTY_BLOB_DIGEST.to_vec(), - size: 0, - executable: false, - }], - symlinks: vec![SymlinkNode { - name: "aa".to_string(), - target: "/nix/store/somewhereelse".to_string(), - }], - }; -} - -/// A Store client that fails if you ask it for a blob or a directory -#[derive(Default)] -struct FailingStoreClient {} - -impl StoreClient for FailingStoreClient { - fn open_blob(&self, digest: Vec) -> std::io::Result> { - panic!( - "open_blob should never be called, but was called with {}", - BASE64.encode(&digest), - ); - } - - fn get_directory(&self, digest: Vec) -> std::io::Result> { - panic!( - "get_directory should never be called, but was called with {}", - BASE64.encode(&digest), - ); - } -} - -/// Only allow a request for a blob with [HELLOWORLD_BLOB_DIGEST] -/// panic on everything else. -#[derive(Default)] -struct HelloWorldBlobStoreClient {} - -impl StoreClient for HelloWorldBlobStoreClient { - fn open_blob(&self, digest: Vec) -> std::io::Result> { - if digest != HELLOWORLD_BLOB_DIGEST.to_vec() { - panic!("open_blob called with {}", BASE64.encode(&digest)); - } - - let b: Box<&[u8]> = Box::new(&HELLOWORLD_BLOB_CONTENTS); - - Ok(b) - } - - fn get_directory(&self, digest: Vec) -> std::io::Result> { - panic!( - "get_directory should never be called, but was called with {}", - BASE64.encode(&digest), - ); - } -} - -/// Allow blob requests for [HELLOWORLD_BLOB_DIGEST] and EMPTY_BLOB_DIGEST, and -/// allow DIRECTORY_WITH_KEEP and DIRECTORY_COMPLICATED. -#[derive(Default)] -struct SomeDirectoryStoreClient {} - -impl StoreClient for SomeDirectoryStoreClient { - fn open_blob(&self, digest: Vec) -> std::io::Result> { - if digest == HELLOWORLD_BLOB_DIGEST.to_vec() { - let b: Box<&[u8]> = Box::new(&HELLOWORLD_BLOB_CONTENTS); - return Ok(b); - } - if digest == EMPTY_BLOB_DIGEST.to_vec() { - let b: Box<&[u8]> = Box::new(&EMPTY_BLOB_CONTENTS); - return Ok(b); - } - panic!("open_blob called with {}", BASE64.encode(&digest)); - } - - fn get_directory(&self, digest: Vec) -> std::io::Result> { - if digest == DIRECTORY_WITH_KEEP.digest() { - return Ok(Some(DIRECTORY_WITH_KEEP.clone())); - } - if digest == DIRECTORY_COMPLICATED.digest() { - return Ok(Some(DIRECTORY_COMPLICATED.clone())); - } - panic!("get_directory called with {}", BASE64.encode(&digest)); - } -} - -#[tokio::test] -async fn single_symlink() -> anyhow::Result<()> { - let mut buf: Vec = vec![]; - let mut store_client = FailingStoreClient::default(); - - write_nar( - &mut buf, - crate::proto::node::Node::Symlink(SymlinkNode { - name: "doesntmatter".to_string(), - target: "/nix/store/somewhereelse".to_string(), - }), - &mut store_client, - ) - .expect("must succeed"); - - assert_eq!( - buf, - vec![ - 13, 0, 0, 0, 0, 0, 0, 0, 110, 105, 120, 45, 97, 114, 99, 104, 105, 118, 101, 45, 49, 0, - 0, 0, // "nix-archive-1" - 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" - 4, 0, 0, 0, 0, 0, 0, 0, 116, 121, 112, 101, 0, 0, 0, 0, // "type" - 7, 0, 0, 0, 0, 0, 0, 0, 115, 121, 109, 108, 105, 110, 107, 0, // "symlink" - 6, 0, 0, 0, 0, 0, 0, 0, 116, 97, 114, 103, 101, 116, 0, 0, // target - 24, 0, 0, 0, 0, 0, 0, 0, 47, 110, 105, 120, 47, 115, 116, 111, 114, 101, 47, 115, 111, - 109, 101, 119, 104, 101, 114, 101, 101, 108, 115, - 101, // "/nix/store/somewhereelse" - 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0 // ")" - ] - ); - Ok(()) -} - -#[tokio::test] -async fn single_file() -> anyhow::Result<()> { - let mut buf: Vec = vec![]; - let mut store_client = HelloWorldBlobStoreClient::default(); - - write_nar( - &mut buf, - crate::proto::node::Node::File(FileNode { - name: "doesntmatter".to_string(), - digest: HELLOWORLD_BLOB_DIGEST.to_vec(), - size: HELLOWORLD_BLOB_CONTENTS.len() as u32, - executable: false, - }), - &mut store_client, - ) - .expect("must succeed"); - - assert_eq!( - buf, - vec![ - 13, 0, 0, 0, 0, 0, 0, 0, 110, 105, 120, 45, 97, 114, 99, 104, 105, 118, 101, 45, 49, 0, - 0, 0, // "nix-archive-1" - 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" - 4, 0, 0, 0, 0, 0, 0, 0, 116, 121, 112, 101, 0, 0, 0, 0, // "type" - 7, 0, 0, 0, 0, 0, 0, 0, 114, 101, 103, 117, 108, 97, 114, 0, // "regular" - 8, 0, 0, 0, 0, 0, 0, 0, 99, 111, 110, 116, 101, 110, 116, 115, // "contents" - 12, 0, 0, 0, 0, 0, 0, 0, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 0, 0, - 0, 0, // "Hello World!" - 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0 // ")" - ] - ); - - Ok(()) -} - -#[tokio::test] -async fn test_complicated() -> anyhow::Result<()> { - let mut buf: Vec = vec![]; - let mut store_client = SomeDirectoryStoreClient::default(); - - write_nar( - &mut buf, - crate::proto::node::Node::Directory(DirectoryNode { - name: "doesntmatter".to_string(), - digest: DIRECTORY_COMPLICATED.digest(), - size: DIRECTORY_COMPLICATED.size() as u32, - }), - &mut store_client, - ) - .expect("must succeed"); - - assert_eq!( - buf, - vec![ - 13, 0, 0, 0, 0, 0, 0, 0, 110, 105, 120, 45, 97, 114, 99, 104, 105, 118, 101, 45, 49, 0, - 0, 0, // "nix-archive-1" - 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" - 4, 0, 0, 0, 0, 0, 0, 0, 116, 121, 112, 101, 0, 0, 0, 0, // "type" - 9, 0, 0, 0, 0, 0, 0, 0, 100, 105, 114, 101, 99, 116, 111, 114, 121, 0, 0, 0, 0, 0, 0, - 0, // "directory" - 5, 0, 0, 0, 0, 0, 0, 0, 101, 110, 116, 114, 121, 0, 0, 0, // "entry" - 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" - 4, 0, 0, 0, 0, 0, 0, 0, 110, 97, 109, 101, 0, 0, 0, 0, // "name" - 5, 0, 0, 0, 0, 0, 0, 0, 46, 107, 101, 101, 112, 0, 0, 0, // ".keep" - 4, 0, 0, 0, 0, 0, 0, 0, 110, 111, 100, 101, 0, 0, 0, 0, // "node" - 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" - 4, 0, 0, 0, 0, 0, 0, 0, 116, 121, 112, 101, 0, 0, 0, 0, // "type" - 7, 0, 0, 0, 0, 0, 0, 0, 114, 101, 103, 117, 108, 97, 114, 0, // "regular" - 8, 0, 0, 0, 0, 0, 0, 0, 99, 111, 110, 116, 101, 110, 116, 115, 0, 0, 0, 0, 0, 0, 0, - 0, // "contents" - 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, // ")" - 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, // ")" - 5, 0, 0, 0, 0, 0, 0, 0, 101, 110, 116, 114, 121, 0, 0, 0, // "entry" - 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" - 4, 0, 0, 0, 0, 0, 0, 0, 110, 97, 109, 101, 0, 0, 0, 0, // "name" - 2, 0, 0, 0, 0, 0, 0, 0, 97, 97, 0, 0, 0, 0, 0, 0, // "aa" - 4, 0, 0, 0, 0, 0, 0, 0, 110, 111, 100, 101, 0, 0, 0, 0, // "node" - 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" - 4, 0, 0, 0, 0, 0, 0, 0, 116, 121, 112, 101, 0, 0, 0, 0, // "type" - 7, 0, 0, 0, 0, 0, 0, 0, 115, 121, 109, 108, 105, 110, 107, 0, // "symlink" - 6, 0, 0, 0, 0, 0, 0, 0, 116, 97, 114, 103, 101, 116, 0, 0, // "target" - 24, 0, 0, 0, 0, 0, 0, 0, 47, 110, 105, 120, 47, 115, 116, 111, 114, 101, 47, 115, 111, - 109, 101, 119, 104, 101, 114, 101, 101, 108, 115, - 101, // "/nix/store/somewhereelse" - 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, // ")" - 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, // ")" - 5, 0, 0, 0, 0, 0, 0, 0, 101, 110, 116, 114, 121, 0, 0, 0, // "entry" - 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" - 4, 0, 0, 0, 0, 0, 0, 0, 110, 97, 109, 101, 0, 0, 0, 0, // "name" - 4, 0, 0, 0, 0, 0, 0, 0, 107, 101, 101, 112, 0, 0, 0, 0, // "keep" - 4, 0, 0, 0, 0, 0, 0, 0, 110, 111, 100, 101, 0, 0, 0, 0, // "node" - 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" - 4, 0, 0, 0, 0, 0, 0, 0, 116, 121, 112, 101, 0, 0, 0, 0, // "type" - 9, 0, 0, 0, 0, 0, 0, 0, 100, 105, 114, 101, 99, 116, 111, 114, 121, 0, 0, 0, 0, 0, 0, - 0, // "directory" - 5, 0, 0, 0, 0, 0, 0, 0, 101, 110, 116, 114, 121, 0, 0, 0, // "entry" - 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" - 4, 0, 0, 0, 0, 0, 0, 0, 110, 97, 109, 101, 0, 0, 0, 0, // "name" - 5, 0, 0, 0, 0, 0, 0, 0, 46, 107, 101, 101, 112, 0, 0, 0, // ".keep" - 4, 0, 0, 0, 0, 0, 0, 0, 110, 111, 100, 101, 0, 0, 0, 0, // "node" - 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" - 4, 0, 0, 0, 0, 0, 0, 0, 116, 121, 112, 101, 0, 0, 0, 0, // "type" - 7, 0, 0, 0, 0, 0, 0, 0, 114, 101, 103, 117, 108, 97, 114, 0, // "regular" - 8, 0, 0, 0, 0, 0, 0, 0, 99, 111, 110, 116, 101, 110, 116, 115, 0, 0, 0, 0, 0, 0, 0, - 0, // "contents" - 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, // ")" - 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, // ")" - 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, // ")" - 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, // ")" - 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0 // ")" - ] - ); - - Ok(()) -} diff --git a/tvix/store/src/tests/nar_renderer.rs b/tvix/store/src/tests/nar_renderer.rs new file mode 100644 index 000000000000..43afdefbf930 --- /dev/null +++ b/tvix/store/src/tests/nar_renderer.rs @@ -0,0 +1,273 @@ +use crate::blobservice::BlobService; +use crate::blobservice::SledBlobService; +use crate::chunkservice::ChunkService; +use crate::chunkservice::SledChunkService; +use crate::directoryservice::DirectoryService; +use crate::directoryservice::SledDirectoryService; +use crate::nar::NARRenderer; +use crate::proto; +use crate::proto::DirectoryNode; +use crate::proto::FileNode; +use crate::proto::SymlinkNode; +use lazy_static::lazy_static; +use std::path::Path; +use tempfile::TempDir; + +const HELLOWORLD_BLOB_CONTENTS: &[u8] = b"Hello World!"; +const EMPTY_BLOB_CONTENTS: &[u8] = b""; + +lazy_static! { + static ref HELLOWORLD_BLOB_DIGEST: Vec = + blake3::hash(HELLOWORLD_BLOB_CONTENTS).as_bytes().to_vec(); + static ref EMPTY_BLOB_DIGEST: Vec = blake3::hash(EMPTY_BLOB_CONTENTS).as_bytes().to_vec(); + static ref DIRECTORY_WITH_KEEP: proto::Directory = proto::Directory { + directories: vec![], + files: vec![FileNode { + name: ".keep".to_string(), + digest: EMPTY_BLOB_DIGEST.to_vec(), + size: 0, + executable: false, + }], + symlinks: vec![], + }; + static ref DIRECTORY_COMPLICATED: proto::Directory = proto::Directory { + directories: vec![DirectoryNode { + name: "keep".to_string(), + digest: DIRECTORY_WITH_KEEP.digest(), + size: DIRECTORY_WITH_KEEP.size(), + }], + files: vec![FileNode { + name: ".keep".to_string(), + digest: EMPTY_BLOB_DIGEST.to_vec(), + size: 0, + executable: false, + }], + symlinks: vec![SymlinkNode { + name: "aa".to_string(), + target: "/nix/store/somewhereelse".to_string(), + }], + }; +} + +fn gen_blob_service(p: &Path) -> impl BlobService { + SledBlobService::new(p.join("blobs")).unwrap() +} + +fn gen_chunk_service(p: &Path) -> impl ChunkService + Clone { + SledChunkService::new(p.join("chunks")).unwrap() +} + +fn gen_directory_service(p: &Path) -> impl DirectoryService { + SledDirectoryService::new(p.join("directories")).unwrap() +} + +#[test] +fn single_symlink() -> anyhow::Result<()> { + let tmpdir = TempDir::new()?; + let renderer = NARRenderer::new( + gen_blob_service(tmpdir.path()), + gen_chunk_service(tmpdir.path()), + gen_directory_service(tmpdir.path()), + ); + // don't put anything in the stores, as we don't actually do any requests. + + let mut buf: Vec = vec![]; + + renderer + .write_nar( + &mut buf, + crate::proto::node::Node::Symlink(SymlinkNode { + name: "doesntmatter".to_string(), + target: "/nix/store/somewhereelse".to_string(), + }), + ) + .expect("must succeed"); + + assert_eq!( + buf, + vec![ + 13, 0, 0, 0, 0, 0, 0, 0, 110, 105, 120, 45, 97, 114, 99, 104, 105, 118, 101, 45, 49, 0, + 0, 0, // "nix-archive-1" + 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" + 4, 0, 0, 0, 0, 0, 0, 0, 116, 121, 112, 101, 0, 0, 0, 0, // "type" + 7, 0, 0, 0, 0, 0, 0, 0, 115, 121, 109, 108, 105, 110, 107, 0, // "symlink" + 6, 0, 0, 0, 0, 0, 0, 0, 116, 97, 114, 103, 101, 116, 0, 0, // target + 24, 0, 0, 0, 0, 0, 0, 0, 47, 110, 105, 120, 47, 115, 116, 111, 114, 101, 47, 115, 111, + 109, 101, 119, 104, 101, 114, 101, 101, 108, 115, + 101, // "/nix/store/somewhereelse" + 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0 // ")" + ] + ); + Ok(()) +} + +#[test] +fn single_file() -> anyhow::Result<()> { + let tmpdir = TempDir::new()?; + + let blob_service = gen_blob_service(tmpdir.path()); + let chunk_service = gen_chunk_service(tmpdir.path()); + + chunk_service + .put(HELLOWORLD_BLOB_CONTENTS.to_vec()) + .unwrap(); + + blob_service + .put( + &HELLOWORLD_BLOB_DIGEST, + proto::BlobMeta { + chunks: vec![proto::blob_meta::ChunkMeta { + digest: HELLOWORLD_BLOB_DIGEST.to_vec(), + size: HELLOWORLD_BLOB_CONTENTS.len() as u32, + }], + ..Default::default() + }, + ) + .unwrap(); + + let renderer = NARRenderer::new( + blob_service, + chunk_service, + gen_directory_service(tmpdir.path()), + ); + let mut buf: Vec = vec![]; + + renderer + .write_nar( + &mut buf, + crate::proto::node::Node::File(FileNode { + name: "doesntmatter".to_string(), + digest: HELLOWORLD_BLOB_DIGEST.to_vec(), + size: HELLOWORLD_BLOB_CONTENTS.len() as u32, + executable: false, + }), + ) + .expect("must succeed"); + + assert_eq!( + buf, + vec![ + 13, 0, 0, 0, 0, 0, 0, 0, 110, 105, 120, 45, 97, 114, 99, 104, 105, 118, 101, 45, 49, 0, + 0, 0, // "nix-archive-1" + 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" + 4, 0, 0, 0, 0, 0, 0, 0, 116, 121, 112, 101, 0, 0, 0, 0, // "type" + 7, 0, 0, 0, 0, 0, 0, 0, 114, 101, 103, 117, 108, 97, 114, 0, // "regular" + 8, 0, 0, 0, 0, 0, 0, 0, 99, 111, 110, 116, 101, 110, 116, 115, // "contents" + 12, 0, 0, 0, 0, 0, 0, 0, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 0, 0, + 0, 0, // "Hello World!" + 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0 // ")" + ] + ); + + Ok(()) +} + +#[test] +fn test_complicated() -> anyhow::Result<()> { + let tmpdir = TempDir::new()?; + + let blob_service = gen_blob_service(tmpdir.path()); + let chunk_service = gen_chunk_service(tmpdir.path()); + let directory_service = gen_directory_service(tmpdir.path()); + + // put all data into the stores. + for blob_contents in [HELLOWORLD_BLOB_CONTENTS, EMPTY_BLOB_CONTENTS] { + let digest = chunk_service.put(blob_contents.to_vec()).unwrap(); + + blob_service + .put( + &digest, + proto::BlobMeta { + chunks: vec![proto::blob_meta::ChunkMeta { + digest: digest.to_vec(), + size: blob_contents.len() as u32, + }], + ..Default::default() + }, + ) + .unwrap(); + } + + directory_service.put(DIRECTORY_WITH_KEEP.clone()).unwrap(); + directory_service + .put(DIRECTORY_COMPLICATED.clone()) + .unwrap(); + + let renderer = NARRenderer::new(blob_service, chunk_service, directory_service); + let mut buf: Vec = vec![]; + + renderer + .write_nar( + &mut buf, + crate::proto::node::Node::Directory(DirectoryNode { + name: "doesntmatter".to_string(), + digest: DIRECTORY_COMPLICATED.digest(), + size: DIRECTORY_COMPLICATED.size(), + }), + ) + .expect("must succeed"); + + assert_eq!( + buf, + vec![ + 13, 0, 0, 0, 0, 0, 0, 0, 110, 105, 120, 45, 97, 114, 99, 104, 105, 118, 101, 45, 49, 0, + 0, 0, // "nix-archive-1" + 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" + 4, 0, 0, 0, 0, 0, 0, 0, 116, 121, 112, 101, 0, 0, 0, 0, // "type" + 9, 0, 0, 0, 0, 0, 0, 0, 100, 105, 114, 101, 99, 116, 111, 114, 121, 0, 0, 0, 0, 0, 0, + 0, // "directory" + 5, 0, 0, 0, 0, 0, 0, 0, 101, 110, 116, 114, 121, 0, 0, 0, // "entry" + 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" + 4, 0, 0, 0, 0, 0, 0, 0, 110, 97, 109, 101, 0, 0, 0, 0, // "name" + 5, 0, 0, 0, 0, 0, 0, 0, 46, 107, 101, 101, 112, 0, 0, 0, // ".keep" + 4, 0, 0, 0, 0, 0, 0, 0, 110, 111, 100, 101, 0, 0, 0, 0, // "node" + 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" + 4, 0, 0, 0, 0, 0, 0, 0, 116, 121, 112, 101, 0, 0, 0, 0, // "type" + 7, 0, 0, 0, 0, 0, 0, 0, 114, 101, 103, 117, 108, 97, 114, 0, // "regular" + 8, 0, 0, 0, 0, 0, 0, 0, 99, 111, 110, 116, 101, 110, 116, 115, 0, 0, 0, 0, 0, 0, 0, + 0, // "contents" + 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, // ")" + 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, // ")" + 5, 0, 0, 0, 0, 0, 0, 0, 101, 110, 116, 114, 121, 0, 0, 0, // "entry" + 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" + 4, 0, 0, 0, 0, 0, 0, 0, 110, 97, 109, 101, 0, 0, 0, 0, // "name" + 2, 0, 0, 0, 0, 0, 0, 0, 97, 97, 0, 0, 0, 0, 0, 0, // "aa" + 4, 0, 0, 0, 0, 0, 0, 0, 110, 111, 100, 101, 0, 0, 0, 0, // "node" + 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" + 4, 0, 0, 0, 0, 0, 0, 0, 116, 121, 112, 101, 0, 0, 0, 0, // "type" + 7, 0, 0, 0, 0, 0, 0, 0, 115, 121, 109, 108, 105, 110, 107, 0, // "symlink" + 6, 0, 0, 0, 0, 0, 0, 0, 116, 97, 114, 103, 101, 116, 0, 0, // "target" + 24, 0, 0, 0, 0, 0, 0, 0, 47, 110, 105, 120, 47, 115, 116, 111, 114, 101, 47, 115, 111, + 109, 101, 119, 104, 101, 114, 101, 101, 108, 115, + 101, // "/nix/store/somewhereelse" + 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, // ")" + 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, // ")" + 5, 0, 0, 0, 0, 0, 0, 0, 101, 110, 116, 114, 121, 0, 0, 0, // "entry" + 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" + 4, 0, 0, 0, 0, 0, 0, 0, 110, 97, 109, 101, 0, 0, 0, 0, // "name" + 4, 0, 0, 0, 0, 0, 0, 0, 107, 101, 101, 112, 0, 0, 0, 0, // "keep" + 4, 0, 0, 0, 0, 0, 0, 0, 110, 111, 100, 101, 0, 0, 0, 0, // "node" + 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" + 4, 0, 0, 0, 0, 0, 0, 0, 116, 121, 112, 101, 0, 0, 0, 0, // "type" + 9, 0, 0, 0, 0, 0, 0, 0, 100, 105, 114, 101, 99, 116, 111, 114, 121, 0, 0, 0, 0, 0, 0, + 0, // "directory" + 5, 0, 0, 0, 0, 0, 0, 0, 101, 110, 116, 114, 121, 0, 0, 0, // "entry" + 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" + 4, 0, 0, 0, 0, 0, 0, 0, 110, 97, 109, 101, 0, 0, 0, 0, // "name" + 5, 0, 0, 0, 0, 0, 0, 0, 46, 107, 101, 101, 112, 0, 0, 0, // ".keep" + 4, 0, 0, 0, 0, 0, 0, 0, 110, 111, 100, 101, 0, 0, 0, 0, // "node" + 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, // "(" + 4, 0, 0, 0, 0, 0, 0, 0, 116, 121, 112, 101, 0, 0, 0, 0, // "type" + 7, 0, 0, 0, 0, 0, 0, 0, 114, 101, 103, 117, 108, 97, 114, 0, // "regular" + 8, 0, 0, 0, 0, 0, 0, 0, 99, 111, 110, 116, 101, 110, 116, 115, 0, 0, 0, 0, 0, 0, 0, + 0, // "contents" + 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, // ")" + 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, // ")" + 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, // ")" + 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, // ")" + 1, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0 // ")" + ] + ); + + Ok(()) +} -- cgit 1.4.1