From df3223fd681f64d54f6f393e53647d3a487eff25 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Mon, 13 Feb 2023 16:44:26 +0100 Subject: chore(tvix/store): move NAR rendering logic into Renderer struct This moves the logic rendering NARs to a struct using the previously introduced, more granular BlobService, ChunkService and DirectoryService. Instead of passing them around to the helper functions, they're kept as members of a struct. Remove the async invocations in the nar_renderer tests, there's nothing async in here. Change-Id: Ic6d24aaad68a1fda46ce29f2cdb5f7b87f481d5c Reviewed-on: https://cl.tvl.fyi/c/depot/+/8095 Reviewed-by: raitobezarius Tested-by: BuildkiteCI --- tvix/store/src/nar/mod.rs | 22 +++++++ tvix/store/src/nar/renderer.rs | 136 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 tvix/store/src/nar/mod.rs create mode 100644 tvix/store/src/nar/renderer.rs (limited to 'tvix/store/src/nar') diff --git a/tvix/store/src/nar/mod.rs b/tvix/store/src/nar/mod.rs new file mode 100644 index 000000000000..d7d2cec4d803 --- /dev/null +++ b/tvix/store/src/nar/mod.rs @@ -0,0 +1,22 @@ +use data_encoding::BASE64; +use thiserror::Error; + +mod renderer; + +pub use renderer::NARRenderer; + +/// Errors that can encounter while rendering NARs. +#[derive(Debug, Error)] +pub enum RenderError { + #[error("failure talking to a backing store client: {0}")] + StoreError(crate::Error), + + #[error("unable to find directory {}, referred from {}", BASE64.encode(.0), .1)] + DirectoryNotFound(Vec, String), + + #[error("unable to find blob {}, referred from {}", BASE64.encode(.0), .1)] + BlobNotFound(Vec, String), + + #[error("failure using the NAR writer: {0}")] + NARWriterError(std::io::Error), +} diff --git a/tvix/store/src/nar/renderer.rs b/tvix/store/src/nar/renderer.rs new file mode 100644 index 000000000000..d8d9886b315d --- /dev/null +++ b/tvix/store/src/nar/renderer.rs @@ -0,0 +1,136 @@ +use crate::{ + blobservice::BlobService, + chunkservice::ChunkService, + directoryservice::DirectoryService, + proto::{self, NamedNode}, + BlobReader, +}; +use nix_compat::nar; + +use super::RenderError; + +/// A NAR renderer, using a blob_service, chunk_service and directory_service +/// to render a NAR to a writer. +#[derive(Clone)] +pub struct NARRenderer { + blob_service: BS, + chunk_service: CS, + directory_service: DS, +} + +impl NARRenderer { + pub fn new(blob_service: BS, chunk_service: CS, directory_service: DS) -> Self { + Self { + blob_service, + chunk_service, + directory_service, + } + } + + /// Consumes a [proto::node::Node] pointing to the root of a (store) path, + /// and writes the contents in NAR serialization to the passed + /// [std::io::Write]. + /// + /// It uses the different clients in the struct to perform the necessary + /// lookups as it traverses the structure. + pub fn write_nar( + &self, + w: &mut W, + proto_root_node: proto::node::Node, + ) -> Result<(), RenderError> { + // Initialize NAR writer + let nar_root_node = nar::writer::open(w).map_err(RenderError::NARWriterError)?; + + self.walk_node(nar_root_node, proto_root_node) + } + + /// Process an intermediate node in the structure. + /// This consumes the node. + fn walk_node( + &self, + nar_node: nar::writer::Node, + proto_node: proto::node::Node, + ) -> Result<(), RenderError> { + match proto_node { + proto::node::Node::Symlink(proto_symlink_node) => { + nar_node + .symlink(&proto_symlink_node.target) + .map_err(RenderError::NARWriterError)?; + } + proto::node::Node::File(proto_file_node) => { + // get the digest we're referring to + let digest = proto_file_node.digest; + // query blob_service for blob_meta + let resp = self + .blob_service + .stat(&proto::StatBlobRequest { + digest: digest.to_vec(), + include_chunks: true, + ..Default::default() + }) + .map_err(RenderError::StoreError)?; + + match resp { + // if it's None, that's an error! + None => { + return Err(RenderError::BlobNotFound(digest, proto_file_node.name)); + } + Some(blob_meta) => { + let mut blob_reader = std::io::BufReader::new(BlobReader::open( + &self.chunk_service, + blob_meta, + )); + nar_node + .file( + proto_file_node.executable, + proto_file_node.size.into(), + &mut blob_reader, + ) + .map_err(RenderError::NARWriterError)?; + } + } + } + proto::node::Node::Directory(proto_directory_node) => { + // get the digest we're referring to + let digest = proto_directory_node.digest; + // look it up with the directory service + let resp = self + .directory_service + .get(&proto::get_directory_request::ByWhat::Digest( + digest.to_vec(), + )) + .map_err(RenderError::StoreError)?; + + match resp { + // if it's None, that's an error! + None => { + return Err(RenderError::DirectoryNotFound( + digest, + proto_directory_node.name, + )) + } + Some(proto_directory) => { + // start a directory node + let mut nar_node_directory = + nar_node.directory().map_err(RenderError::NARWriterError)?; + + // for each node in the directory, create a new entry with its name, + // and then invoke walk_node on that entry. + for proto_node in proto_directory.nodes() { + let child_node = nar_node_directory + .entry(proto_node.get_name()) + .map_err(RenderError::NARWriterError)?; + self.walk_node(child_node, proto_node)?; + } + + // close the directory + nar_node_directory + .close() + .map_err(RenderError::NARWriterError)?; + } + } + } + } + Ok(()) + } +} -- cgit 1.4.1