From 32f41458c0a0f62bf906021ef096c465ccc45581 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Thu, 21 Sep 2023 22:32:44 +0300 Subject: refactor(tvix): move castore into tvix-castore crate This splits the pure content-addressed layers from tvix-store into a `castore` crate, and only leaves PathInfo related things, as well as the CLI entrypoint in the tvix-store crate. Notable changes: - `fixtures` and `utils` had to be moved out of the `test` cfg, so they can be imported from tvix-store. - Some ad-hoc fixtures in the test were moved to proper fixtures in the same step. - The protos are now created by a (more static) recipe in the protos/ directory. The (now two) golang targets are commented out, as it's not possible to update them properly in the same CL. This will be done by a followup CL once this is merged (and whitby deployed) Bug: https://b.tvl.fyi/issues/301 Change-Id: I8d675d4bf1fb697eb7d479747c1b1e3635718107 Reviewed-on: https://cl.tvl.fyi/c/depot/+/9370 Reviewed-by: tazjin Reviewed-by: flokli Autosubmit: flokli Tested-by: BuildkiteCI Reviewed-by: Connor Brewster --- tvix/store/src/nar/mod.rs | 7 +++---- tvix/store/src/nar/renderer.rs | 33 +++++++++++++++++---------------- 2 files changed, 20 insertions(+), 20 deletions(-) (limited to 'tvix/store/src/nar') diff --git a/tvix/store/src/nar/mod.rs b/tvix/store/src/nar/mod.rs index 5a8bc21ae953..fc6805e9e758 100644 --- a/tvix/store/src/nar/mod.rs +++ b/tvix/store/src/nar/mod.rs @@ -1,16 +1,15 @@ -use crate::B3Digest; use data_encoding::BASE64; -use thiserror::Error; +use tvix_castore::{B3Digest, Error}; mod renderer; pub use renderer::calculate_size_and_sha256; pub use renderer::write_nar; /// Errors that can encounter while rendering NARs. -#[derive(Debug, Error)] +#[derive(Debug, thiserror::Error)] pub enum RenderError { #[error("failure talking to a backing store client: {0}")] - StoreError(crate::Error), + StoreError(Error), #[error("unable to find directory {}, referred from {:?}", .0, .1)] DirectoryNotFound(B3Digest, bytes::Bytes), diff --git a/tvix/store/src/nar/renderer.rs b/tvix/store/src/nar/renderer.rs index f1392472a50e..55dce911ee1a 100644 --- a/tvix/store/src/nar/renderer.rs +++ b/tvix/store/src/nar/renderer.rs @@ -1,20 +1,21 @@ use super::RenderError; -use crate::{ - blobservice::BlobService, - directoryservice::DirectoryService, - proto::{self, NamedNode}, -}; use count_write::CountWrite; use nix_compat::nar; use sha2::{Digest, Sha256}; use std::{io, sync::Arc}; use tokio::{io::BufReader, task::spawn_blocking}; use tracing::warn; +use tvix_castore::{ + blobservice::BlobService, + directoryservice::DirectoryService, + proto::{self as castorepb, NamedNode}, + Error, +}; /// Invoke [write_nar], and return the size and sha256 digest of the produced /// NAR output. pub async fn calculate_size_and_sha256( - root_node: &proto::node::Node, + root_node: &castorepb::node::Node, blob_service: Arc, directory_service: Arc, ) -> Result<(u64, [u8; 32]), RenderError> { @@ -26,9 +27,9 @@ pub async fn calculate_size_and_sha256( Ok((cw.count(), cw.into_inner().finalize().into())) } -/// Accepts a [proto::node::Node] pointing to the root of a (store) path, -/// and uses the passed blob_service and directory_service to -/// perform the necessary lookups as it traverses the structure. +/// Accepts a [castorepb::node::Node] pointing to the root of a (store) path, +/// and uses the passed blob_service and directory_service to perform the +/// necessary lookups as it traverses the structure. /// The contents in NAR serialization are writen to the passed [std::io::Write]. /// /// The writer is passed back in the return value. This is done because async Rust @@ -39,7 +40,7 @@ pub async fn calculate_size_and_sha256( /// This will panic if called outside the context of a Tokio runtime. pub async fn write_nar( mut w: W, - proto_root_node: &proto::node::Node, + proto_root_node: &castorepb::node::Node, blob_service: Arc, directory_service: Arc, ) -> Result { @@ -69,24 +70,24 @@ pub async fn write_nar( fn walk_node( tokio_handle: tokio::runtime::Handle, nar_node: nar::writer::Node, - proto_node: &proto::node::Node, + proto_node: &castorepb::node::Node, blob_service: Arc, directory_service: Arc, ) -> Result<(), RenderError> { match proto_node { - proto::node::Node::Symlink(proto_symlink_node) => { + castorepb::node::Node::Symlink(proto_symlink_node) => { nar_node .symlink(&proto_symlink_node.target) .map_err(RenderError::NARWriterError)?; } - proto::node::Node::File(proto_file_node) => { + castorepb::node::Node::File(proto_file_node) => { let digest = proto_file_node.digest.clone().try_into().map_err(|_e| { warn!( file_node = ?proto_file_node, "invalid digest length in file node", ); - RenderError::StoreError(crate::Error::StorageError( + RenderError::StoreError(Error::StorageError( "invalid digest len in file node".to_string(), )) })?; @@ -110,13 +111,13 @@ fn walk_node( ) .map_err(RenderError::NARWriterError)?; } - proto::node::Node::Directory(proto_directory_node) => { + castorepb::node::Node::Directory(proto_directory_node) => { let digest = proto_directory_node .digest .clone() .try_into() .map_err(|_e| { - RenderError::StoreError(crate::Error::StorageError( + RenderError::StoreError(Error::StorageError( "invalid digest len in directory node".to_string(), )) })?; -- cgit 1.4.1