diff options
Diffstat (limited to 'tvix/store/src/import.rs')
| -rw-r--r-- | tvix/store/src/import.rs | 153 |
1 files changed, 66 insertions, 87 deletions
diff --git a/tvix/store/src/import.rs b/tvix/store/src/import.rs index 69f68d46a2fa..4c1bd51eeb61 100644 --- a/tvix/store/src/import.rs +++ b/tvix/store/src/import.rs @@ -1,7 +1,8 @@ +use bstr::ByteSlice; use std::path::Path; use tracing::{debug, instrument}; use tvix_castore::{ - blobservice::BlobService, directoryservice::DirectoryService, proto::node::Node, B3Digest, + blobservice::BlobService, directoryservice::DirectoryService, import::fs::ingest_path, Node, }; use nix_compat::{ @@ -10,8 +11,9 @@ use nix_compat::{ }; use crate::{ - pathinfoservice::PathInfoService, - proto::{nar_info, NarInfo, PathInfo}, + nar::NarCalculationService, + pathinfoservice::{PathInfo, PathInfoService}, + proto::nar_info, }; impl From<CAHash> for nar_info::Ca { @@ -25,29 +27,29 @@ impl From<CAHash> for nar_info::Ca { } } -pub fn log_node(node: &Node, path: &Path) { +pub fn log_node(name: &[u8], node: &Node, path: &Path) { match node { - Node::Directory(directory_node) => { + Node::Directory { digest, .. } => { debug!( path = ?path, - name = ?directory_node.name, - digest = %B3Digest::try_from(directory_node.digest.clone()).unwrap(), + name = %name.as_bstr(), + digest = %digest, "import successful", ) } - Node::File(file_node) => { + Node::File { digest, .. } => { debug!( path = ?path, - name = ?file_node.name, - digest = %B3Digest::try_from(file_node.digest.clone()).unwrap(), + name = %name.as_bstr(), + digest = %digest, "import successful" ) } - Node::Symlink(symlink_node) => { + Node::Symlink { target } => { debug!( path = ?path, - name = ?symlink_node.name, - target = ?symlink_node.target, + name = %name.as_bstr(), + target = ?target, "import successful" ) } @@ -68,87 +70,63 @@ pub fn path_to_name(path: &Path) -> std::io::Result<&str> { }) } -/// Takes the NAR size, SHA-256 of the NAR representation, the root node and optionally -/// a CA hash information. -/// -/// Returns the path information object for a NAR-style object. -/// -/// This [`PathInfo`] can be further filled for signatures, deriver or verified for the expected -/// hashes. -#[inline] -pub fn derive_nar_ca_path_info( - nar_size: u64, - nar_sha256: [u8; 32], - ca: Option<CAHash>, - root_node: Node, -) -> PathInfo { - // assemble the [crate::proto::PathInfo] object. - PathInfo { - node: Some(tvix_castore::proto::Node { - node: Some(root_node), - }), - // There's no reference scanning on path contents ingested like this. - references: vec![], - narinfo: Some(NarInfo { - nar_size, - nar_sha256: nar_sha256.to_vec().into(), - signatures: vec![], - reference_names: vec![], - deriver: None, - ca: ca.map(|ca_hash| ca_hash.into()), - }), - } -} - -/// Ingest the given path `path` and register the resulting output path in the -/// [`PathInfoService`] as a recursive fixed output NAR. +/// Ingest the contents at the given path `path` into castore, and registers the +/// resulting root node in the passed PathInfoService, using the "NAR sha256 +/// digest" and the passed name for output path calculation. +/// Inserts the PathInfo into the PathInfoService and returns it back to the caller. #[instrument(skip_all, fields(store_name=name, path=?path), err)] -pub async fn import_path_as_nar_ca<BS, DS, PS, P>( +pub async fn import_path_as_nar_ca<BS, DS, PS, NS, P>( path: P, name: &str, blob_service: BS, directory_service: DS, path_info_service: PS, -) -> Result<StorePath, std::io::Error> + nar_calculation_service: NS, +) -> Result<PathInfo, std::io::Error> where P: AsRef<Path> + std::fmt::Debug, - BS: AsRef<dyn BlobService> + Clone, - DS: AsRef<dyn DirectoryService>, + BS: BlobService + Clone, + DS: DirectoryService, PS: AsRef<dyn PathInfoService>, + NS: NarCalculationService, { + // Ingest the contents at the given path `path` into castore. let root_node = - tvix_castore::import::ingest_path(blob_service, directory_service, &path).await?; + ingest_path::<_, _, _, &[u8]>(blob_service, directory_service, path.as_ref(), None) + .await + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + + // Ask for the NAR size and sha256 + let (nar_size, nar_sha256) = nar_calculation_service.calculate_nar(&root_node).await?; - // Ask the PathInfoService for the NAR size and sha256 - let (nar_size, nar_sha256) = path_info_service.as_ref().calculate_nar(&root_node).await?; + let ca = CAHash::Nar(NixHash::Sha256(nar_sha256)); // Calculate the output path. This might still fail, as some names are illegal. // FUTUREWORK: express the `name` at the type level to be valid and move the conversion // at the caller level. - let output_path = store_path::build_nar_based_store_path(&nar_sha256, name).map_err(|_| { - std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("invalid name: {}", name), - ) - })?; - - // assemble a new root_node with a name that is derived from the nar hash. - let root_node = root_node.rename(output_path.to_string().into_bytes().into()); - log_node(&root_node, path.as_ref()); - - let path_info = derive_nar_ca_path_info( - nar_size, - nar_sha256, - Some(CAHash::Nar(NixHash::Sha256(nar_sha256))), - root_node, - ); - - // This new [`PathInfo`] that we get back from there might contain additional signatures or - // information set by the service itself. In this function, we silently swallow it because - // callers doesn't really need it. - let _path_info = path_info_service.as_ref().put(path_info).await?; - - Ok(output_path.to_owned()) + let output_path: StorePath<String> = + store_path::build_ca_path(name, &ca, std::iter::empty::<&str>(), false).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("invalid name: {}", name), + ) + })?; + + // Insert a PathInfo. On success, return it back to the caller. + Ok(path_info_service + .as_ref() + .put(PathInfo { + store_path: output_path.to_owned(), + node: root_node, + // There's no reference scanning on imported paths + references: vec![], + nar_size, + nar_sha256, + signatures: vec![], + deriver: None, + ca: Some(ca), + }) + .await?) } #[cfg(test)] @@ -156,21 +134,22 @@ mod tests { use std::{ffi::OsStr, path::PathBuf}; use crate::import::path_to_name; - use test_case::test_case; + use rstest::rstest; - #[test_case("a/b/c", "c"; "simple path")] - #[test_case("a/b/../c", "c"; "simple path containing ..")] - #[test_case("a/b/../c/d/../e", "e"; "path containing multiple ..")] + #[rstest] + #[case::simple_path("a/b/c", "c")] + #[case::simple_path_containing_dotdot("a/b/../c", "c")] + #[case::path_containing_multiple_dotdot("a/b/../c/d/../e", "e")] - fn test_path_to_name(path: &str, expected_name: &str) { + fn test_path_to_name(#[case] path: &str, #[case] expected_name: &str) { let path: PathBuf = path.into(); assert_eq!(path_to_name(&path).expect("must succeed"), expected_name); } - #[test_case(b"a/b/.."; "path ending in ..")] - #[test_case(b"\xf8\xa1\xa1\xa1\xa1"; "non unicode path")] - - fn test_invalid_path_to_name(invalid_path: &[u8]) { + #[rstest] + #[case::path_ending_in_dotdot(b"a/b/..")] + #[case::non_unicode_path(b"\xf8\xa1\xa1\xa1\xa1")] + fn test_invalid_path_to_name(#[case] invalid_path: &[u8]) { let path: PathBuf = unsafe { OsStr::from_encoded_bytes_unchecked(invalid_path) }.into(); path_to_name(&path).expect_err("must fail"); } |