diff options
author | Florian Klink <flokli@flokli.de> | 2024-08-13T17·04+0300 |
---|---|---|
committer | clbot <clbot@tvl.fyi> | 2024-08-13T18·39+0000 |
commit | c7845f3c882d0f1b215813bf0ef8231c2c03d7b6 (patch) | |
tree | 681774cbfa98e5021aa68b361bc937b88881b5c8 | |
parent | 2f4185ff1a54e1bdbaa062a9e4e1c8137d141762 (diff) |
refactor(tvix/castore): move *Node and Directory to crate root r/8486
*Node and Directory are types of the tvix-castore model, not the tvix DirectoryService model. A DirectoryService only happens to send Directories. Move types into individual files in a nodes/ subdirectory, as it's gotten too cluttered in a single file, and (re-)export all types from the crate root. This has the effect that we now cannot poke at private fields directly from other files inside `crate::directoryservice` (as it's not all in the same file anymore), but that's a good thing, it now forces us to go through the proper accessors. For the same reasons, we currently also need to introduce the `rename` functions on each *Node directly. A followup is gonna move the names out of the individual enum kinds, so we can better represent "unnamed nodes". Change-Id: Icdb34dcfe454c41c94f2396e8e99973d27db8418 Reviewed-on: https://cl.tvl.fyi/c/depot/+/12199 Reviewed-by: yuka <yuka@yuka.dev> Autosubmit: flokli <flokli@flokli.de> Tested-by: BuildkiteCI
42 files changed, 620 insertions, 622 deletions
diff --git a/tvix/build/src/proto/mod.rs b/tvix/build/src/proto/mod.rs index 528817b9707d..2164803b6193 100644 --- a/tvix/build/src/proto/mod.rs +++ b/tvix/build/src/proto/mod.rs @@ -1,9 +1,8 @@ use std::path::{Path, PathBuf}; use itertools::Itertools; -use tvix_castore::directoryservice::NamedNode; -use tvix_castore::directoryservice::Node; use tvix_castore::ValidateNodeError; +use tvix_castore::{NamedNode, Node}; mod grpc_buildservice_wrapper; diff --git a/tvix/castore/src/directoryservice/directory_graph.rs b/tvix/castore/src/directoryservice/directory_graph.rs index aa60f68a3197..0cf2f71adb06 100644 --- a/tvix/castore/src/directoryservice/directory_graph.rs +++ b/tvix/castore/src/directoryservice/directory_graph.rs @@ -10,8 +10,7 @@ use petgraph::{ use tracing::instrument; use super::order_validator::{LeavesToRootValidator, OrderValidator, RootToLeavesValidator}; -use super::{Directory, DirectoryNode}; -use crate::B3Digest; +use crate::{B3Digest, Directory, DirectoryNode, NamedNode}; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -72,11 +71,11 @@ pub struct ValidatedDirectoryGraph { fn check_edge(dir: &DirectoryNode, child: &Directory) -> Result<(), Error> { // Ensure the size specified in the child node matches our records. - if dir.size != child.size() { + if dir.size() != child.size() { return Err(Error::ValidationError(format!( "'{}' has wrong size, specified {}, recorded {}", - dir.name.as_bstr(), - dir.size, + dir.get_name().as_bstr(), + dir.size(), child.size(), ))); } @@ -145,7 +144,7 @@ impl<O: OrderValidator> DirectoryGraph<O> { for subdir in directory.directories() { let child_ix = *self .digest_to_node_ix - .entry(subdir.digest.clone()) + .entry(subdir.digest().clone()) .or_insert_with(|| self.graph.add_node(None)); let pending_edge_check = match &self.graph[child_ix] { @@ -268,8 +267,8 @@ impl ValidatedDirectoryGraph { */ #[cfg(test)] mod tests { - use crate::directoryservice::{Directory, DirectoryNode, Node}; use crate::fixtures::{DIRECTORY_A, DIRECTORY_B, DIRECTORY_C}; + use crate::{Directory, DirectoryNode, Node}; use lazy_static::lazy_static; use rstest::rstest; diff --git a/tvix/castore/src/directoryservice/mod.rs b/tvix/castore/src/directoryservice/mod.rs index 0141a48f75bc..9f23176e31d0 100644 --- a/tvix/castore/src/directoryservice/mod.rs +++ b/tvix/castore/src/directoryservice/mod.rs @@ -1,9 +1,6 @@ use crate::composition::{Registry, ServiceBuilder}; -use crate::proto; -use crate::{B3Digest, Error}; -use crate::{ValidateDirectoryError, ValidateNodeError}; +use crate::{B3Digest, Directory, Error}; -use bytes::Bytes; use futures::stream::BoxStream; use tonic::async_trait; mod combinators; @@ -148,461 +145,3 @@ pub(crate) fn register_directory_services(reg: &mut Registry) { reg.register::<Box<dyn ServiceBuilder<Output = dyn DirectoryService>>, super::directoryservice::BigtableParameters>("bigtable"); } } - -/// A Directory can contain Directory, File or Symlink nodes. -/// Each of these nodes have a name attribute, which is the basename in that -/// directory and node type specific attributes. -/// While a Node by itself may have any name, the names of Directory entries: -/// - MUST not contain slashes or null bytes -/// - MUST not be '.' or '..' -/// - MUST be unique across all three lists -#[derive(Default, Debug, Clone, PartialEq, Eq)] -pub struct Directory { - nodes: Vec<Node>, -} - -/// A DirectoryNode is a pointer to a [Directory], by its [Directory::digest]. -/// It also gives it a `name` and `size`. -/// Such a node is either an element in the [Directory] it itself is contained in, -/// or a standalone root node./ -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DirectoryNode { - /// The (base)name of the directory - name: Bytes, - /// The blake3 hash of a Directory message, serialized in protobuf canonical form. - digest: B3Digest, - /// Number of child elements in the Directory referred to by `digest`. - /// Calculated by summing up the numbers of nodes, and for each directory. - /// its size field. Can be used for inode allocation. - /// This field is precisely as verifiable as any other Merkle tree edge. - /// Resolve `digest`, and you can compute it incrementally. Resolve the entire - /// tree, and you can fully compute it from scratch. - /// A credulous implementation won't reject an excessive size, but this is - /// harmless: you'll have some ordinals without nodes. Undersizing is obvious - /// and easy to reject: you won't have an ordinal for some nodes. - size: u64, -} - -impl DirectoryNode { - pub fn new(name: Bytes, digest: B3Digest, size: u64) -> Result<Self, ValidateNodeError> { - Ok(Self { name, digest, size }) - } - - pub fn digest(&self) -> &B3Digest { - &self.digest - } - pub fn size(&self) -> u64 { - self.size - } -} - -/// A FileNode represents a regular or executable file in a Directory or at the root. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FileNode { - /// The (base)name of the file - name: Bytes, - - /// The blake3 digest of the file contents - digest: B3Digest, - - /// The file content size - size: u64, - - /// Whether the file is executable - executable: bool, -} - -impl FileNode { - pub fn new( - name: Bytes, - digest: B3Digest, - size: u64, - executable: bool, - ) -> Result<Self, ValidateNodeError> { - Ok(Self { - name, - digest, - size, - executable, - }) - } - - pub fn digest(&self) -> &B3Digest { - &self.digest - } - - pub fn size(&self) -> u64 { - self.size - } - - pub fn executable(&self) -> bool { - self.executable - } -} - -/// A SymlinkNode represents a symbolic link in a Directory or at the root. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SymlinkNode { - /// The (base)name of the symlink - name: Bytes, - /// The target of the symlink. - target: Bytes, -} - -impl SymlinkNode { - pub fn new(name: Bytes, target: Bytes) -> Result<Self, ValidateNodeError> { - if target.is_empty() || target.contains(&b'\0') { - return Err(ValidateNodeError::InvalidSymlinkTarget(target)); - } - Ok(Self { name, target }) - } - - pub fn target(&self) -> &bytes::Bytes { - &self.target - } -} - -/// A Node is either a [DirectoryNode], [FileNode] or [SymlinkNode]. -/// While a Node by itself may have any name, only those matching specific requirements -/// can can be added as entries to a [Directory] (see the documentation on [Directory] for details). -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Node { - Directory(DirectoryNode), - File(FileNode), - Symlink(SymlinkNode), -} - -/// NamedNode is implemented for [FileNode], [DirectoryNode] and [SymlinkNode] -/// and [Node], so we can ask all of them for the name easily. -pub trait NamedNode { - fn get_name(&self) -> &bytes::Bytes; -} - -impl NamedNode for &FileNode { - fn get_name(&self) -> &bytes::Bytes { - &self.name - } -} -impl NamedNode for FileNode { - fn get_name(&self) -> &bytes::Bytes { - &self.name - } -} - -impl NamedNode for &DirectoryNode { - fn get_name(&self) -> &bytes::Bytes { - &self.name - } -} -impl NamedNode for DirectoryNode { - fn get_name(&self) -> &bytes::Bytes { - &self.name - } -} - -impl NamedNode for &SymlinkNode { - fn get_name(&self) -> &bytes::Bytes { - &self.name - } -} -impl NamedNode for SymlinkNode { - fn get_name(&self) -> &bytes::Bytes { - &self.name - } -} - -impl NamedNode for &Node { - fn get_name(&self) -> &bytes::Bytes { - match self { - Node::File(node_file) => &node_file.name, - Node::Directory(node_directory) => &node_directory.name, - Node::Symlink(node_symlink) => &node_symlink.name, - } - } -} -impl NamedNode for Node { - fn get_name(&self) -> &bytes::Bytes { - match self { - Node::File(node_file) => &node_file.name, - Node::Directory(node_directory) => &node_directory.name, - Node::Symlink(node_symlink) => &node_symlink.name, - } - } -} - -impl Node { - /// Returns the node with a new name. - pub fn rename(self, name: bytes::Bytes) -> Self { - match self { - Node::Directory(n) => Node::Directory(DirectoryNode { name, ..n }), - Node::File(n) => Node::File(FileNode { name, ..n }), - Node::Symlink(n) => Node::Symlink(SymlinkNode { name, ..n }), - } - } -} - -impl PartialOrd for Node { - fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { - Some(self.cmp(other)) - } -} - -impl Ord for Node { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.get_name().cmp(other.get_name()) - } -} - -impl PartialOrd for FileNode { - fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { - Some(self.cmp(other)) - } -} - -impl Ord for FileNode { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.get_name().cmp(other.get_name()) - } -} - -impl PartialOrd for DirectoryNode { - fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { - Some(self.cmp(other)) - } -} - -impl Ord for DirectoryNode { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.get_name().cmp(other.get_name()) - } -} - -impl PartialOrd for SymlinkNode { - fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { - Some(self.cmp(other)) - } -} - -impl Ord for SymlinkNode { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.get_name().cmp(other.get_name()) - } -} - -fn checked_sum(iter: impl IntoIterator<Item = u64>) -> Option<u64> { - iter.into_iter().try_fold(0u64, |acc, i| acc.checked_add(i)) -} - -impl Directory { - pub fn new() -> Self { - Directory { nodes: vec![] } - } - - /// The size of a directory is the number of all regular and symlink elements, - /// the number of directory elements, and their size fields. - pub fn size(&self) -> u64 { - // It's impossible to create a Directory where the size overflows, because we - // check before every add() that the size won't overflow. - (self.nodes.len() as u64) + self.directories().map(|e| e.size).sum::<u64>() - } - - /// Calculates the digest of a Directory, which is the blake3 hash of a - /// Directory protobuf message, serialized in protobuf canonical form. - pub fn digest(&self) -> B3Digest { - proto::Directory::from(self).digest() - } - - /// Allows iterating over all nodes (directories, files and symlinks) - /// ordered by their name. - pub fn nodes(&self) -> impl Iterator<Item = &Node> + Send + Sync + '_ { - self.nodes.iter() - } - - /// Allows iterating over the FileNode entries of this directory - /// ordered by their name - pub fn files(&self) -> impl Iterator<Item = &FileNode> + Send + Sync + '_ { - self.nodes.iter().filter_map(|node| match node { - Node::File(n) => Some(n), - _ => None, - }) - } - - /// Allows iterating over the subdirectories of this directory - /// ordered by their name - pub fn directories(&self) -> impl Iterator<Item = &DirectoryNode> + Send + Sync + '_ { - self.nodes.iter().filter_map(|node| match node { - Node::Directory(n) => Some(n), - _ => None, - }) - } - - /// Allows iterating over the SymlinkNode entries of this directory - /// ordered by their name - pub fn symlinks(&self) -> impl Iterator<Item = &SymlinkNode> + Send + Sync + '_ { - self.nodes.iter().filter_map(|node| match node { - Node::Symlink(n) => Some(n), - _ => None, - }) - } - - /// Checks a Node name for validity as a directory entry - /// We disallow slashes, null bytes, '.', '..' and the empty string. - pub(crate) fn validate_node_name(name: &[u8]) -> Result<(), ValidateNodeError> { - if name.is_empty() - || name == b".." - || name == b"." - || name.contains(&0x00) - || name.contains(&b'/') - { - Err(ValidateNodeError::InvalidName(name.to_owned().into())) - } else { - Ok(()) - } - } - - /// Adds the specified [Node] to the [Directory], preserving sorted entries. - /// - /// Inserting an element that already exists with the same name in the directory will yield an - /// error. - /// Inserting an element will validate that its name fulfills the stricter requirements for - /// directory entries and yield an error if it is not. - pub fn add(&mut self, node: Node) -> Result<(), ValidateDirectoryError> { - Self::validate_node_name(node.get_name()) - .map_err(|e| ValidateDirectoryError::InvalidNode(node.get_name().clone().into(), e))?; - - // Check that the even after adding this new directory entry, the size calculation will not - // overflow - // FUTUREWORK: add some sort of batch add interface which only does this check once with - // all the to-be-added entries - checked_sum([ - self.size(), - 1, - match node { - Node::Directory(ref dir) => dir.size, - _ => 0, - }, - ]) - .ok_or(ValidateDirectoryError::SizeOverflow)?; - - // This assumes the [Directory] is sorted, since we don't allow accessing the nodes list - // directly and all previous inserts should have been in-order - let pos = match self - .nodes - .binary_search_by_key(&node.get_name(), |n| n.get_name()) - { - Err(pos) => pos, // There is no node with this name; good! - Ok(_) => { - return Err(ValidateDirectoryError::DuplicateName( - node.get_name().to_vec(), - )) - } - }; - - self.nodes.insert(pos, node); - Ok(()) - } -} - -#[cfg(test)] -mod test { - use super::{Directory, DirectoryNode, FileNode, Node, SymlinkNode}; - use crate::fixtures::DUMMY_DIGEST; - use crate::ValidateDirectoryError; - - #[test] - fn add_nodes_to_directory() { - let mut d = Directory::new(); - - d.add(Node::Directory( - DirectoryNode::new("b".into(), DUMMY_DIGEST.clone(), 1).unwrap(), - )) - .unwrap(); - d.add(Node::Directory( - DirectoryNode::new("a".into(), DUMMY_DIGEST.clone(), 1).unwrap(), - )) - .unwrap(); - d.add(Node::Directory( - DirectoryNode::new("z".into(), DUMMY_DIGEST.clone(), 1).unwrap(), - )) - .unwrap(); - - d.add(Node::File( - FileNode::new("f".into(), DUMMY_DIGEST.clone(), 1, true).unwrap(), - )) - .unwrap(); - d.add(Node::File( - FileNode::new("c".into(), DUMMY_DIGEST.clone(), 1, true).unwrap(), - )) - .unwrap(); - d.add(Node::File( - FileNode::new("g".into(), DUMMY_DIGEST.clone(), 1, true).unwrap(), - )) - .unwrap(); - - d.add(Node::Symlink( - SymlinkNode::new("t".into(), "a".into()).unwrap(), - )) - .unwrap(); - d.add(Node::Symlink( - SymlinkNode::new("o".into(), "a".into()).unwrap(), - )) - .unwrap(); - d.add(Node::Symlink( - SymlinkNode::new("e".into(), "a".into()).unwrap(), - )) - .unwrap(); - - // Convert to proto struct and back to ensure we are not generating any invalid structures - crate::directoryservice::Directory::try_from(crate::proto::Directory::from(d)) - .expect("directory should be valid"); - } - - #[test] - fn validate_overflow() { - let mut d = Directory::new(); - - assert_eq!( - d.add(Node::Directory( - DirectoryNode::new("foo".into(), DUMMY_DIGEST.clone(), u64::MAX).unwrap(), - )), - Err(ValidateDirectoryError::SizeOverflow) - ); - } - - #[test] - fn add_duplicate_node_to_directory() { - let mut d = Directory::new(); - - d.add(Node::Directory( - DirectoryNode::new("a".into(), DUMMY_DIGEST.clone(), 1).unwrap(), - )) - .unwrap(); - assert_eq!( - format!( - "{}", - d.add(Node::File( - FileNode::new("a".into(), DUMMY_DIGEST.clone(), 1, true).unwrap(), - )) - .expect_err("adding duplicate dir entry must fail") - ), - "\"a\" is a duplicate name" - ); - } - - /// Attempt to add a directory entry with a name which should be rejected. - #[tokio::test] - async fn directory_reject_invalid_name() { - let mut dir = Directory::new(); - assert!( - dir.add(Node::Symlink( - SymlinkNode::new( - "".into(), // wrong! can not be added to directory - "doesntmatter".into(), - ) - .unwrap() - )) - .is_err(), - "invalid symlink entry be rejected" - ); - } -} diff --git a/tvix/castore/src/directoryservice/order_validator.rs b/tvix/castore/src/directoryservice/order_validator.rs index 17431fc8d3b0..f03bb7ff9a86 100644 --- a/tvix/castore/src/directoryservice/order_validator.rs +++ b/tvix/castore/src/directoryservice/order_validator.rs @@ -50,7 +50,7 @@ impl RootToLeavesValidator { for subdir in directory.directories() { // Allow the children to appear next - self.expected_digests.insert(subdir.digest.clone()); + self.expected_digests.insert(subdir.digest().clone()); } } } @@ -80,10 +80,10 @@ impl OrderValidator for LeavesToRootValidator { let digest = directory.digest(); for subdir in directory.directories() { - if !self.allowed_references.contains(&subdir.digest) { + if !self.allowed_references.contains(subdir.digest()) { warn!( directory.digest = %digest, - subdirectory.digest = %subdir.digest, + subdirectory.digest = %subdir.digest(), "unexpected directory reference" ); return false; diff --git a/tvix/castore/src/directoryservice/tests/mod.rs b/tvix/castore/src/directoryservice/tests/mod.rs index 3923e66dc2c2..7a15f44a686e 100644 --- a/tvix/castore/src/directoryservice/tests/mod.rs +++ b/tvix/castore/src/directoryservice/tests/mod.rs @@ -8,8 +8,8 @@ use rstest_reuse::{self, *}; use super::DirectoryService; use crate::directoryservice; -use crate::directoryservice::{Directory, DirectoryNode, Node}; use crate::fixtures::{DIRECTORY_A, DIRECTORY_B, DIRECTORY_C, DIRECTORY_D}; +use crate::{Directory, DirectoryNode, Node}; mod utils; use self::utils::make_grpc_directory_service_client; diff --git a/tvix/castore/src/directoryservice/traverse.rs b/tvix/castore/src/directoryservice/traverse.rs index 3e6dc73a4d6b..ff667e0f65c0 100644 --- a/tvix/castore/src/directoryservice/traverse.rs +++ b/tvix/castore/src/directoryservice/traverse.rs @@ -1,5 +1,4 @@ -use super::{DirectoryService, NamedNode, Node}; -use crate::{Error, Path}; +use crate::{directoryservice::DirectoryService, Error, NamedNode, Node, Path}; use tracing::{instrument, warn}; /// This descends from a (root) node to the given (sub)path, returning the Node @@ -25,15 +24,15 @@ where // fetch the linked node from the directory_service. let directory = directory_service .as_ref() - .get(&directory_node.digest) + .get(directory_node.digest()) .await? .ok_or_else(|| { // If we didn't get the directory node that's linked, that's a store inconsistency, bail out! - warn!("directory {} does not exist", directory_node.digest); + warn!("directory {} does not exist", directory_node.digest()); Error::StorageError(format!( "directory {} does not exist", - directory_node.digest + directory_node.digest() )) })?; @@ -59,9 +58,8 @@ where mod tests { use crate::{ directoryservice, - directoryservice::{DirectoryNode, Node}, fixtures::{DIRECTORY_COMPLICATED, DIRECTORY_WITH_KEEP}, - PathBuf, + DirectoryNode, Node, PathBuf, }; use super::descend_to; diff --git a/tvix/castore/src/directoryservice/utils.rs b/tvix/castore/src/directoryservice/utils.rs index 352362811a97..f1133663ede7 100644 --- a/tvix/castore/src/directoryservice/utils.rs +++ b/tvix/castore/src/directoryservice/utils.rs @@ -59,7 +59,7 @@ pub fn traverse_directory<'a, DS: DirectoryService + 'static>( // This panics if the digest looks invalid, it's supposed to be checked first. for child_directory_node in current_directory.directories() { // TODO: propagate error - let child_digest: B3Digest = child_directory_node.digest.clone(); + let child_digest: B3Digest = child_directory_node.digest().clone(); if worklist_directory_digests.contains(&child_digest) || sent_directory_digests.contains(&child_digest) diff --git a/tvix/castore/src/fixtures.rs b/tvix/castore/src/fixtures.rs index 0e423d348522..06366f2057ab 100644 --- a/tvix/castore/src/fixtures.rs +++ b/tvix/castore/src/fixtures.rs @@ -1,6 +1,5 @@ use crate::{ - directoryservice::{Directory, DirectoryNode, FileNode, Node, SymlinkNode}, - B3Digest, + B3Digest, {Directory, DirectoryNode, FileNode, Node, SymlinkNode}, }; use lazy_static::lazy_static; diff --git a/tvix/castore/src/fs/fuse/tests.rs b/tvix/castore/src/fs/fuse/tests.rs index 726beb5858c5..3dad7d83aedd 100644 --- a/tvix/castore/src/fs/fuse/tests.rs +++ b/tvix/castore/src/fs/fuse/tests.rs @@ -15,10 +15,8 @@ use super::FuseDaemon; use crate::fs::{TvixStoreFs, XATTR_NAME_BLOB_DIGEST, XATTR_NAME_DIRECTORY_DIGEST}; use crate::{ blobservice::{BlobService, MemoryBlobService}, - directoryservice::{ - DirectoryNode, DirectoryService, FileNode, MemoryDirectoryService, Node, SymlinkNode, - }, - fixtures, + directoryservice::{DirectoryService, MemoryDirectoryService}, + fixtures, {DirectoryNode, FileNode, Node, SymlinkNode}, }; const BLOB_A_NAME: &str = "00000000000000000000000000000000-test"; diff --git a/tvix/castore/src/fs/inodes.rs b/tvix/castore/src/fs/inodes.rs index 379d4ab87318..b04505c9fa98 100644 --- a/tvix/castore/src/fs/inodes.rs +++ b/tvix/castore/src/fs/inodes.rs @@ -4,8 +4,7 @@ use std::time::Duration; use bytes::Bytes; -use crate::directoryservice::{NamedNode, Node}; -use crate::B3Digest; +use crate::{B3Digest, NamedNode, Node}; #[derive(Clone, Debug)] pub enum InodeData { diff --git a/tvix/castore/src/fs/mod.rs b/tvix/castore/src/fs/mod.rs index 4a6ca88d73fd..90c1f371b546 100644 --- a/tvix/castore/src/fs/mod.rs +++ b/tvix/castore/src/fs/mod.rs @@ -17,8 +17,8 @@ use self::{ }; use crate::{ blobservice::{BlobReader, BlobService}, - directoryservice::{DirectoryService, NamedNode, Node}, - B3Digest, + directoryservice::DirectoryService, + {B3Digest, NamedNode, Node}, }; use bstr::ByteVec; use bytes::Bytes; diff --git a/tvix/castore/src/fs/root_nodes.rs b/tvix/castore/src/fs/root_nodes.rs index 6d78b243d064..c237142c8d91 100644 --- a/tvix/castore/src/fs/root_nodes.rs +++ b/tvix/castore/src/fs/root_nodes.rs @@ -1,7 +1,6 @@ use std::collections::BTreeMap; -use crate::{directoryservice::Node, Error}; -use bytes::Bytes; +use crate::{Error, Node}; use futures::stream::BoxStream; use tonic::async_trait; @@ -23,7 +22,7 @@ pub trait RootNodes: Send + Sync { /// the key is the node name. impl<T> RootNodes for T where - T: AsRef<BTreeMap<Bytes, Node>> + Send + Sync, + T: AsRef<BTreeMap<bytes::Bytes, Node>> + Send + Sync, { async fn get_by_basename(&self, name: &[u8]) -> Result<Option<Node>, Error> { Ok(self.as_ref().get(name).cloned()) diff --git a/tvix/castore/src/import/archive.rs b/tvix/castore/src/import/archive.rs index 930f33f68b46..167f799efa0f 100644 --- a/tvix/castore/src/import/archive.rs +++ b/tvix/castore/src/import/archive.rs @@ -11,8 +11,9 @@ use tokio_tar::Archive; use tracing::{instrument, warn, Level}; use crate::blobservice::BlobService; -use crate::directoryservice::{DirectoryService, Node}; +use crate::directoryservice::DirectoryService; use crate::import::{ingest_entries, IngestionEntry, IngestionError}; +use crate::Node; use super::blobs::{self, ConcurrentBlobUploader}; diff --git a/tvix/castore/src/import/fs.rs b/tvix/castore/src/import/fs.rs index f156c8a73527..1332fdfe57b5 100644 --- a/tvix/castore/src/import/fs.rs +++ b/tvix/castore/src/import/fs.rs @@ -15,8 +15,8 @@ use walkdir::DirEntry; use walkdir::WalkDir; use crate::blobservice::BlobService; -use crate::directoryservice::{DirectoryService, Node}; -use crate::B3Digest; +use crate::directoryservice::DirectoryService; +use crate::{B3Digest, Node}; use super::ingest_entries; use super::IngestionEntry; diff --git a/tvix/castore/src/import/mod.rs b/tvix/castore/src/import/mod.rs index 03e2b6c7db41..b40dfd74e9e4 100644 --- a/tvix/castore/src/import/mod.rs +++ b/tvix/castore/src/import/mod.rs @@ -4,15 +4,9 @@ //! Specific implementations, such as ingesting from the filesystem, live in //! child modules. -use crate::directoryservice::Directory; -use crate::directoryservice::DirectoryNode; -use crate::directoryservice::DirectoryPutter; -use crate::directoryservice::DirectoryService; -use crate::directoryservice::FileNode; -use crate::directoryservice::Node; -use crate::directoryservice::SymlinkNode; +use crate::directoryservice::{DirectoryPutter, DirectoryService}; use crate::path::{Path, PathBuf}; -use crate::B3Digest; +use crate::{B3Digest, Directory, DirectoryNode, FileNode, Node, SymlinkNode}; use futures::{Stream, StreamExt}; use tracing::Level; @@ -219,9 +213,9 @@ impl IngestionEntry { mod test { use rstest::rstest; - use crate::directoryservice::{Directory, DirectoryNode, FileNode, Node, SymlinkNode}; use crate::fixtures::{DIRECTORY_COMPLICATED, DIRECTORY_WITH_KEEP, EMPTY_BLOB_DIGEST}; use crate::{directoryservice::MemoryDirectoryService, fixtures::DUMMY_DIGEST}; + use crate::{Directory, DirectoryNode, FileNode, Node, SymlinkNode}; use super::ingest_entries; use super::IngestionEntry; diff --git a/tvix/castore/src/lib.rs b/tvix/castore/src/lib.rs index 91302e90996c..e09926afe70d 100644 --- a/tvix/castore/src/lib.rs +++ b/tvix/castore/src/lib.rs @@ -10,6 +10,9 @@ pub mod fixtures; #[cfg(feature = "fs")] pub mod fs; +mod nodes; +pub use nodes::*; + mod path; pub use path::{Path, PathBuf}; diff --git a/tvix/castore/src/nodes/directory.rs b/tvix/castore/src/nodes/directory.rs new file mode 100644 index 000000000000..1752681c89c3 --- /dev/null +++ b/tvix/castore/src/nodes/directory.rs @@ -0,0 +1,234 @@ +use crate::{ + proto, B3Digest, DirectoryNode, FileNode, NamedNode, Node, SymlinkNode, ValidateDirectoryError, + ValidateNodeError, +}; + +/// A Directory can contain Directory, File or Symlink nodes. +/// Each of these nodes have a name attribute, which is the basename in that +/// directory and node type specific attributes. +/// While a Node by itself may have any name, the names of Directory entries: +/// - MUST not contain slashes or null bytes +/// - MUST not be '.' or '..' +/// - MUST be unique across all three lists +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct Directory { + nodes: Vec<Node>, +} + +impl Directory { + pub fn new() -> Self { + Directory { nodes: vec![] } + } + + /// The size of a directory is the number of all regular and symlink elements, + /// the number of directory elements, and their size fields. + pub fn size(&self) -> u64 { + // It's impossible to create a Directory where the size overflows, because we + // check before every add() that the size won't overflow. + (self.nodes.len() as u64) + self.directories().map(|e| e.size()).sum::<u64>() + } + + /// Calculates the digest of a Directory, which is the blake3 hash of a + /// Directory protobuf message, serialized in protobuf canonical form. + pub fn digest(&self) -> B3Digest { + proto::Directory::from(self.clone()).digest() + } + + /// Allows iterating over all nodes (directories, files and symlinks) + /// ordered by their name. + pub fn nodes(&self) -> impl Iterator<Item = &Node> + Send + Sync + '_ { + self.nodes.iter() + } + + /// Allows iterating over the FileNode entries of this directory + /// ordered by their name + pub fn files(&self) -> impl Iterator<Item = &FileNode> + Send + Sync + '_ { + self.nodes.iter().filter_map(|node| match node { + Node::File(n) => Some(n), + _ => None, + }) + } + + /// Allows iterating over the subdirectories of this directory + /// ordered by their name + pub fn directories(&self) -> impl Iterator<Item = &DirectoryNode> + Send + Sync + '_ { + self.nodes.iter().filter_map(|node| match node { + Node::Directory(n) => Some(n), + _ => None, + }) + } + + /// Allows iterating over the SymlinkNode entries of this directory + /// ordered by their name + pub fn symlinks(&self) -> impl Iterator<Item = &SymlinkNode> + Send + Sync + '_ { + self.nodes.iter().filter_map(|node| match node { + Node::Symlink(n) => Some(n), + _ => None, + }) + } + + /// Checks a Node name for validity as a directory entry + /// We disallow slashes, null bytes, '.', '..' and the empty string. + pub(crate) fn validate_node_name(name: &[u8]) -> Result<(), ValidateNodeError> { + if name.is_empty() + || name == b".." + || name == b"." + || name.contains(&0x00) + || name.contains(&b'/') + { + Err(ValidateNodeError::InvalidName(name.to_owned().into())) + } else { + Ok(()) + } + } + + /// Adds the specified [Node] to the [Directory], preserving sorted entries. + /// + /// Inserting an element that already exists with the same name in the directory will yield an + /// error. + /// Inserting an element will validate that its name fulfills the stricter requirements for + /// directory entries and yield an error if it is not. + pub fn add(&mut self, node: Node) -> Result<(), ValidateDirectoryError> { + Self::validate_node_name(node.get_name()) + .map_err(|e| ValidateDirectoryError::InvalidNode(node.get_name().clone().into(), e))?; + + // Check that the even after adding this new directory entry, the size calculation will not + // overflow + // FUTUREWORK: add some sort of batch add interface which only does this check once with + // all the to-be-added entries + checked_sum([ + self.size(), + 1, + match node { + Node::Directory(ref dir) => dir.size(), + _ => 0, + }, + ]) + .ok_or(ValidateDirectoryError::SizeOverflow)?; + + // This assumes the [Directory] is sorted, since we don't allow accessing the nodes list + // directly and all previous inserts should have been in-order + let pos = match self + .nodes + .binary_search_by_key(&node.get_name(), |n| n.get_name()) + { + Err(pos) => pos, // There is no node with this name; good! + Ok(_) => { + return Err(ValidateDirectoryError::DuplicateName( + node.get_name().to_vec(), + )) + } + }; + + self.nodes.insert(pos, node); + Ok(()) + } +} + +fn checked_sum(iter: impl IntoIterator<Item = u64>) -> Option<u64> { + iter.into_iter().try_fold(0u64, |acc, i| acc.checked_add(i)) +} + +#[cfg(test)] +mod test { + use super::{Directory, DirectoryNode, FileNode, Node, SymlinkNode}; + use crate::fixtures::DUMMY_DIGEST; + use crate::ValidateDirectoryError; + + #[test] + fn add_nodes_to_directory() { + let mut d = Directory::new(); + + d.add(Node::Directory( + DirectoryNode::new("b".into(), DUMMY_DIGEST.clone(), 1).unwrap(), + )) + .unwrap(); + d.add(Node::Directory( + DirectoryNode::new("a".into(), DUMMY_DIGEST.clone(), 1).unwrap(), + )) + .unwrap(); + d.add(Node::Directory( + DirectoryNode::new("z".into(), DUMMY_DIGEST.clone(), 1).unwrap(), + )) + .unwrap(); + + d.add(Node::File( + FileNode::new("f".into(), DUMMY_DIGEST.clone(), 1, true).unwrap(), + )) + .unwrap(); + d.add(Node::File( + FileNode::new("c".into(), DUMMY_DIGEST.clone(), 1, true).unwrap(), + )) + .unwrap(); + d.add(Node::File( + FileNode::new("g".into(), DUMMY_DIGEST.clone(), 1, true).unwrap(), + )) + .unwrap(); + + d.add(Node::Symlink( + SymlinkNode::new("t".into(), "a".into()).unwrap(), + )) + .unwrap(); + d.add(Node::Symlink( + SymlinkNode::new("o".into(), "a".into()).unwrap(), + )) + .unwrap(); + d.add(Node::Symlink( + SymlinkNode::new("e".into(), "a".into()).unwrap(), + )) + .unwrap(); + + // Convert to proto struct and back to ensure we are not generating any invalid structures + crate::Directory::try_from(crate::proto::Directory::from(d)) + .expect("directory should be valid"); + } + + #[test] + fn validate_overflow() { + let mut d = Directory::new(); + + assert_eq!( + d.add(Node::Directory( + DirectoryNode::new("foo".into(), DUMMY_DIGEST.clone(), u64::MAX).unwrap(), + )), + Err(ValidateDirectoryError::SizeOverflow) + ); + } + + #[test] + fn add_duplicate_node_to_directory() { + let mut d = Directory::new(); + + d.add(Node::Directory( + DirectoryNode::new("a".into(), DUMMY_DIGEST.clone(), 1).unwrap(), + )) + .unwrap(); + assert_eq!( + format!( + "{}", + d.add(Node::File( + FileNode::new("a".into(), DUMMY_DIGEST.clone(), 1, true).unwrap(), + )) + .expect_err("adding duplicate dir entry must fail") + ), + "\"a\" is a duplicate name" + ); + } + + /// Attempt to add a directory entry with a name which should be rejected. + #[tokio::test] + async fn directory_reject_invalid_name() { + let mut dir = Directory::new(); + assert!( + dir.add(Node::Symlink( + SymlinkNode::new( + "".into(), // wrong! can not be added to directory + "doesntmatter".into(), + ) + .unwrap() + )) + .is_err(), + "invalid symlink entry be rejected" + ); + } +} diff --git a/tvix/castore/src/nodes/directory_node.rs b/tvix/castore/src/nodes/directory_node.rs new file mode 100644 index 000000000000..afbae1695c63 --- /dev/null +++ b/tvix/castore/src/nodes/directory_node.rs @@ -0,0 +1,64 @@ +use crate::{B3Digest, NamedNode, ValidateNodeError}; + +/// A DirectoryNode is a pointer to a [Directory], by its [Directory::digest]. +/// It also gives it a `name` and `size`. +/// Such a node is either an element in the [Directory] it itself is contained in, +/// or a standalone root node./ +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DirectoryNode { + /// The (base)name of the directory + name: bytes::Bytes, + /// The blake3 hash of a Directory message, serialized in protobuf canonical form. + digest: B3Digest, + /// Number of child elements in the Directory referred to by `digest`. + /// Calculated by summing up the numbers of nodes, and for each directory. + /// its size field. Can be used for inode allocation. + /// This field is precisely as verifiable as any other Merkle tree edge. + /// Resolve `digest`, and you can compute it incrementally. Resolve the entire + /// tree, and you can fully compute it from scratch. + /// A credulous implementation won't reject an excessive size, but this is + /// harmless: you'll have some ordinals without nodes. Undersizing is obvious + /// and easy to reject: you won't have an ordinal for some nodes. + size: u64, +} + +impl DirectoryNode { + pub fn new(name: bytes::Bytes, digest: B3Digest, size: u64) -> Result<Self, ValidateNodeError> { + Ok(Self { name, digest, size }) + } + + pub fn digest(&self) -> &B3Digest { + &self.digest + } + + pub fn size(&self) -> u64 { + self.size + } + + pub fn rename(self, name: bytes::Bytes) -> Self { + Self { name, ..self } + } +} + +impl PartialOrd for DirectoryNode { + fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { + Some(self.cmp(other)) + } +} + +impl Ord for DirectoryNode { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.get_name().cmp(other.get_name()) + } +} + +impl NamedNode for &DirectoryNode { + fn get_name(&self) -> &bytes::Bytes { + &self.name + } +} +impl NamedNode for DirectoryNode { + fn get_name(&self) -> &bytes::Bytes { + &self.name + } +} diff --git a/tvix/castore/src/nodes/file_node.rs b/tvix/castore/src/nodes/file_node.rs new file mode 100644 index 000000000000..5dd677c17b49 --- /dev/null +++ b/tvix/castore/src/nodes/file_node.rs @@ -0,0 +1,72 @@ +use crate::{B3Digest, NamedNode, ValidateNodeError}; + +/// A FileNode represents a regular or executable file in a Directory or at the root. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileNode { + /// The (base)name of the file + name: bytes::Bytes, + + /// The blake3 digest of the file contents + digest: B3Digest, + + /// The file content size + size: u64, + + /// Whether the file is executable + executable: bool, +} + +impl FileNode { + pub fn new( + name: bytes::Bytes, + digest: B3Digest, + size: u64, + executable: bool, + ) -> Result<Self, ValidateNodeError> { + Ok(Self { + name, + digest, + size, + executable, + }) + } + + pub fn digest(&self) -> &B3Digest { + &self.digest + } + + pub fn size(&self) -> u64 { + self.size + } + + pub fn executable(&self) -> bool { + self.executable + } + + pub fn rename(self, name: bytes::Bytes) -> Self { + Self { name, ..self } + } +} + +impl PartialOrd for FileNode { + fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { + Some(self.cmp(other)) + } +} + +impl Ord for FileNode { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.get_name().cmp(other.get_name()) + } +} + +impl NamedNode for &FileNode { + fn get_name(&self) -> &bytes::Bytes { + &self.name + } +} +impl NamedNode for FileNode { + fn get_name(&self) -> &bytes::Bytes { + &self.name + } +} diff --git a/tvix/castore/src/nodes/mod.rs b/tvix/castore/src/nodes/mod.rs new file mode 100644 index 000000000000..bb8e14cf856e --- /dev/null +++ b/tvix/castore/src/nodes/mod.rs @@ -0,0 +1,69 @@ +//! This holds types describing nodes in the tvix-castore model. +mod directory; +mod directory_node; +mod file_node; +mod symlink_node; + +use bytes::Bytes; +pub use directory::Directory; +pub use directory_node::DirectoryNode; +pub use file_node::FileNode; +pub use symlink_node::SymlinkNode; + +/// A Node is either a [DirectoryNode], [FileNode] or [SymlinkNode]. +/// While a Node by itself may have any name, only those matching specific requirements +/// can can be added as entries to a [Directory] (see the documentation on [Directory] for details). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Node { + Directory(DirectoryNode), + File(FileNode), + Symlink(SymlinkNode), +} + +impl Node { + /// Returns the node with a new name. + pub fn rename(self, name: Bytes) -> Self { + match self { + Node::Directory(n) => Node::Directory(n.rename(name)), + Node::File(n) => Node::File(n.rename(name)), + Node::Symlink(n) => Node::Symlink(n.rename(name)), + } + } +} + +impl PartialOrd for Node { + fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { + Some(self.cmp(other)) + } +} + +impl Ord for Node { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.get_name().cmp(other.get_name()) + } +} + +/// NamedNode is implemented for [FileNode], [DirectoryNode] and [SymlinkNode] +/// and [Node], so we can ask all of them for the name easily. +pub trait NamedNode { + fn get_name(&self) -> &Bytes; +} + +impl NamedNode for &Node { + fn get_name(&self) -> &Bytes { + match self { + Node::File(node_file) => node_file.get_name(), + Node::Directory(node_directory) => node_directory.get_name(), + Node::Symlink(node_symlink) => node_symlink.get_name(), + } + } +} +impl NamedNode for Node { + fn get_name(&self) -> &Bytes { + match self { + Node::File(node_file) => node_file.get_name(), + Node::Directory(node_directory) => node_directory.get_name(), + Node::Symlink(node_symlink) => node_symlink.get_name(), + } + } +} diff --git a/tvix/castore/src/nodes/symlink_node.rs b/tvix/castore/src/nodes/symlink_node.rs new file mode 100644 index 000000000000..2fac27231a85 --- /dev/null +++ b/tvix/castore/src/nodes/symlink_node.rs @@ -0,0 +1,50 @@ +use crate::{NamedNode, ValidateNodeError}; + +/// A SymlinkNode represents a symbolic link in a Directory or at the root. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SymlinkNode { + /// The (base)name of the symlink + name: bytes::Bytes, + /// The target of the symlink. + target: bytes::Bytes, +} + +impl SymlinkNode { + pub fn new(name: bytes::Bytes, target: bytes::Bytes) -> Result<Self, ValidateNodeError> { + if target.is_empty() || target.contains(&b'\0') { + return Err(ValidateNodeError::InvalidSymlinkTarget(target)); + } + Ok(Self { name, target }) + } + + pub fn target(&self) -> &bytes::Bytes { + &self.target + } + + pub(crate) fn rename(self, name: bytes::Bytes) -> Self { + Self { name, ..self } + } +} + +impl PartialOrd for SymlinkNode { + fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { + Some(self.cmp(other)) + } +} + +impl Ord for SymlinkNode { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.get_name().cmp(other.get_name()) + } +} + +impl NamedNode for &SymlinkNode { + fn get_name(&self) -> &bytes::Bytes { + &self.name + } +} +impl NamedNode for SymlinkNode { + fn get_name(&self) -> &bytes::Bytes { + &self.name + } +} diff --git a/tvix/castore/src/path.rs b/tvix/castore/src/path.rs index 77cabeccdb8c..fb340478c08c 100644 --- a/tvix/castore/src/path.rs +++ b/tvix/castore/src/path.rs @@ -10,7 +10,7 @@ use std::{ use bstr::ByteSlice; -use crate::directoryservice::Directory; +use crate::Directory; /// Represents a Path in the castore model. /// These are always relative, and platform-independent, which distinguishes diff --git a/tvix/castore/src/proto/mod.rs b/tvix/castore/src/proto/mod.rs index 7e98cd74c591..8f8f3c08347a 100644 --- a/tvix/castore/src/proto/mod.rs +++ b/tvix/castore/src/proto/mod.rs @@ -8,7 +8,7 @@ mod grpc_directoryservice_wrapper; pub use grpc_blobservice_wrapper::GRPCBlobServiceWrapper; pub use grpc_directoryservice_wrapper::GRPCDirectoryServiceWrapper; -use crate::directoryservice::NamedNode; +use crate::NamedNode; use crate::{B3Digest, ValidateDirectoryError, ValidateNodeError}; tonic::include_proto!("tvix.castore.v1"); @@ -82,22 +82,22 @@ fn update_if_lt_prev<'n>( Ok(()) } -impl TryFrom<&node::Node> for crate::directoryservice::Node { +impl TryFrom<&node::Node> for crate::Node { type Error = ValidateNodeError; - fn try_from(node: &node::Node) -> Result<crate::directoryservice::Node, ValidateNodeError> { + fn try_from(node: &node::Node) -> Result<crate::Node, ValidateNodeError> { Ok(match node { - node::Node::Directory(n) => crate::directoryservice::Node::Directory(n.try_into()?), - node::Node::File(n) => crate::directoryservice::Node::File(n.try_into()?), - node::Node::Symlink(n) => crate::directoryservice::Node::Symlink(n.try_into()?), + node::Node::Directory(n) => crate::Node::Directory(n.try_into()?), + node::Node::File(n) => crate::Node::File(n.try_into()?), + node::Node::Symlink(n) => crate::Node::Symlink(n.try_into()?), }) } } -impl TryFrom<&Node> for crate::directoryservice::Node { +impl TryFrom<&Node> for crate::Node { type Error = ValidateNodeError; - fn try_from(node: &Node) -> Result<crate::directoryservice::Node, ValidateNodeError> { + fn try_from(node: &Node) -> Result<crate::Node, ValidateNodeError> { match node { Node { node: None } => Err(ValidateNodeError::NoNodeSet), Node { node: Some(node) } => node.try_into(), @@ -105,13 +105,11 @@ impl TryFrom<&Node> for crate::directoryservice::Node { } } -impl TryFrom<&DirectoryNode> for crate::directoryservice::DirectoryNode { +impl TryFrom<&DirectoryNode> for crate::DirectoryNode { type Error = ValidateNodeError; - fn try_from( - node: &DirectoryNode, - ) -> Result<crate::directoryservice::DirectoryNode, ValidateNodeError> { - crate::directoryservice::DirectoryNode::new( + fn try_from(node: &DirectoryNode) -> Result<crate::DirectoryNode, ValidateNodeError> { + crate::DirectoryNode::new( node.name.clone(), node.digest.clone().try_into()?, node.size, @@ -119,21 +117,19 @@ impl TryFrom<&DirectoryNode> for crate::directoryservice::DirectoryNode { } } -impl TryFrom<&SymlinkNode> for crate::directoryservice::SymlinkNode { +impl TryFrom<&SymlinkNode> for crate::SymlinkNode { type Error = ValidateNodeError; - fn try_from( - node: &SymlinkNode, - ) -> Result<crate::directoryservice::SymlinkNode, ValidateNodeError> { - crate::directoryservice::SymlinkNode::new(node.name.clone(), node.target.clone()) + fn try_from(node: &SymlinkNode) -> Result<crate::SymlinkNode, ValidateNodeError> { + crate::SymlinkNode::new(node.name.clone(), node.target.clone()) } } -impl TryFrom<&FileNode> for crate::directoryservice::FileNode { +impl TryFrom<&FileNode> for crate::FileNode { type Error = ValidateNodeError; - fn try_from(node: &FileNode) -> Result<crate::directoryservice::FileNode, ValidateNodeError> { - crate::directoryservice::FileNode::new( + fn try_from(node: &FileNode) -> Result<crate::FileNode, ValidateNodeError> { + crate::FileNode::new( node.name.clone(), node.digest.clone().try_into()?, node.size, @@ -142,80 +138,70 @@ impl TryFrom<&FileNode> for crate::directoryservice::FileNode { } } -impl TryFrom<Directory> for crate::directoryservice::Directory { +impl TryFrom<Directory> for crate::Directory { type Error = ValidateDirectoryError; - fn try_from( - directory: Directory, - ) -> Result<crate::directoryservice::Directory, ValidateDirectoryError> { + fn try_from(directory: Directory) -> Result<crate::Directory, ValidateDirectoryError> { (&directory).try_into() } } -impl TryFrom<&Directory> for crate::directoryservice::Directory { +impl TryFrom<&Directory> for crate::Directory { type Error = ValidateDirectoryError; - fn try_from( - directory: &Directory, - ) -> Result<crate::directoryservice::Directory, ValidateDirectoryError> { - let mut dir = crate::directoryservice::Directory::new(); + fn try_from(directory: &Directory) -> Result<crate::Directory, ValidateDirectoryError> { + let mut dir = crate::Directory::new(); let mut last_file_name: &[u8] = b""; for file in directory.files.iter().map(move |file| { update_if_lt_prev(&mut last_file_name, &file.name).map(|()| file.clone()) }) { let file = file?; - dir.add(crate::directoryservice::Node::File( - (&file) - .try_into() - .map_err(|e| ValidateDirectoryError::InvalidNode(file.name.into(), e))?, - ))?; + dir.add(crate::Node::File((&file).try_into().map_err(|e| { + ValidateDirectoryError::InvalidNode(file.name.into(), e) + })?))?; } let mut last_directory_name: &[u8] = b""; for directory in directory.directories.iter().map(move |directory| { update_if_lt_prev(&mut last_directory_name, &directory.name).map(|()| directory.clone()) }) { let directory = directory?; - dir.add(crate::directoryservice::Node::Directory( - (&directory) - .try_into() - .map_err(|e| ValidateDirectoryError::InvalidNode(directory.name.into(), e))?, - ))?; + dir.add(crate::Node::Directory((&directory).try_into().map_err( + |e| ValidateDirectoryError::InvalidNode(directory.name.into(), e), + )?))?; } let mut last_symlink_name: &[u8] = b""; for symlink in directory.symlinks.iter().map(move |symlink| { update_if_lt_prev(&mut last_symlink_name, &symlink.name).map(|()| symlink.clone()) }) { let symlink = symlink?; - dir.add(crate::directoryservice::Node::Symlink( - (&symlink) - .try_into() - .map_err(|e| ValidateDirectoryError::InvalidNode(symlink.name.into(), e))?, - ))?; + dir.add(crate::Node::Symlink((&symlink).try_into().map_err( + |e| ValidateDirectoryError::InvalidNode(symlink.name.into(), e), + )?))?; } Ok(dir) } } -impl From<&crate::directoryservice::Node> for node::Node { - fn from(node: &crate::directoryservice::Node) -> node::Node { +impl From<&crate::Node> for node::Node { + fn from(node: &crate::Node) -> node::Node { match node { - crate::directoryservice::Node::Directory(n) => node::Node::Directory(n.into()), - crate::directoryservice::Node::File(n) => node::Node::File(n.into()), - crate::directoryservice::Node::Symlink(n) => node::Node::Symlink(n.into()), + crate::Node::Directory(n) => node::Node::Directory(n.into()), + crate::Node::File(n) => node::Node::File(n.into()), + crate::Node::Symlink(n) => node::Node::Symlink(n.into()), } } } -impl From<&crate::directoryservice::Node> for Node { - fn from(node: &crate::directoryservice::Node) -> Node { +impl From<&crate::Node> for Node { + fn from(node: &crate::Node) -> Node { Node { node: Some(node.into()), } } } -impl From<&crate::directoryservice::DirectoryNode> for DirectoryNode { - fn from(node: &crate::directoryservice::DirectoryNode) -> DirectoryNode { +impl From<&crate::DirectoryNode> for DirectoryNode { + fn from(node: &crate::DirectoryNode) -> DirectoryNode { DirectoryNode { digest: node.digest().clone().into(), size: node.size(), @@ -224,8 +210,8 @@ impl From<&crate::directoryservice::DirectoryNode> for DirectoryNode { } } -impl From<&crate::directoryservice::FileNode> for FileNode { - fn from(node: &crate::directoryservice::FileNode) -> FileNode { +impl From<&crate::FileNode> for FileNode { + fn from(node: &crate::FileNode) -> FileNode { FileNode { digest: node.digest().clone().into(), size: node.size(), @@ -235,8 +221,8 @@ impl From<&crate::directoryservice::FileNode> for FileNode { } } -impl From<&crate::directoryservice::SymlinkNode> for SymlinkNode { - fn from(node: &crate::directoryservice::SymlinkNode) -> SymlinkNode { +impl From<&crate::SymlinkNode> for SymlinkNode { + fn from(node: &crate::SymlinkNode) -> SymlinkNode { SymlinkNode { name: node.get_name().clone(), target: node.target().clone(), @@ -244,26 +230,26 @@ impl From<&crate::directoryservice::SymlinkNode> for SymlinkNode { } } -impl From<crate::directoryservice::Directory> for Directory { - fn from(directory: crate::directoryservice::Directory) -> Directory { +impl From<crate::Directory> for Directory { + fn from(directory: crate::Directory) -> Directory { (&directory).into() } } -impl From<&crate::directoryservice::Directory> for Directory { - fn from(directory: &crate::directoryservice::Directory) -> Directory { +impl From<&crate::Directory> for Directory { + fn from(directory: &crate::Directory) -> Directory { let mut directories = vec![]; let mut files = vec![]; let mut symlinks = vec![]; for node in directory.nodes() { match node { - crate::directoryservice::Node::File(n) => { + crate::Node::File(n) => { files.push(n.into()); } - crate::directoryservice::Node::Directory(n) => { + crate::Node::Directory(n) => { directories.push(n.into()); } - crate::directoryservice::Node::Symlink(n) => { + crate::Node::Symlink(n) => { symlinks.push(n.into()); } } diff --git a/tvix/castore/src/proto/tests/directory.rs b/tvix/castore/src/proto/tests/directory.rs index 78b2cf7668e3..7847b9c4c9cb 100644 --- a/tvix/castore/src/proto/tests/directory.rs +++ b/tvix/castore/src/proto/tests/directory.rs @@ -147,7 +147,7 @@ fn digest() { #[test] fn validate_empty() { let d = Directory::default(); - assert!(crate::directoryservice::Directory::try_from(d).is_ok()); + assert!(crate::Directory::try_from(d).is_ok()); } #[test] @@ -161,7 +161,7 @@ fn validate_invalid_names() { }], ..Default::default() }; - match crate::directoryservice::Directory::try_from(d).expect_err("must fail") { + match crate::Directory::try_from(d).expect_err("must fail") { ValidateDirectoryError::InvalidNode(n, ValidateNodeError::InvalidName(_)) => { assert_eq!(n, b"") } @@ -178,7 +178,7 @@ fn validate_invalid_names() { }], ..Default::default() }; - match crate::directoryservice::Directory::try_from(d).expect_err("must fail") { + match crate::Directory::try_from(d).expect_err("must fail") { ValidateDirectoryError::InvalidNode(n, ValidateNodeError::InvalidName(_)) => { assert_eq!(n, b".") } @@ -196,7 +196,7 @@ fn validate_invalid_names() { }], ..Default::default() }; - match crate::directoryservice::Directory::try_from(d).expect_err("must fail") { + match crate::Directory::try_from(d).expect_err("must fail") { ValidateDirectoryError::InvalidNode(n, ValidateNodeError::InvalidName(_)) => { assert_eq!(n, b"..") } @@ -212,7 +212,7 @@ fn validate_invalid_names() { }], ..Default::default() }; - match crate::directoryservice::Directory::try_from(d).expect_err("must fail") { + match crate::Directory::try_from(d).expect_err("must fail") { ValidateDirectoryError::InvalidNode(n, ValidateNodeError::InvalidName(_)) => { assert_eq!(n, b"\x00") } @@ -228,7 +228,7 @@ fn validate_invalid_names() { }], ..Default::default() }; - match crate::directoryservice::Directory::try_from(d).expect_err("must fail") { + match crate::Directory::try_from(d).expect_err("must fail") { ValidateDirectoryError::InvalidNode(n, ValidateNodeError::InvalidName(_)) => { assert_eq!(n, b"foo/bar") } @@ -247,7 +247,7 @@ fn validate_invalid_digest() { }], ..Default::default() }; - match crate::directoryservice::Directory::try_from(d).expect_err("must fail") { + match crate::Directory::try_from(d).expect_err("must fail") { ValidateDirectoryError::InvalidNode(_, ValidateNodeError::InvalidDigestLen(n)) => { assert_eq!(n, 2) } @@ -274,7 +274,7 @@ fn validate_sorting() { ], ..Default::default() }; - match crate::directoryservice::Directory::try_from(d).expect_err("must fail") { + match crate::Directory::try_from(d).expect_err("must fail") { ValidateDirectoryError::WrongSorting(s) => { assert_eq!(s, b"a"); } @@ -299,7 +299,7 @@ fn validate_sorting() { ], ..Default::default() }; - match crate::directoryservice::Directory::try_from(d).expect_err("must fail") { + match crate::Directory::try_from(d).expect_err("must fail") { ValidateDirectoryError::DuplicateName(s) => { assert_eq!(s, b"a"); } @@ -325,7 +325,7 @@ fn validate_sorting() { ..Default::default() }; - crate::directoryservice::Directory::try_from(d).expect("validate shouldn't error"); + crate::Directory::try_from(d).expect("validate shouldn't error"); } // [b, c] and [a] are both properly sorted. @@ -350,6 +350,6 @@ fn validate_sorting() { ..Default::default() }; - crate::directoryservice::Directory::try_from(d).expect("validate shouldn't error"); + crate::Directory::try_from(d).expect("validate shouldn't error"); } } diff --git a/tvix/castore/src/tests/import.rs b/tvix/castore/src/tests/import.rs index ba54078653e2..5dff61782cf2 100644 --- a/tvix/castore/src/tests/import.rs +++ b/tvix/castore/src/tests/import.rs @@ -1,9 +1,9 @@ use crate::blobservice::{self, BlobService}; use crate::directoryservice; -use crate::directoryservice::{DirectoryNode, Node, SymlinkNode}; use crate::fixtures::*; use crate::import::fs::ingest_path; use crate::proto; +use crate::{DirectoryNode, Node, SymlinkNode}; use tempfile::TempDir; diff --git a/tvix/glue/src/builtins/derivation.rs b/tvix/glue/src/builtins/derivation.rs index 9074d418422e..27ae2e1d6925 100644 --- a/tvix/glue/src/builtins/derivation.rs +++ b/tvix/glue/src/builtins/derivation.rs @@ -179,7 +179,7 @@ pub(crate) mod derivation_builtins { use nix_compat::nixhash::CAHash; use nix_compat::store_path::{build_ca_path, hash_placeholder}; use sha2::Sha256; - use tvix_castore::directoryservice::{FileNode, Node}; + use tvix_castore::{FileNode, Node}; use tvix_eval::generators::Gen; use tvix_eval::{NixContext, NixContextElement, NixString}; use tvix_store::proto::{NarInfo, PathInfo}; diff --git a/tvix/glue/src/builtins/import.rs b/tvix/glue/src/builtins/import.rs index aa0d0d54fc8a..1cf4145427b8 100644 --- a/tvix/glue/src/builtins/import.rs +++ b/tvix/glue/src/builtins/import.rs @@ -2,9 +2,8 @@ use crate::builtins::errors::ImportError; use std::path::Path; -use tvix_castore::directoryservice::FileNode; -use tvix_castore::directoryservice::Node; use tvix_castore::import::ingest_entries; +use tvix_castore::{FileNode, Node}; use tvix_eval::{ builtin_macros::builtins, generators::{self, GenCo}, diff --git a/tvix/glue/src/fetchers/mod.rs b/tvix/glue/src/fetchers/mod.rs index 9cab1adc04a3..88d6da9f1e3c 100644 --- a/tvix/glue/src/fetchers/mod.rs +++ b/tvix/glue/src/fetchers/mod.rs @@ -11,9 +11,7 @@ use tokio_util::io::{InspectReader, InspectWriter}; use tracing::{instrument, warn, Span}; use tracing_indicatif::span_ext::IndicatifSpanExt; use tvix_castore::{ - blobservice::BlobService, - directoryservice::{DirectoryService, FileNode, Node}, - ValidateNodeError, + blobservice::BlobService, directoryservice::DirectoryService, FileNode, Node, ValidateNodeError, }; use tvix_store::{nar::NarCalculationService, pathinfoservice::PathInfoService, proto::PathInfo}; use url::Url; diff --git a/tvix/glue/src/tvix_build.rs b/tvix/glue/src/tvix_build.rs index 0f930bfe5099..7fff0b9cf337 100644 --- a/tvix/glue/src/tvix_build.rs +++ b/tvix/glue/src/tvix_build.rs @@ -10,7 +10,7 @@ use tvix_build::proto::{ build_request::{AdditionalFile, BuildConstraints, EnvVar}, BuildRequest, }; -use tvix_castore::directoryservice::Node; +use tvix_castore::Node; /// These are the environment variables that Nix sets in its sandbox for every /// build. @@ -197,8 +197,8 @@ mod test { build_request::{AdditionalFile, BuildConstraints, EnvVar}, BuildRequest, }; - use tvix_castore::directoryservice::{DirectoryNode, Node}; use tvix_castore::fixtures::DUMMY_DIGEST; + use tvix_castore::{DirectoryNode, Node}; use crate::tvix_build::NIX_ENVIRONMENT_VARS; diff --git a/tvix/glue/src/tvix_store_io.rs b/tvix/glue/src/tvix_store_io.rs index c92a9c57e0c7..8c5631382844 100644 --- a/tvix/glue/src/tvix_store_io.rs +++ b/tvix/glue/src/tvix_store_io.rs @@ -20,7 +20,8 @@ use tvix_store::nar::NarCalculationService; use tvix_castore::{ blobservice::BlobService, - directoryservice::{self, DirectoryService, NamedNode, Node}, + directoryservice::{self, DirectoryService}, + {NamedNode, Node}, }; use tvix_store::{pathinfoservice::PathInfoService, proto::PathInfo}; diff --git a/tvix/nar-bridge/src/nar.rs b/tvix/nar-bridge/src/nar.rs index 1dba05f5b38f..acbcfa2817bc 100644 --- a/tvix/nar-bridge/src/nar.rs +++ b/tvix/nar-bridge/src/nar.rs @@ -52,7 +52,7 @@ pub async fn get( StatusCode::NOT_FOUND })?; - let root_node: tvix_castore::directoryservice::Node = (&root_node).try_into().map_err(|e| { + let root_node: tvix_castore::Node = (&root_node).try_into().map_err(|e| { warn!(err=%e, "root node validation failed"); StatusCode::BAD_REQUEST })?; diff --git a/tvix/nar-bridge/src/narinfo.rs b/tvix/nar-bridge/src/narinfo.rs index a883ac88ce18..fc19bdc871cd 100644 --- a/tvix/nar-bridge/src/narinfo.rs +++ b/tvix/nar-bridge/src/narinfo.rs @@ -67,11 +67,10 @@ pub async fn get( })?; // encode the (unnamed) root node in the NAR url itself. - let root_node = tvix_castore::directoryservice::Node::try_from( - path_info.node.as_ref().expect("root node must not be none"), - ) - .unwrap() // PathInfo is validated - .rename("".into()); + let root_node = + tvix_castore::Node::try_from(path_info.node.as_ref().expect("root node must not be none")) + .unwrap() // PathInfo is validated + .rename("".into()); let mut buf = Vec::new(); Node::encode(&(&root_node).into(), &mut buf); @@ -126,7 +125,7 @@ pub async fn put( // Lookup root node with peek, as we don't want to update the LRU list. // We need to be careful to not hold the RwLock across the await point. - let maybe_root_node: Option<tvix_castore::directoryservice::Node> = root_nodes + let maybe_root_node: Option<tvix_castore::Node> = root_nodes .read() .peek(&narinfo.nar_hash) .and_then(|v| v.try_into().ok()); diff --git a/tvix/store/src/import.rs b/tvix/store/src/import.rs index 7dd4770f4b99..a94ff9f2cdfc 100644 --- a/tvix/store/src/import.rs +++ b/tvix/store/src/import.rs @@ -2,8 +2,9 @@ use std::path::Path; use tracing::{debug, instrument}; use tvix_castore::{ blobservice::BlobService, - directoryservice::{DirectoryService, NamedNode, Node}, + directoryservice::DirectoryService, import::fs::ingest_path, + {NamedNode, Node}, }; use nix_compat::{ diff --git a/tvix/store/src/nar/import.rs b/tvix/store/src/nar/import.rs index 9f3d0e0d74ea..ac50e7e4301f 100644 --- a/tvix/store/src/nar/import.rs +++ b/tvix/store/src/nar/import.rs @@ -7,12 +7,12 @@ use tokio::{ }; use tvix_castore::{ blobservice::BlobService, - directoryservice::{DirectoryService, NamedNode, Node}, + directoryservice::DirectoryService, import::{ blobs::{self, ConcurrentBlobUploader}, ingest_entries, IngestionEntry, IngestionError, }, - PathBuf, + PathBuf, {NamedNode, Node}, }; /// Ingests the contents from a [AsyncRead] providing NAR into the tvix store, @@ -171,13 +171,12 @@ mod test { use rstest::*; use tokio_stream::StreamExt; use tvix_castore::blobservice::BlobService; - use tvix_castore::directoryservice::{ - Directory, DirectoryNode, DirectoryService, FileNode, Node, SymlinkNode, - }; + use tvix_castore::directoryservice::DirectoryService; use tvix_castore::fixtures::{ DIRECTORY_COMPLICATED, DIRECTORY_WITH_KEEP, EMPTY_BLOB_DIGEST, HELLOWORLD_BLOB_CONTENTS, HELLOWORLD_BLOB_DIGEST, }; + use tvix_castore::{Directory, DirectoryNode, FileNode, Node, SymlinkNode}; use crate::tests::fixtures::{ blob_service, directory_service, NAR_CONTENTS_COMPLICATED, NAR_CONTENTS_HELLOWORLD, diff --git a/tvix/store/src/nar/mod.rs b/tvix/store/src/nar/mod.rs index 8a19f6bd6c47..ca2423b5ce59 100644 --- a/tvix/store/src/nar/mod.rs +++ b/tvix/store/src/nar/mod.rs @@ -8,7 +8,7 @@ pub use import::ingest_nar_and_hash; pub use renderer::calculate_size_and_sha256; pub use renderer::write_nar; pub use renderer::SimpleRenderer; -use tvix_castore::directoryservice::Node; +use tvix_castore::Node; #[async_trait] pub trait NarCalculationService: Send + Sync { diff --git a/tvix/store/src/nar/renderer.rs b/tvix/store/src/nar/renderer.rs index bb60f7835810..fefc76956e7e 100644 --- a/tvix/store/src/nar/renderer.rs +++ b/tvix/store/src/nar/renderer.rs @@ -10,7 +10,8 @@ use tracing::{instrument, Span}; use tracing_indicatif::span_ext::IndicatifSpanExt; use tvix_castore::{ blobservice::BlobService, - directoryservice::{DirectoryService, NamedNode, Node}, + directoryservice::DirectoryService, + {NamedNode, Node}, }; pub struct SimpleRenderer<BS, DS> { diff --git a/tvix/store/src/pathinfoservice/fs/mod.rs b/tvix/store/src/pathinfoservice/fs/mod.rs index 664cb8bbd54e..9a991a41d28d 100644 --- a/tvix/store/src/pathinfoservice/fs/mod.rs +++ b/tvix/store/src/pathinfoservice/fs/mod.rs @@ -1,10 +1,9 @@ use futures::stream::BoxStream; use futures::StreamExt; use tonic::async_trait; -use tvix_castore::directoryservice::Node; use tvix_castore::fs::{RootNodes, TvixStoreFs}; use tvix_castore::{blobservice::BlobService, directoryservice::DirectoryService}; -use tvix_castore::{Error, ValidateNodeError}; +use tvix_castore::{Error, Node, ValidateNodeError}; use super::PathInfoService; diff --git a/tvix/store/src/pathinfoservice/grpc.rs b/tvix/store/src/pathinfoservice/grpc.rs index 1ba1279ec626..187d9a148472 100644 --- a/tvix/store/src/pathinfoservice/grpc.rs +++ b/tvix/store/src/pathinfoservice/grpc.rs @@ -11,8 +11,8 @@ use tonic::{async_trait, Code}; use tracing::{instrument, Span}; use tracing_indicatif::span_ext::IndicatifSpanExt; use tvix_castore::composition::{CompositionContext, ServiceBuilder}; -use tvix_castore::directoryservice::Node; use tvix_castore::Error; +use tvix_castore::Node; /// Connects to a (remote) tvix-store PathInfoService over gRPC. #[derive(Clone)] diff --git a/tvix/store/src/pathinfoservice/lru.rs b/tvix/store/src/pathinfoservice/lru.rs index 5d808cd988aa..3a7f01eb70ac 100644 --- a/tvix/store/src/pathinfoservice/lru.rs +++ b/tvix/store/src/pathinfoservice/lru.rs @@ -109,7 +109,7 @@ mod test { let root_node = p.node.as_mut().unwrap(); if let castorepb::Node { node: Some(node) } = root_node { let n = node.to_owned(); - *node = (&tvix_castore::directoryservice::Node::try_from(&n) + *node = (&tvix_castore::Node::try_from(&n) .unwrap() .rename("11111111111111111111111111111111-dummy2".into())) .into(); diff --git a/tvix/store/src/proto/mod.rs b/tvix/store/src/proto/mod.rs index 5434df49daaa..375fb0dcadb8 100644 --- a/tvix/store/src/proto/mod.rs +++ b/tvix/store/src/proto/mod.rs @@ -9,8 +9,7 @@ use nix_compat::{ store_path::{self, StorePathRef}, }; use thiserror::Error; -use tvix_castore::directoryservice::NamedNode; -use tvix_castore::ValidateNodeError; +use tvix_castore::{NamedNode, ValidateNodeError}; mod grpc_pathinfoservice_wrapper; @@ -162,7 +161,7 @@ impl PathInfo { None => Err(ValidatePathInfoError::NoNodePresent)?, Some(node) => { // TODO save result somewhere - let node: tvix_castore::directoryservice::Node = node + let node: tvix_castore::Node = node .try_into() .map_err(ValidatePathInfoError::InvalidRootNode)?; // parse the name of the node itself and return diff --git a/tvix/store/src/proto/tests/pathinfo.rs b/tvix/store/src/proto/tests/pathinfo.rs index 1e4e7199049a..928bb8c8b185 100644 --- a/tvix/store/src/proto/tests/pathinfo.rs +++ b/tvix/store/src/proto/tests/pathinfo.rs @@ -231,7 +231,7 @@ fn validate_symlink_empty_target_invalid() { target: "".into(), }); - tvix_castore::directoryservice::Node::try_from(&node).expect_err("must fail validation"); + tvix_castore::Node::try_from(&node).expect_err("must fail validation"); } /// Create a node with a symlink target including null bytes, and ensure it @@ -243,7 +243,7 @@ fn validate_symlink_target_null_byte_invalid() { target: "foo\0".into(), }); - tvix_castore::directoryservice::Node::try_from(&node).expect_err("must fail validation"); + tvix_castore::Node::try_from(&node).expect_err("must fail validation"); } /// Create a PathInfo with a correct deriver field and ensure it succeeds. diff --git a/tvix/store/src/tests/nar_renderer.rs b/tvix/store/src/tests/nar_renderer.rs index d2ee42f8df18..5c2c60ade410 100644 --- a/tvix/store/src/tests/nar_renderer.rs +++ b/tvix/store/src/tests/nar_renderer.rs @@ -9,9 +9,8 @@ use std::io; use std::sync::Arc; use tokio::io::sink; use tvix_castore::blobservice::BlobService; -use tvix_castore::directoryservice::{ - DirectoryNode, DirectoryService, FileNode, Node, SymlinkNode, -}; +use tvix_castore::directoryservice::DirectoryService; +use tvix_castore::{DirectoryNode, FileNode, Node, SymlinkNode}; #[rstest] #[tokio::test] |