From f9b5fc49b123cb4db3941ee2ae9b891f5262deef Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Sun, 3 Sep 2023 17:10:06 +0300 Subject: feat(tvix/store/fuse): allow listing This provides an additional configuration flag to the tvix-store mount subcommand, and logic in the fuse module to request listing for the root of the mountpoint. Change-Id: I05a8bc11f7991b574696f27a30afe0f4e718a58c Reviewed-on: https://cl.tvl.fyi/c/depot/+/9217 Autosubmit: flokli Reviewed-by: adisbladis Tested-by: BuildkiteCI --- tvix/store/src/bin/tvix-store.rs | 12 ++++++++- tvix/store/src/fuse/mod.rs | 57 ++++++++++++++++++++++++++++++++++++++-- tvix/store/src/fuse/tests.rs | 46 +++++++++++++++++++++++++++++++- 3 files changed, 111 insertions(+), 4 deletions(-) (limited to 'tvix/store') diff --git a/tvix/store/src/bin/tvix-store.rs b/tvix/store/src/bin/tvix-store.rs index 5b6c98e4ee45..d2a8927351f9 100644 --- a/tvix/store/src/bin/tvix-store.rs +++ b/tvix/store/src/bin/tvix-store.rs @@ -90,6 +90,10 @@ enum Commands { #[arg(long, env, default_value = "grpc+http://[::1]:8000")] path_info_service_addr: String, + + /// Whether to list elements at the root of the mount point. + #[clap(long, short, action)] + list_root: bool, }, } @@ -250,6 +254,7 @@ async fn main() -> Result<(), Box> { blob_service_addr, directory_service_addr, path_info_service_addr, + list_root, } => { let blob_service = blobservice::from_addr(&blob_service_addr)?; let directory_service = directoryservice::from_addr(&directory_service_addr)?; @@ -260,7 +265,12 @@ async fn main() -> Result<(), Box> { )?; tokio::task::spawn_blocking(move || { - let f = FUSE::new(blob_service, directory_service, path_info_service); + let f = FUSE::new( + blob_service, + directory_service, + path_info_service, + list_root, + ); fuser::mount2(f, &dest, &[]) }) .await?? diff --git a/tvix/store/src/fuse/mod.rs b/tvix/store/src/fuse/mod.rs index 8b44f7db550e..0015abb9d557 100644 --- a/tvix/store/src/fuse/mod.rs +++ b/tvix/store/src/fuse/mod.rs @@ -66,6 +66,9 @@ pub struct FUSE { directory_service: Arc, path_info_service: Arc, + /// Whether to (try) listing elements in the root. + list_root: bool, + /// This maps a given StorePath to the inode we allocated for the root inode. store_paths: HashMap, @@ -83,12 +86,15 @@ impl FUSE { blob_service: Arc, directory_service: Arc, path_info_service: Arc, + list_root: bool, ) -> Self { Self { blob_service, directory_service, path_info_service, + list_root, + store_paths: HashMap::default(), inode_tracker: Default::default(), @@ -311,8 +317,55 @@ impl fuser::Filesystem for FUSE { debug!("readdir"); if ino == fuser::FUSE_ROOT_ID { - reply.error(libc::EPERM); // same error code as ipfs/kubo - return; + if !self.list_root { + reply.error(libc::EPERM); // same error code as ipfs/kubo + return; + } else { + for (i, path_info) in self + .path_info_service + .list() + .skip(offset as usize) + .enumerate() + { + let path_info = match path_info { + Err(e) => { + warn!("failed to retrieve pathinfo: {}", e); + reply.error(libc::EPERM); + return; + } + Ok(path_info) => path_info, + }; + + // We know the root node exists and the store_path can be parsed because clients MUST validate. + let root_node = path_info.node.unwrap().node.unwrap(); + let store_path = StorePath::from_bytes(root_node.get_name()).unwrap(); + + let ino = match self.store_paths.get(&store_path) { + Some(ino) => *ino, + None => { + // insert the (sparse) inode data and register in + // self.store_paths. + let ino = self.inode_tracker.put((&root_node).into()); + self.store_paths.insert(store_path.clone(), ino); + ino + } + }; + + let ty = match root_node { + Node::Directory(_) => fuser::FileType::Directory, + Node::File(_) => fuser::FileType::RegularFile, + Node::Symlink(_) => fuser::FileType::Symlink, + }; + + let full = + reply.add(ino, offset + i as i64 + 1_i64, ty, store_path.to_string()); + if full { + break; + } + } + reply.ok(); + return; + } } // lookup the inode data. diff --git a/tvix/store/src/fuse/tests.rs b/tvix/store/src/fuse/tests.rs index e5e36579b6c2..2c99f75471a5 100644 --- a/tvix/store/src/fuse/tests.rs +++ b/tvix/store/src/fuse/tests.rs @@ -25,6 +25,17 @@ fn setup_and_mount, F>( mountpoint: P, setup_fn: F, ) -> Result +where + F: Fn(Arc, Arc, Arc), +{ + setup_and_mount_with_listing(mountpoint, setup_fn, false) +} + +fn setup_and_mount_with_listing, F>( + mountpoint: P, + setup_fn: F, + list_root: bool, +) -> Result where F: Fn(Arc, Arc, Arc), { @@ -38,7 +49,12 @@ where path_info_service.clone(), ); - let fs = FUSE::new(blob_service, directory_service, path_info_service); + let fs = FUSE::new( + blob_service, + directory_service, + path_info_service, + list_root, + ); fuser::spawn_mount2(fs, mountpoint, &[]) } @@ -280,6 +296,34 @@ fn root() { fuser_session.join() } +/// Ensure listing the root is allowed if configured explicitly +#[test] +fn root_with_listing() { + // https://plume.benboeckel.net/~/JustAnotherBlog/skipping-tests-in-rust + if !std::path::Path::new("/dev/fuse").exists() { + eprintln!("skipping test"); + return; + } + let tmpdir = TempDir::new().unwrap(); + + let fuser_session = + setup_and_mount_with_listing(tmpdir.path(), populate_blob_a, true).expect("must succeed"); + + { + // read_dir succeeds, but getting the first element will fail. + let mut it = fs::read_dir(tmpdir).expect("must succeed"); + + let e = it.next().expect("must be some").expect("must succeed"); + + let metadata = e.metadata().expect("must succeed"); + assert!(metadata.is_file()); + assert!(metadata.permissions().readonly()); + assert_eq!(fixtures::BLOB_A.len() as u64, metadata.len()); + } + + fuser_session.join() +} + /// Ensure we can stat a file at the root #[test] fn stat_file_at_root() { -- cgit 1.4.1