From cfb810d81a4f5ba60e8d3c5502390d60799aa636 Mon Sep 17 00:00:00 2001 From: Brian Olsen Date: Fri, 29 Sep 2023 18:50:50 +0200 Subject: fix(tvix/store): Fix FUSE support on MacOS This partially fixes b/312 and gets FUSE to work again on MacOS. It is mostly small type changes and an update to fuse-backend-rs because upstream currently doesn't work with MacFuse. It also sets the default FUSE thread count on MacOS to 1 because otherwise the mount command will hang when shutting down as only one thread gets ENODEV and all the others just keep blocking. Change-Id: Ifb3c4268caf296c487049c1dc4618acb32497f44 Reviewed-on: https://cl.tvl.fyi/c/depot/+/9490 Tested-by: BuildkiteCI Reviewed-by: Connor Brewster Reviewed-by: flokli --- tvix/Cargo.lock | 2 +- tvix/Cargo.nix | 6 +++--- tvix/crate-hashes.json | 2 +- tvix/store/Cargo.toml | 6 +++--- tvix/store/default.nix | 4 ++-- tvix/store/src/bin/tvix-store.rs | 9 ++++++++- tvix/store/src/fs/file_attr.rs | 16 +++++++++++----- tvix/store/src/fs/fuse.rs | 8 +++++++- tvix/store/src/fs/mod.rs | 11 ++++++----- 9 files changed, 42 insertions(+), 22 deletions(-) (limited to 'tvix') diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock index 5129a2ba33..a8d790e05f 100644 --- a/tvix/Cargo.lock +++ b/tvix/Cargo.lock @@ -773,7 +773,7 @@ checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" [[package]] name = "fuse-backend-rs" version = "0.10.5" -source = "git+https://github.com/cloud-hypervisor/fuse-backend-rs?rev=402e7c531bc75bc44ac366dc59477de8b5d4ca08#402e7c531bc75bc44ac366dc59477de8b5d4ca08" +source = "git+https://github.com/griff/fuse-backend-rs?branch=macfuse-fix#70b835cada7e1f18e5cbb13f6c4b698ba203c820" dependencies = [ "arc-swap", "bitflags", diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix index 356c1e9929..f0eb27a6ef 100644 --- a/tvix/Cargo.nix +++ b/tvix/Cargo.nix @@ -2190,9 +2190,9 @@ rec { edition = "2018"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/cloud-hypervisor/fuse-backend-rs"; - rev = "402e7c531bc75bc44ac366dc59477de8b5d4ca08"; - sha256 = "0f70f0wxkx4h18wvkpnnpxhyzvg6f9z1063334w1nlfg0n15wb9y"; + url = "https://github.com/griff/fuse-backend-rs"; + rev = "70b835cada7e1f18e5cbb13f6c4b698ba203c820"; + sha256 = "107iaw8zqsz888xh9nkq3vvki1c1rqqqg0mncdplradhhn7wp3kp"; }; authors = [ "Liu Bo " diff --git a/tvix/crate-hashes.json b/tvix/crate-hashes.json index 6c7f1e04ce..93ab68a5b8 100644 --- a/tvix/crate-hashes.json +++ b/tvix/crate-hashes.json @@ -1,5 +1,5 @@ { - "fuse-backend-rs 0.10.5 (git+https://github.com/cloud-hypervisor/fuse-backend-rs?rev=402e7c531bc75bc44ac366dc59477de8b5d4ca08#402e7c531bc75bc44ac366dc59477de8b5d4ca08)": "0f70f0wxkx4h18wvkpnnpxhyzvg6f9z1063334w1nlfg0n15wb9y", + "fuse-backend-rs 0.10.5 (git+https://github.com/griff/fuse-backend-rs?branch=macfuse-fix#70b835cada7e1f18e5cbb13f6c4b698ba203c820)": "107iaw8zqsz888xh9nkq3vvki1c1rqqqg0mncdplradhhn7wp3kp", "test-generator 0.3.0 (git+https://github.com/JamesGuthrie/test-generator.git?rev=82e799979980962aec1aa324ec6e0e4cad781f41#82e799979980962aec1aa324ec6e0e4cad781f41)": "08brp3qqa55hijc7xby3lam2cc84hvx1zzfqv6lj7smlczh8k32y", "tonic-mock 0.1.0 (git+https://github.com/brainrake/tonic-mock?branch=bump-dependencies#ec1a15510875de99d709d684190db5d9beab175e)": "0lwa03hpp0mxa6aa1zv5w68k61y4hccfm0q2ykyq392fwal8vb50", "wu-manber 0.1.0 (git+https://github.com/tvlfyi/wu-manber.git#0d5b22bea136659f7de60b102a7030e0daaa503d)": "1zhk83lbq99xzyjwphv2qrb8f8qgfqwa5bbbvyzm0z0bljsjv0pd" diff --git a/tvix/store/Cargo.toml b/tvix/store/Cargo.toml index da3d23cb30..d566ebed70 100644 --- a/tvix/store/Cargo.toml +++ b/tvix/store/Cargo.toml @@ -34,9 +34,9 @@ tokio-listener = { version = "0.2.1" } [dependencies.fuse-backend-rs] optional = true -# TODO: Switch back to upstream version once https://github.com/cloud-hypervisor/fuse-backend-rs/pull/153 lands. -git = "https://github.com/cloud-hypervisor/fuse-backend-rs" -rev = "402e7c531bc75bc44ac366dc59477de8b5d4ca08" +# TODO: Switch back to upstream version once https://github.com/cloud-hypervisor/fuse-backend-rs/pull/157 lands. +git = "https://github.com/griff/fuse-backend-rs" +branch = "macfuse-fix" [dependencies.vhost] optional = true diff --git a/tvix/store/default.nix b/tvix/store/default.nix index 90fa3f06a2..0372047e94 100644 --- a/tvix/store/default.nix +++ b/tvix/store/default.nix @@ -24,8 +24,8 @@ in (depot.tvix.crates.workspaceMembers.tvix-store.build.override { runTests = true; - # both fuse and virtiofs features currently fail to build on Darwin. - features = if pkgs.stdenv.isDarwin then [ "tonic-reflection" ] else [ "default" ]; + # virtiofs feature currently fails to build on Darwin. + features = if pkgs.stdenv.isDarwin then [ "fuse" "tonic-reflection" ] else [ "default" ]; }).overrideAttrs (_: { meta.ci.extraSteps = { import-docs = (mkImportCheck "tvix/store/docs" ./docs); diff --git a/tvix/store/src/bin/tvix-store.rs b/tvix/store/src/bin/tvix-store.rs index aeae270f35..548176e24f 100644 --- a/tvix/store/src/bin/tvix-store.rs +++ b/tvix/store/src/bin/tvix-store.rs @@ -139,12 +139,19 @@ enum Commands { }, } -#[cfg(feature = "fuse")] +#[cfg(all(feature = "fuse", not(target_os = "macos")))] fn default_threads() -> usize { std::thread::available_parallelism() .map(|threads| threads.into()) .unwrap_or(4) } +// On MacFUSE only a single channel will receive ENODEV when the file system is +// unmounted and so all the other channels will block forever. +// See https://github.com/osxfuse/osxfuse/issues/974 +#[cfg(all(feature = "fuse", target_os = "macos"))] +fn default_threads() -> usize { + 1 +} #[tokio::main] async fn main() -> Result<(), Box> { diff --git a/tvix/store/src/fs/file_attr.rs b/tvix/store/src/fs/file_attr.rs index b946aa977a..562cd9f190 100644 --- a/tvix/store/src/fs/file_attr.rs +++ b/tvix/store/src/fs/file_attr.rs @@ -7,7 +7,7 @@ pub const ROOT_FILE_ATTR: Attr = Attr { size: 0, blksize: 1024, blocks: 0, - mode: libc::S_IFDIR | 0o555, + mode: libc::S_IFDIR as u32 | 0o555, atime: 0, mtime: 0, ctime: 0, @@ -19,6 +19,12 @@ pub const ROOT_FILE_ATTR: Attr = Attr { gid: 0, rdev: 0, flags: 0, + #[cfg(target_os = "macos")] + crtime: 0, + #[cfg(target_os = "macos")] + crtimensec: 0, + #[cfg(target_os = "macos")] + padding: 0, }; /// for given &Node and inode, construct an [Attr] @@ -36,10 +42,10 @@ pub fn gen_file_attr(inode_data: &InodeData, inode: u64) -> Attr { } }, mode: match inode_data { - InodeData::Regular(_, _, false) => libc::S_IFREG | 0o444, // no-executable files - InodeData::Regular(_, _, true) => libc::S_IFREG | 0o555, // executable files - InodeData::Symlink(_) => libc::S_IFLNK | 0o444, - InodeData::Directory(_) => libc::S_IFDIR | 0o555, + InodeData::Regular(_, _, false) => libc::S_IFREG as u32 | 0o444, // no-executable files + InodeData::Regular(_, _, true) => libc::S_IFREG as u32 | 0o555, // executable files + InodeData::Symlink(_) => libc::S_IFLNK as u32 | 0o444, + InodeData::Directory(_) => libc::S_IFDIR as u32 | 0o555, }, ..Default::default() } diff --git a/tvix/store/src/fs/fuse.rs b/tvix/store/src/fs/fuse.rs index 8535c78584..d2a7348821 100644 --- a/tvix/store/src/fs/fuse.rs +++ b/tvix/store/src/fs/fuse.rs @@ -11,6 +11,11 @@ where channel: fuse_backend_rs::transport::FuseChannel, } +#[cfg(target_os = "macos")] +const BADFD: libc::c_int = libc::EBADF; +#[cfg(target_os = "linux")] +const BADFD: libc::c_int = libc::EBADFD; + impl FuseServer where FS: FileSystem + Sync + Send, @@ -29,7 +34,7 @@ where match e { // This indicates the session has been shut down. fuse_backend_rs::Error::EncodeMessage(e) - if e.raw_os_error() == Some(libc::EBADFD) => + if e.raw_os_error() == Some(BADFD) => { break; } @@ -63,6 +68,7 @@ impl FuseDaemon { let mut session = FuseSession::new(mountpoint.as_ref(), "tvix-store", "", true) .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + #[cfg(target_os = "linux")] session.set_allow_other(false); session .mount() diff --git a/tvix/store/src/fs/mod.rs b/tvix/store/src/fs/mod.rs index 91adfa35f0..1333983460 100644 --- a/tvix/store/src/fs/mod.rs +++ b/tvix/store/src/fs/mod.rs @@ -13,6 +13,7 @@ mod tests; use crate::pathinfoservice::PathInfoService; +use fuse_backend_rs::abi::fuse_abi::stat64; use fuse_backend_rs::api::filesystem::{Context, FileSystem, FsOptions, ROOT_ID}; use futures::StreamExt; use nix_compat::store_path::StorePath; @@ -253,7 +254,7 @@ impl FileSystem for TvixStoreFs { _ctx: &Context, inode: Self::Inode, _handle: Option, - ) -> io::Result<(libc::stat64, Duration)> { + ) -> io::Result<(stat64, Duration)> { if inode == ROOT_ID { return Ok((ROOT_FILE_ATTR.into(), Duration::MAX)); } @@ -441,7 +442,7 @@ impl FileSystem for TvixStoreFs { let written = add_entry(fuse_backend_rs::api::filesystem::DirEntry { ino, offset: offset + i as u64 + 1, - type_: ty, + type_: ty as u32, name: store_path.to_string().as_bytes(), })?; // If the buffer is full, add_entry will return `Ok(0)`. @@ -490,9 +491,9 @@ impl FileSystem for TvixStoreFs { ino: *ino, offset: offset + i as u64 + 1, type_: match child_node { - Node::Directory(_) => libc::S_IFDIR, - Node::File(_) => libc::S_IFREG, - Node::Symlink(_) => libc::S_IFLNK, + Node::Directory(_) => libc::S_IFDIR as u32, + Node::File(_) => libc::S_IFREG as u32, + Node::Symlink(_) => libc::S_IFLNK as u32, }, name: child_node.get_name(), })?; -- cgit 1.4.1