fix(tvix/store): Fix FUSE support on MacOS r/6687

This partially fixes b/312 and gets FUSE to work again on MacOS.

It is mostly small type changes and an update to fuse-backend-rs because
upstream currently doesn't work with MacFuse. It also sets the default
FUSE thread count on MacOS to 1 because otherwise the mount command will
hang when shutting down as only one thread gets ENODEV and all the others
just keep blocking.

Change-Id: Ifb3c4268caf296c487049c1dc4618acb32497f44
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9490
Tested-by: BuildkiteCI
Reviewed-by: Connor Brewster <cbrewster@hey.com>
Reviewed-by: flokli <flokli@flokli.de>

9 files changed, 42 insertions, 22 deletions

diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock
index 5129a2ba33bc..a8d790e05f45 100644
--- a/tvix/Cargo.lock
+++ b/tvix/Cargo.lock
@@ -773,7 +773,7 @@ checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
 [[package]]
 name = "fuse-backend-rs"
 version = "0.10.5"
-source = "git+https://github.com/cloud-hypervisor/fuse-backend-rs?rev=402e7c531bc75bc44ac366dc59477de8b5d4ca08#402e7c531bc75bc44ac366dc59477de8b5d4ca08"
+source = "git+https://github.com/griff/fuse-backend-rs?branch=macfuse-fix#70b835cada7e1f18e5cbb13f6c4b698ba203c820"
 dependencies = [
  "arc-swap",
  "bitflags",
diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix
index 356c1e992990..f0eb27a6ef35 100644
--- a/tvix/Cargo.nix
+++ b/tvix/Cargo.nix
@@ -2190,9 +2190,9 @@ rec {
         edition = "2018";
         workspace_member = null;
         src = pkgs.fetchgit {
-          url = "https://github.com/cloud-hypervisor/fuse-backend-rs";
-          rev = "402e7c531bc75bc44ac366dc59477de8b5d4ca08";
-          sha256 = "0f70f0wxkx4h18wvkpnnpxhyzvg6f9z1063334w1nlfg0n15wb9y";
+          url = "https://github.com/griff/fuse-backend-rs";
+          rev = "70b835cada7e1f18e5cbb13f6c4b698ba203c820";
+          sha256 = "107iaw8zqsz888xh9nkq3vvki1c1rqqqg0mncdplradhhn7wp3kp";
         };
         authors = [
           "Liu Bo <bo.liu@linux.alibaba.com>"
diff --git a/tvix/crate-hashes.json b/tvix/crate-hashes.json
index 6c7f1e04ce5d..93ab68a5b821 100644
--- a/tvix/crate-hashes.json
+++ b/tvix/crate-hashes.json
@@ -1,5 +1,5 @@
 {
-  "fuse-backend-rs 0.10.5 (git+https://github.com/cloud-hypervisor/fuse-backend-rs?rev=402e7c531bc75bc44ac366dc59477de8b5d4ca08#402e7c531bc75bc44ac366dc59477de8b5d4ca08)": "0f70f0wxkx4h18wvkpnnpxhyzvg6f9z1063334w1nlfg0n15wb9y",
+  "fuse-backend-rs 0.10.5 (git+https://github.com/griff/fuse-backend-rs?branch=macfuse-fix#70b835cada7e1f18e5cbb13f6c4b698ba203c820)": "107iaw8zqsz888xh9nkq3vvki1c1rqqqg0mncdplradhhn7wp3kp",
   "test-generator 0.3.0 (git+https://github.com/JamesGuthrie/test-generator.git?rev=82e799979980962aec1aa324ec6e0e4cad781f41#82e799979980962aec1aa324ec6e0e4cad781f41)": "08brp3qqa55hijc7xby3lam2cc84hvx1zzfqv6lj7smlczh8k32y",
   "tonic-mock 0.1.0 (git+https://github.com/brainrake/tonic-mock?branch=bump-dependencies#ec1a15510875de99d709d684190db5d9beab175e)": "0lwa03hpp0mxa6aa1zv5w68k61y4hccfm0q2ykyq392fwal8vb50",
   "wu-manber 0.1.0 (git+https://github.com/tvlfyi/wu-manber.git#0d5b22bea136659f7de60b102a7030e0daaa503d)": "1zhk83lbq99xzyjwphv2qrb8f8qgfqwa5bbbvyzm0z0bljsjv0pd"
diff --git a/tvix/store/Cargo.toml b/tvix/store/Cargo.toml
index da3d23cb309a..d566ebed7059 100644
--- a/tvix/store/Cargo.toml
+++ b/tvix/store/Cargo.toml
@@ -34,9 +34,9 @@ tokio-listener = { version = "0.2.1" }
 
 [dependencies.fuse-backend-rs]
 optional = true
-# TODO: Switch back to upstream version once https://github.com/cloud-hypervisor/fuse-backend-rs/pull/153 lands.
-git = "https://github.com/cloud-hypervisor/fuse-backend-rs"
-rev = "402e7c531bc75bc44ac366dc59477de8b5d4ca08"
+# TODO: Switch back to upstream version once https://github.com/cloud-hypervisor/fuse-backend-rs/pull/157 lands.
+git = "https://github.com/griff/fuse-backend-rs"
+branch = "macfuse-fix"
 
 [dependencies.vhost]
 optional = true
diff --git a/tvix/store/default.nix b/tvix/store/default.nix
index 90fa3f06a2d9..0372047e94fe 100644
--- a/tvix/store/default.nix
+++ b/tvix/store/default.nix
@@ -24,8 +24,8 @@ in
 
 (depot.tvix.crates.workspaceMembers.tvix-store.build.override {
   runTests = true;
-  # both fuse and virtiofs features currently fail to build on Darwin.
-  features = if pkgs.stdenv.isDarwin then [ "tonic-reflection" ] else [ "default" ];
+  # virtiofs feature currently fails to build on Darwin.
+  features = if pkgs.stdenv.isDarwin then [ "fuse" "tonic-reflection" ] else [ "default" ];
 }).overrideAttrs (_: {
   meta.ci.extraSteps = {
     import-docs = (mkImportCheck "tvix/store/docs" ./docs);
diff --git a/tvix/store/src/bin/tvix-store.rs b/tvix/store/src/bin/tvix-store.rs
index aeae270f3582..548176e24fa2 100644
--- a/tvix/store/src/bin/tvix-store.rs
+++ b/tvix/store/src/bin/tvix-store.rs
@@ -139,12 +139,19 @@ enum Commands {
     },
 }
 
-#[cfg(feature = "fuse")]
+#[cfg(all(feature = "fuse", not(target_os = "macos")))]
 fn default_threads() -> usize {
     std::thread::available_parallelism()
         .map(|threads| threads.into())
         .unwrap_or(4)
 }
+// On MacFUSE only a single channel will receive ENODEV when the file system is
+// unmounted and so all the other channels will block forever.
+// See https://github.com/osxfuse/osxfuse/issues/974
+#[cfg(all(feature = "fuse", target_os = "macos"))]
+fn default_threads() -> usize {
+    1
+}
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
diff --git a/tvix/store/src/fs/file_attr.rs b/tvix/store/src/fs/file_attr.rs
index b946aa977a0a..562cd9f19002 100644
--- a/tvix/store/src/fs/file_attr.rs
+++ b/tvix/store/src/fs/file_attr.rs
@@ -7,7 +7,7 @@ pub const ROOT_FILE_ATTR: Attr = Attr {
     size: 0,
     blksize: 1024,
     blocks: 0,
-    mode: libc::S_IFDIR | 0o555,
+    mode: libc::S_IFDIR as u32 | 0o555,
     atime: 0,
     mtime: 0,
     ctime: 0,
@@ -19,6 +19,12 @@ pub const ROOT_FILE_ATTR: Attr = Attr {
     gid: 0,
     rdev: 0,
     flags: 0,
+    #[cfg(target_os = "macos")]
+    crtime: 0,
+    #[cfg(target_os = "macos")]
+    crtimensec: 0,
+    #[cfg(target_os = "macos")]
+    padding: 0,
 };
 
 /// for given &Node and inode, construct an [Attr]
@@ -36,10 +42,10 @@ pub fn gen_file_attr(inode_data: &InodeData, inode: u64) -> Attr {
             }
         },
         mode: match inode_data {
-            InodeData::Regular(_, _, false) => libc::S_IFREG | 0o444, // no-executable files
-            InodeData::Regular(_, _, true) => libc::S_IFREG | 0o555,  // executable files
-            InodeData::Symlink(_) => libc::S_IFLNK | 0o444,
-            InodeData::Directory(_) => libc::S_IFDIR | 0o555,
+            InodeData::Regular(_, _, false) => libc::S_IFREG as u32 | 0o444, // no-executable files
+            InodeData::Regular(_, _, true) => libc::S_IFREG as u32 | 0o555,  // executable files
+            InodeData::Symlink(_) => libc::S_IFLNK as u32 | 0o444,
+            InodeData::Directory(_) => libc::S_IFDIR as u32 | 0o555,
         },
         ..Default::default()
     }
diff --git a/tvix/store/src/fs/fuse.rs b/tvix/store/src/fs/fuse.rs
index 8535c7858450..d2a734882196 100644
--- a/tvix/store/src/fs/fuse.rs
+++ b/tvix/store/src/fs/fuse.rs
@@ -11,6 +11,11 @@ where
     channel: fuse_backend_rs::transport::FuseChannel,
 }
 
+#[cfg(target_os = "macos")]
+const BADFD: libc::c_int = libc::EBADF;
+#[cfg(target_os = "linux")]
+const BADFD: libc::c_int = libc::EBADFD;
+
 impl<FS> FuseServer<FS>
 where
     FS: FileSystem + Sync + Send,
@@ -29,7 +34,7 @@ where
                     match e {
                         // This indicates the session has been shut down.
                         fuse_backend_rs::Error::EncodeMessage(e)
-                            if e.raw_os_error() == Some(libc::EBADFD) =>
+                            if e.raw_os_error() == Some(BADFD) =>
                         {
                             break;
                         }
@@ -63,6 +68,7 @@ impl FuseDaemon {
         let mut session = FuseSession::new(mountpoint.as_ref(), "tvix-store", "", true)
             .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
 
+        #[cfg(target_os = "linux")]
         session.set_allow_other(false);
         session
             .mount()
diff --git a/tvix/store/src/fs/mod.rs b/tvix/store/src/fs/mod.rs
index 91adfa35f0e0..1333983460ea 100644
--- a/tvix/store/src/fs/mod.rs
+++ b/tvix/store/src/fs/mod.rs
@@ -13,6 +13,7 @@ mod tests;
 
 use crate::pathinfoservice::PathInfoService;
 
+use fuse_backend_rs::abi::fuse_abi::stat64;
 use fuse_backend_rs::api::filesystem::{Context, FileSystem, FsOptions, ROOT_ID};
 use futures::StreamExt;
 use nix_compat::store_path::StorePath;
@@ -253,7 +254,7 @@ impl FileSystem for TvixStoreFs {
         _ctx: &Context,
         inode: Self::Inode,
         _handle: Option<Self::Handle>,
-    ) -> io::Result<(libc::stat64, Duration)> {
+    ) -> io::Result<(stat64, Duration)> {
         if inode == ROOT_ID {
             return Ok((ROOT_FILE_ATTR.into(), Duration::MAX));
         }
@@ -441,7 +442,7 @@ impl FileSystem for TvixStoreFs {
                     let written = add_entry(fuse_backend_rs::api::filesystem::DirEntry {
                         ino,
                         offset: offset + i as u64 + 1,
-                        type_: ty,
+                        type_: ty as u32,
                         name: store_path.to_string().as_bytes(),
                     })?;
                     // If the buffer is full, add_entry will return `Ok(0)`.
@@ -490,9 +491,9 @@ impl FileSystem for TvixStoreFs {
                     ino: *ino,
                     offset: offset + i as u64 + 1,
                     type_: match child_node {
-                        Node::Directory(_) => libc::S_IFDIR,
-                        Node::File(_) => libc::S_IFREG,
-                        Node::Symlink(_) => libc::S_IFLNK,
+                        Node::Directory(_) => libc::S_IFDIR as u32,
+                        Node::File(_) => libc::S_IFREG as u32,
+                        Node::Symlink(_) => libc::S_IFLNK as u32,
                     },
                     name: child_node.get_name(),
                 })?;