diff options
author | Florian Klink <flokli@flokli.de> | 2023-12-16T22·16+0200 |
---|---|---|
committer | flokli <flokli@flokli.de> | 2023-12-22T16·55+0000 |
commit | a5167c508cf2ed92f8a39696a6b4376cf25ee872 (patch) | |
tree | 5ffb2f8d0d331b6fea1aeb4f6391e0408df6d234 /tvix | |
parent | 52cad8619511b97c4bcd5768ce9b3579ff665505 (diff) |
chore(tvix): move store/fs to castore/fs r/7256
With the recent introduction of the RootNodes trait, there's nothing in the fs module pulling in tvix-store dependencies, so it can live in tvix-castore. This allows other crates to make use of TvixStoreFS, without having to pull in tvix-store. For example, a tvix-build using a fuse mountpoint at /nix/store doesn't need a PathInfoService to hold the root nodes that should be present, but just a list. tvix-store now has a pathinfoservice/fs module, which contains the necessary glue logic to implement the RootNodes trait for a PathInfoService. To satisfy Rust orphan rules for trait implementations, we had to add a small wrapper struct. It's mostly hidden away by the make_fs helper function returning a TvixStoreFs. It can't be entirely private, as its still leaking into the concrete type of TvixStoreFS. tvix-store still has `fuse` and `virtiofs` features, but they now simply enable these features in the `tvix-castore` crate they depend on. The tests for the fuse functionality stay in tvix-store for now, as they populate the root nodes through a PathInfoService. Once above mentioned "list of root nodes" implementation exists, we might want to shuffle this around one more time. Fixes b/341. Change-Id: I989f664827a5a361b23b34368d242d10c157c756 Reviewed-on: https://cl.tvl.fyi/c/depot/+/10378 Autosubmit: flokli <flokli@flokli.de> Tested-by: BuildkiteCI Reviewed-by: sterni <sternenseemann@systemli.org>
Diffstat (limited to 'tvix')
-rw-r--r-- | tvix/Cargo.lock | 17 | ||||
-rw-r--r-- | tvix/Cargo.nix | 104 | ||||
-rw-r--r-- | tvix/castore/Cargo.toml | 46 | ||||
-rw-r--r-- | tvix/castore/src/fs/file_attr.rs (renamed from tvix/store/src/fs/file_attr.rs) | 0 | ||||
-rw-r--r-- | tvix/castore/src/fs/fuse.rs (renamed from tvix/store/src/fs/fuse.rs) | 0 | ||||
-rw-r--r-- | tvix/castore/src/fs/inode_tracker.rs (renamed from tvix/store/src/fs/inode_tracker.rs) | 4 | ||||
-rw-r--r-- | tvix/castore/src/fs/inodes.rs (renamed from tvix/store/src/fs/inodes.rs) | 6 | ||||
-rw-r--r-- | tvix/castore/src/fs/mod.rs (renamed from tvix/store/src/fs/mod.rs) | 20 | ||||
-rw-r--r-- | tvix/castore/src/fs/root_nodes.rs | 18 | ||||
-rw-r--r-- | tvix/castore/src/fs/virtiofs.rs (renamed from tvix/store/src/fs/virtiofs.rs) | 0 | ||||
-rw-r--r-- | tvix/castore/src/lib.rs | 4 | ||||
-rw-r--r-- | tvix/store/Cargo.toml | 51 | ||||
-rw-r--r-- | tvix/store/src/bin/tvix-store.rs | 14 | ||||
-rw-r--r-- | tvix/store/src/fs/root_nodes.rs | 61 | ||||
-rw-r--r-- | tvix/store/src/lib.rs | 3 | ||||
-rw-r--r-- | tvix/store/src/pathinfoservice/fs/mod.rs | 84 | ||||
-rw-r--r-- | tvix/store/src/pathinfoservice/fs/tests.rs (renamed from tvix/store/src/fs/tests.rs) | 18 | ||||
-rw-r--r-- | tvix/store/src/pathinfoservice/mod.rs | 6 |
18 files changed, 257 insertions, 199 deletions
diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock index 297adf5c846a..2f04f037a78e 100644 --- a/tvix/Cargo.lock +++ b/tvix/Cargo.lock @@ -3108,9 +3108,12 @@ dependencies = [ "bstr", "bytes", "data-encoding", + "fuse-backend-rs", "futures", "hex-literal", "lazy_static", + "libc", + "parking_lot 0.12.1", "pin-project-lite", "prost", "prost-build", @@ -3128,6 +3131,12 @@ dependencies = [ "tower", "tracing", "url", + "vhost", + "vhost-user-backend", + "virtio-bindings 0.2.1", + "virtio-queue", + "vm-memory", + "vmm-sys-util", "walkdir", ] @@ -3233,12 +3242,10 @@ dependencies = [ "clap", "count-write", "data-encoding", - "fuse-backend-rs", "futures", "lazy_static", "libc", "nix-compat", - "parking_lot 0.12.1", "pin-project-lite", "prost", "prost-build", @@ -3261,12 +3268,6 @@ dependencies = [ "tracing-subscriber", "tvix-castore", "url", - "vhost", - "vhost-user-backend", - "virtio-bindings 0.2.1", - "virtio-queue", - "vm-memory", - "vmm-sys-util", "walkdir", "xz2", ] diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix index 2c9120376871..5bae02070bcb 100644 --- a/tvix/Cargo.nix +++ b/tvix/Cargo.nix @@ -9609,6 +9609,11 @@ rec { packageId = "data-encoding"; } { + name = "fuse-backend-rs"; + packageId = "fuse-backend-rs"; + optional = true; + } + { name = "futures"; packageId = "futures"; } @@ -9617,6 +9622,15 @@ rec { packageId = "lazy_static"; } { + name = "libc"; + packageId = "libc"; + optional = true; + } + { + name = "parking_lot"; + packageId = "parking_lot 0.12.1"; + } + { name = "pin-project-lite"; packageId = "pin-project-lite"; } @@ -9669,6 +9683,36 @@ rec { packageId = "url"; } { + name = "vhost"; + packageId = "vhost"; + optional = true; + } + { + name = "vhost-user-backend"; + packageId = "vhost-user-backend"; + optional = true; + } + { + name = "virtio-bindings"; + packageId = "virtio-bindings 0.2.1"; + optional = true; + } + { + name = "virtio-queue"; + packageId = "virtio-queue"; + optional = true; + } + { + name = "vm-memory"; + packageId = "vm-memory"; + optional = true; + } + { + name = "vmm-sys-util"; + packageId = "vmm-sys-util"; + optional = true; + } + { name = "walkdir"; packageId = "walkdir"; } @@ -9702,9 +9746,12 @@ rec { } ]; features = { + "fs" = [ "dep:libc" "dep:fuse-backend-rs" ]; + "fuse" = [ "fs" ]; "tonic-reflection" = [ "dep:tonic-reflection" ]; + "virtiofs" = [ "fs" "dep:vhost" "dep:vhost-user-backend" "dep:virtio-queue" "dep:vm-memory" "dep:vmm-sys-util" "dep:virtio-bindings" "fuse-backend-rs?/vhost-user-fs" "fuse-backend-rs?/virtiofs" ]; }; - resolvedDefaultFeatures = [ "default" "tonic-reflection" ]; + resolvedDefaultFeatures = [ "default" "fs" "fuse" "tonic-reflection" "virtiofs" ]; }; "tvix-cli" = rec { crateName = "tvix-cli"; @@ -10111,11 +10158,6 @@ rec { packageId = "data-encoding"; } { - name = "fuse-backend-rs"; - packageId = "fuse-backend-rs"; - optional = true; - } - { name = "futures"; packageId = "futures"; } @@ -10124,20 +10166,11 @@ rec { packageId = "lazy_static"; } { - name = "libc"; - packageId = "libc"; - optional = true; - } - { name = "nix-compat"; packageId = "nix-compat"; features = [ "async" ]; } { - name = "parking_lot"; - packageId = "parking_lot 0.12.1"; - } - { name = "pin-project-lite"; packageId = "pin-project-lite"; } @@ -10215,36 +10248,6 @@ rec { packageId = "url"; } { - name = "vhost"; - packageId = "vhost"; - optional = true; - } - { - name = "vhost-user-backend"; - packageId = "vhost-user-backend"; - optional = true; - } - { - name = "virtio-bindings"; - packageId = "virtio-bindings 0.2.1"; - optional = true; - } - { - name = "virtio-queue"; - packageId = "virtio-queue"; - optional = true; - } - { - name = "vm-memory"; - packageId = "vm-memory"; - optional = true; - } - { - name = "vmm-sys-util"; - packageId = "vmm-sys-util"; - optional = true; - } - { name = "walkdir"; packageId = "walkdir"; } @@ -10265,6 +10268,10 @@ rec { ]; devDependencies = [ { + name = "libc"; + packageId = "libc"; + } + { name = "tempfile"; packageId = "tempfile"; } @@ -10279,12 +10286,11 @@ rec { ]; features = { "default" = [ "fuse" "tonic-reflection" ]; - "fs" = [ "dep:libc" "dep:fuse-backend-rs" ]; - "fuse" = [ "fs" ]; + "fuse" = [ "tvix-castore/fuse" ]; "tonic-reflection" = [ "dep:tonic-reflection" "tvix-castore/tonic-reflection" ]; - "virtiofs" = [ "fs" "dep:vhost" "dep:vhost-user-backend" "dep:virtio-queue" "dep:vm-memory" "dep:vmm-sys-util" "dep:virtio-bindings" "fuse-backend-rs?/vhost-user-fs" "fuse-backend-rs?/virtiofs" ]; + "virtiofs" = [ "tvix-castore/virtiofs" ]; }; - resolvedDefaultFeatures = [ "default" "fs" "fuse" "tonic-reflection" "virtiofs" ]; + resolvedDefaultFeatures = [ "default" "fuse" "tonic-reflection" "virtiofs" ]; }; "typenum" = rec { crateName = "typenum"; diff --git a/tvix/castore/Cargo.toml b/tvix/castore/Cargo.toml index 2a421280b82b..0f0146972969 100644 --- a/tvix/castore/Cargo.toml +++ b/tvix/castore/Cargo.toml @@ -10,6 +10,7 @@ bytes = "1.4.0" data-encoding = "2.3.3" futures = "0.3.28" lazy_static = "1.4.0" +parking_lot = "0.12.1" pin-project-lite = "0.2.13" prost = "0.12.1" sled = { version = "0.34.7" } @@ -25,10 +26,42 @@ walkdir = "2.4.0" bstr = "1.6.0" async-tempfile = "0.4.0" +[dependencies.fuse-backend-rs] +optional = true +version = "0.11.0" + +[dependencies.libc] +optional = true +version = "0.2.144" + [dependencies.tonic-reflection] optional = true version = "0.10.2" +[dependencies.vhost] +optional = true +version = "0.6" + +[dependencies.vhost-user-backend] +optional = true +version = "0.8" + +[dependencies.virtio-queue] +optional = true +version = "0.7" + +[dependencies.vm-memory] +optional = true +version = "0.10" + +[dependencies.vmm-sys-util] +optional = true +version = "0.11" + +[dependencies.virtio-bindings] +optional = true +version = "0.2.1" + [build-dependencies] prost-build = "0.12.1" tonic-build = "0.10.2" @@ -41,4 +74,17 @@ hex-literal = "0.4.1" [features] default = [] +fs = ["dep:libc", "dep:fuse-backend-rs"] +virtiofs = [ + "fs", + "dep:vhost", + "dep:vhost-user-backend", + "dep:virtio-queue", + "dep:vm-memory", + "dep:vmm-sys-util", + "dep:virtio-bindings", + "fuse-backend-rs?/vhost-user-fs", # impl FsCacheReqHandler for SlaveFsCacheReq + "fuse-backend-rs?/virtiofs", +] +fuse = ["fs"] tonic-reflection = ["dep:tonic-reflection"] diff --git a/tvix/store/src/fs/file_attr.rs b/tvix/castore/src/fs/file_attr.rs index ad41f036a253..ad41f036a253 100644 --- a/tvix/store/src/fs/file_attr.rs +++ b/tvix/castore/src/fs/file_attr.rs diff --git a/tvix/store/src/fs/fuse.rs b/tvix/castore/src/fs/fuse.rs index 98793bf47d58..98793bf47d58 100644 --- a/tvix/store/src/fs/fuse.rs +++ b/tvix/castore/src/fs/fuse.rs diff --git a/tvix/store/src/fs/inode_tracker.rs b/tvix/castore/src/fs/inode_tracker.rs index 3cabbbd247b5..4a8283b6b144 100644 --- a/tvix/store/src/fs/inode_tracker.rs +++ b/tvix/castore/src/fs/inode_tracker.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, sync::Arc}; use super::inodes::{DirectoryInodeData, InodeData}; -use tvix_castore::B3Digest; +use crate::B3Digest; /// InodeTracker keeps track of inodes, stores data being these inodes and deals /// with inode allocation. @@ -132,7 +132,7 @@ impl InodeTracker { #[cfg(test)] mod tests { - use crate::tests::fixtures; + use crate::fixtures; use super::InodeData; use super::InodeTracker; diff --git a/tvix/store/src/fs/inodes.rs b/tvix/castore/src/fs/inodes.rs index 4047199982b2..9131b703bae0 100644 --- a/tvix/store/src/fs/inodes.rs +++ b/tvix/castore/src/fs/inodes.rs @@ -1,7 +1,7 @@ //! This module contains all the data structures used to track information -//! about inodes, which present tvix-store nodes in a filesystem. -use tvix_castore::proto as castorepb; -use tvix_castore::B3Digest; +//! about inodes, which present tvix-castore nodes in a filesystem. +use crate::proto as castorepb; +use crate::B3Digest; #[derive(Clone, Debug)] pub enum InodeData { diff --git a/tvix/store/src/fs/mod.rs b/tvix/castore/src/fs/mod.rs index c11bd0a44c7e..9bd021cb09fe 100644 --- a/tvix/store/src/fs/mod.rs +++ b/tvix/castore/src/fs/mod.rs @@ -9,9 +9,13 @@ pub mod fuse; #[cfg(feature = "virtiofs")] pub mod virtiofs; -#[cfg(test)] -mod tests; - +use crate::proto as castorepb; +use crate::{ + blobservice::{BlobReader, BlobService}, + directoryservice::DirectoryService, + proto::{node::Node, NamedNode}, + B3Digest, +}; use fuse_backend_rs::abi::fuse_abi::stat64; use fuse_backend_rs::api::filesystem::{Context, FileSystem, FsOptions, ROOT_ID}; use futures::StreamExt; @@ -29,15 +33,8 @@ use tokio::{ sync::mpsc, }; use tracing::{debug, info_span, instrument, warn}; -use tvix_castore::proto as castorepb; -use tvix_castore::{ - blobservice::{BlobReader, BlobService}, - directoryservice::DirectoryService, - proto::{node::Node, NamedNode}, - B3Digest, -}; -use self::root_nodes::RootNodes; +pub use self::root_nodes::RootNodes; use self::{ file_attr::{gen_file_attr, ROOT_FILE_ATTR}, inode_tracker::InodeTracker, @@ -75,6 +72,7 @@ use self::{ /// merkle structure is a DAG, not a tree, this also means we can't do "bucketed /// allocation", aka reserve Directory.size inodes for each directory node we /// explore. +/// Tests for this live in the tvix-store crate. pub struct TvixStoreFs<BS, DS, RN> { blob_service: BS, directory_service: DS, diff --git a/tvix/castore/src/fs/root_nodes.rs b/tvix/castore/src/fs/root_nodes.rs new file mode 100644 index 000000000000..8d27b477ff69 --- /dev/null +++ b/tvix/castore/src/fs/root_nodes.rs @@ -0,0 +1,18 @@ +use std::pin::Pin; + +use crate::{proto::node::Node, Error}; +use futures::Stream; +use tonic::async_trait; + +/// Provides an interface for looking up root nodes in tvix-castore by given +/// a lookup key (usually the basename), and optionally allow a listing. +#[async_trait] +pub trait RootNodes: Send + Sync { + /// Looks up a root CA node based on the basename of the node in the root + /// directory of the filesystem. + async fn get_by_basename(&self, name: &[u8]) -> Result<Option<Node>, Error>; + + /// Lists all root CA nodes in the filesystem. An error can be returned + /// in case listing is not allowed + fn list(&self) -> Pin<Box<dyn Stream<Item = Result<Node, Error>> + Send>>; +} diff --git a/tvix/store/src/fs/virtiofs.rs b/tvix/castore/src/fs/virtiofs.rs index 846270d28568..846270d28568 100644 --- a/tvix/store/src/fs/virtiofs.rs +++ b/tvix/castore/src/fs/virtiofs.rs diff --git a/tvix/castore/src/lib.rs b/tvix/castore/src/lib.rs index 8d3dc7b4c4a5..8da0edef786b 100644 --- a/tvix/castore/src/lib.rs +++ b/tvix/castore/src/lib.rs @@ -4,6 +4,10 @@ mod errors; pub mod blobservice; pub mod directoryservice; pub mod fixtures; + +#[cfg(feature = "fs")] +pub mod fs; + pub mod import; pub mod proto; pub mod tonic; diff --git a/tvix/store/Cargo.toml b/tvix/store/Cargo.toml index 044b305b96ae..0a8e690e8250 100644 --- a/tvix/store/Cargo.toml +++ b/tvix/store/Cargo.toml @@ -14,7 +14,6 @@ data-encoding = "2.3.3" futures = "0.3.28" lazy_static = "1.4.0" nix-compat = { path = "../nix-compat", features = ["async"] } -parking_lot = "0.12.1" pin-project-lite = "0.2.13" prost = "0.12.1" sha2 = "0.10.6" @@ -35,42 +34,10 @@ async-recursion = "1.0.5" reqwest = { version = "0.11.22", features = ["rustls-tls", "stream"], default-features = false } xz2 = "0.1.7" -[dependencies.fuse-backend-rs] -optional = true -version = "0.11.0" - -[dependencies.vhost] -optional = true -version = "0.6" - -[dependencies.vhost-user-backend] -optional = true -version = "0.8" - -[dependencies.virtio-queue] -optional = true -version = "0.7" - -[dependencies.vm-memory] -optional = true -version = "0.10" - -[dependencies.vmm-sys-util] -optional = true -version = "0.11" - -[dependencies.virtio-bindings] -optional = true -version = "0.2.1" - [dependencies.tonic-reflection] optional = true version = "0.10.2" -[dependencies.libc] -optional = true -version = "0.2.144" - [build-dependencies] prost-build = "0.12.1" tonic-build = "0.10.2" @@ -80,19 +47,11 @@ test-case = "2.2.2" tempfile = "3.3.0" tokio-retry = "0.3.0" +[dev-dependencies.libc] +version = "0.2.144" + [features] default = ["fuse", "tonic-reflection"] -fs = ["dep:libc", "dep:fuse-backend-rs"] -virtiofs = [ - "fs", - "dep:vhost", - "dep:vhost-user-backend", - "dep:virtio-queue", - "dep:vm-memory", - "dep:vmm-sys-util", - "dep:virtio-bindings", - "fuse-backend-rs?/vhost-user-fs", # impl FsCacheReqHandler for SlaveFsCacheReq - "fuse-backend-rs?/virtiofs", -] -fuse = ["fs"] +fuse = ["tvix-castore/fuse"] +virtiofs = ["tvix-castore/virtiofs"] tonic-reflection = ["dep:tonic-reflection", "tvix-castore/tonic-reflection"] diff --git a/tvix/store/src/bin/tvix-store.rs b/tvix/store/src/bin/tvix-store.rs index e4f2e0801b81..bf59366eb34e 100644 --- a/tvix/store/src/bin/tvix-store.rs +++ b/tvix/store/src/bin/tvix-store.rs @@ -29,14 +29,14 @@ use tvix_store::proto::GRPCPathInfoServiceWrapper; use tvix_store::proto::NarInfo; use tvix_store::proto::PathInfo; -#[cfg(feature = "fs")] -use tvix_store::fs::TvixStoreFs; +#[cfg(any(feature = "fuse", feature = "virtiofs"))] +use tvix_store::pathinfoservice::make_fs; #[cfg(feature = "fuse")] -use tvix_store::fs::fuse::FuseDaemon; +use tvix_castore::fs::fuse::FuseDaemon; #[cfg(feature = "virtiofs")] -use tvix_store::fs::virtiofs::start_virtiofs_daemon; +use tvix_castore::fs::virtiofs::start_virtiofs_daemon; #[cfg(feature = "tonic-reflection")] use tvix_castore::proto::FILE_DESCRIPTOR_SET as CASTORE_FILE_DESCRIPTOR_SET; @@ -365,7 +365,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> { let path_info_service: Arc<dyn PathInfoService> = path_info_service.into(); let mut fuse_daemon = tokio::task::spawn_blocking(move || { - let f = TvixStoreFs::new( + let fs = make_fs( blob_service, directory_service, path_info_service, @@ -373,7 +373,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> { ); info!("mounting tvix-store on {:?}", &dest); - FuseDaemon::new(f, &dest, threads) + FuseDaemon::new(fs, &dest, threads) }) .await??; @@ -409,7 +409,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> { let path_info_service: Arc<dyn PathInfoService> = path_info_service.into(); tokio::task::spawn_blocking(move || { - let fs = TvixStoreFs::new( + let fs = make_fs( blob_service, directory_service, path_info_service, diff --git a/tvix/store/src/fs/root_nodes.rs b/tvix/store/src/fs/root_nodes.rs deleted file mode 100644 index e672c6e647f3..000000000000 --- a/tvix/store/src/fs/root_nodes.rs +++ /dev/null @@ -1,61 +0,0 @@ -use std::{ops::Deref, pin::Pin}; - -use futures::{Stream, StreamExt}; -use nix_compat::store_path::StorePath; -use tonic::async_trait; -use tvix_castore::{proto::node::Node, Error}; - -use crate::pathinfoservice::PathInfoService; - -/// Provides an interface for looking up root nodes in tvix-castore by given -/// a lookup key (usually the basename), and optionally allow a listing. -/// -#[async_trait] -pub trait RootNodes: Send + Sync { - /// Looks up a root CA node based on the basename of the node in the root - /// directory of the filesystem. - async fn get_by_basename(&self, name: &[u8]) -> Result<Option<Node>, Error>; - - /// Lists all root CA nodes in the filesystem. An error can be returned - /// in case listing is not allowed - fn list(&self) -> Pin<Box<dyn Stream<Item = Result<Node, Error>> + Send>>; -} - -/// Implements root node lookup for any [PathInfoService]. This represents a flat -/// directory structure like /nix/store where each entry in the root filesystem -/// directory corresponds to a CA node. -#[async_trait] -impl<T> RootNodes for T -where - T: Deref<Target = dyn PathInfoService> + Send + Sync, -{ - async fn get_by_basename(&self, name: &[u8]) -> Result<Option<Node>, Error> { - let Ok(store_path) = StorePath::from_bytes(name) else { - return Ok(None); - }; - - Ok(self - .deref() - .get(*store_path.digest()) - .await? - .map(|path_info| { - path_info - .node - .expect("missing root node") - .node - .expect("empty node") - })) - } - - fn list(&self) -> Pin<Box<dyn Stream<Item = Result<Node, Error>> + Send>> { - Box::pin(self.deref().list().map(|result| { - result.map(|path_info| { - path_info - .node - .expect("missing root node") - .node - .expect("empty node") - }) - })) - } -} diff --git a/tvix/store/src/lib.rs b/tvix/store/src/lib.rs index c59121453352..5b57781c4dab 100644 --- a/tvix/store/src/lib.rs +++ b/tvix/store/src/lib.rs @@ -1,6 +1,3 @@ -#[cfg(feature = "fs")] -pub mod fs; - pub mod nar; pub mod pathinfoservice; pub mod proto; diff --git a/tvix/store/src/pathinfoservice/fs/mod.rs b/tvix/store/src/pathinfoservice/fs/mod.rs new file mode 100644 index 000000000000..524aa10391f8 --- /dev/null +++ b/tvix/store/src/pathinfoservice/fs/mod.rs @@ -0,0 +1,84 @@ +use futures::Stream; +use futures::StreamExt; +use std::ops::Deref; +use std::pin::Pin; +use tonic::async_trait; +use tvix_castore::fs::{RootNodes, TvixStoreFs}; +use tvix_castore::proto as castorepb; +use tvix_castore::Error; +use tvix_castore::{blobservice::BlobService, directoryservice::DirectoryService}; + +use super::PathInfoService; + +#[cfg(test)] +mod tests; + +/// Helper to construct a [TvixStoreFs] from a [BlobService], [DirectoryService] +/// and [PathInfoService]. +/// This avoids users to have to interact with the wrapper struct directly, as +/// it leaks into the type signature of TvixStoreFS. +pub fn make_fs<BS, DS, PS>( + blob_service: BS, + directory_service: DS, + path_info_service: PS, + list_root: bool, +) -> TvixStoreFs<BS, DS, RootNodesWrapper<PS>> +where + BS: Deref<Target = dyn BlobService> + Send + Clone + 'static, + DS: Deref<Target = dyn DirectoryService> + Send + Clone + 'static, + PS: Deref<Target = dyn PathInfoService> + Send + Sync + Clone + 'static, +{ + TvixStoreFs::new( + blob_service, + directory_service, + RootNodesWrapper(path_info_service), + list_root, + ) +} + +/// Wrapper to satisfy Rust's orphan rules for trait implementations, as +/// RootNodes is coming from the [tvix-castore] crate. +#[doc(hidden)] +#[derive(Clone, Debug)] +pub struct RootNodesWrapper<T>(pub(crate) T); + +/// Implements root node lookup for any [PathInfoService]. This represents a flat +/// directory structure like /nix/store where each entry in the root filesystem +/// directory corresponds to a CA node. +#[cfg(any(feature = "fuse", feature = "virtiofs"))] +#[async_trait] +impl<T> RootNodes for RootNodesWrapper<T> +where + T: Deref<Target = dyn PathInfoService> + Send + Sync, +{ + async fn get_by_basename(&self, name: &[u8]) -> Result<Option<castorepb::node::Node>, Error> { + let Ok(store_path) = nix_compat::store_path::StorePath::from_bytes(name) else { + return Ok(None); + }; + + Ok(self + .0 + .deref() + .get(*store_path.digest()) + .await? + .map(|path_info| { + path_info + .node + .expect("missing root node") + .node + .expect("empty node") + })) + } + + fn list(&self) -> Pin<Box<dyn Stream<Item = Result<castorepb::node::Node, Error>> + Send>> { + Box::pin(self.0.deref().list().map(|result| { + result.map(|path_info| { + path_info + .node + .expect("missing root node") + .node + .expect("empty node") + }) + })) + } +} diff --git a/tvix/store/src/fs/tests.rs b/tvix/store/src/pathinfoservice/fs/tests.rs index a3977c727505..d12ee2f2a03e 100644 --- a/tvix/store/src/fs/tests.rs +++ b/tvix/store/src/pathinfoservice/fs/tests.rs @@ -1,22 +1,22 @@ +use crate::pathinfoservice::PathInfoService; +use crate::proto::PathInfo; +use crate::tests::fixtures; +use crate::tests::utils::{gen_blob_service, gen_directory_service, gen_pathinfo_service}; use futures::StreamExt; use std::io::Cursor; use std::os::unix::prelude::MetadataExt; use std::path::Path; use std::sync::Arc; +use tempfile::TempDir; use tokio::{fs, io}; use tokio_stream::wrappers::ReadDirStream; use tvix_castore::blobservice::BlobService; use tvix_castore::directoryservice::DirectoryService; - -use tempfile::TempDir; - -use crate::fs::{fuse::FuseDaemon, TvixStoreFs}; -use crate::pathinfoservice::PathInfoService; -use crate::proto::PathInfo; -use crate::tests::fixtures; -use crate::tests::utils::{gen_blob_service, gen_directory_service, gen_pathinfo_service}; +use tvix_castore::fs::fuse::FuseDaemon; use tvix_castore::proto as castorepb; +use super::make_fs; + const BLOB_A_NAME: &str = "00000000000000000000000000000000-test"; const BLOB_B_NAME: &str = "55555555555555555555555555555555-test"; const HELLOWORLD_BLOB_NAME: &str = "66666666666666666666666666666666-test"; @@ -44,7 +44,7 @@ fn do_mount<P: AsRef<Path>>( mountpoint: P, list_root: bool, ) -> io::Result<FuseDaemon> { - let fs = TvixStoreFs::new( + let fs = make_fs( blob_service, directory_service, path_info_service, diff --git a/tvix/store/src/pathinfoservice/mod.rs b/tvix/store/src/pathinfoservice/mod.rs index 5faa0900a0b0..3bd0ef206998 100644 --- a/tvix/store/src/pathinfoservice/mod.rs +++ b/tvix/store/src/pathinfoservice/mod.rs @@ -4,6 +4,9 @@ mod memory; mod nix_http; mod sled; +#[cfg(any(feature = "fuse", feature = "virtiofs"))] +mod fs; + use futures::Stream; use std::pin::Pin; use tonic::async_trait; @@ -18,6 +21,9 @@ pub use self::memory::MemoryPathInfoService; pub use self::nix_http::NixHTTPPathInfoService; pub use self::sled::SledPathInfoService; +#[cfg(any(feature = "fuse", feature = "virtiofs"))] +pub use self::fs::make_fs; + /// The base trait all PathInfo services need to implement. #[async_trait] pub trait PathInfoService: Send + Sync { |