feat(tvix): add experimental-store-composition option r/8400

Change-Id: I61661fbb0e77ce3c00c2a467dfabdf3fc77d8575
Reviewed-on: https://cl.tvl.fyi/c/depot/+/12011
Autosubmit: yuka <yuka@yuka.dev>
Tested-by: BuildkiteCI
Reviewed-by: flokli <flokli@flokli.de>

5 files changed, 230 insertions, 52 deletions

diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock
index d41c678144d4..2c67116cd83c 100644
--- a/tvix/Cargo.lock
+++ b/tvix/Cargo.lock
@@ -3768,9 +3768,9 @@ dependencies = [
 
 [[package]]
 name = "serde_spanned"
-version = "0.6.5"
+version = "0.6.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1"
+checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0"
 dependencies = [
  "serde",
 ]
@@ -4377,8 +4377,20 @@ checksum = "4fb9d890e4dc9298b70f740f615f2e05b9db37dce531f6b24fb77ac993f9f217"
 dependencies = [
  "serde",
  "serde_spanned",
- "toml_datetime",
- "toml_edit",
+ "toml_datetime 0.5.1",
+ "toml_edit 0.18.1",
+]
+
+[[package]]
+name = "toml"
+version = "0.8.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac2caab0bf757388c6c0ae23b3293fdb463fee59434529014f85e3263b995c28"
+dependencies = [
+ "serde",
+ "serde_spanned",
+ "toml_datetime 0.6.6",
+ "toml_edit 0.22.16",
 ]
 
 [[package]]
@@ -4391,6 +4403,15 @@ dependencies = [
 ]
 
 [[package]]
+name = "toml_datetime"
+version = "0.6.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf"
+dependencies = [
+ "serde",
+]
+
+[[package]]
 name = "toml_edit"
 version = "0.18.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4400,7 +4421,20 @@ dependencies = [
  "nom8",
  "serde",
  "serde_spanned",
- "toml_datetime",
+ "toml_datetime 0.5.1",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.22.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "278f3d518e152219c994ce877758516bca5e118eaed6996192a774fb9fbf0788"
+dependencies = [
+ "indexmap 2.1.0",
+ "serde",
+ "serde_spanned",
+ "toml_datetime 0.6.6",
+ "winnow",
 ]
 
 [[package]]
@@ -4857,7 +4891,7 @@ dependencies = [
  "tempfile",
  "test-strategy",
  "tikv-jemallocator",
- "toml",
+ "toml 0.6.0",
  "tvix-eval-builtin-macros",
 ]
 
@@ -4964,6 +4998,7 @@ dependencies = [
  "tokio-retry",
  "tokio-stream",
  "tokio-util",
+ "toml 0.8.15",
  "tonic 0.12.1",
  "tonic-build 0.12.1",
  "tonic-reflection",
@@ -5612,6 +5647,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
 
 [[package]]
+name = "winnow"
+version = "0.6.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "374ec40a2d767a3c1b4972d9475ecd557356637be906f2cb3f7fe17a6eb5e22f"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
 name = "winreg"
 version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix
index bb2a1dd62979..640880d4910c 100644
--- a/tvix/Cargo.nix
+++ b/tvix/Cargo.nix
@@ -11982,9 +11982,9 @@ rec {
       };
       "serde_spanned" = rec {
         crateName = "serde_spanned";
-        version = "0.6.5";
+        version = "0.6.6";
         edition = "2021";
-        sha256 = "1hgh6s3jjwyzhfk3xwb6pnnr1misq9nflwq0f026jafi37s24dpb";
+        sha256 = "1839b6m5p9ijjmcwamiya2r612ks2vg6w2pp95yg76lr3zh79rkr";
         dependencies = [
           {
             name = "serde";
@@ -13889,7 +13889,7 @@ rec {
         };
         resolvedDefaultFeatures = [ "codec" "compat" "default" "futures-io" "io" "io-util" "net" "tracing" ];
       };
-      "toml" = rec {
+      "toml 0.6.0" = rec {
         crateName = "toml";
         version = "0.6.0";
         edition = "2021";
@@ -13909,12 +13909,12 @@ rec {
           }
           {
             name = "toml_datetime";
-            packageId = "toml_datetime";
+            packageId = "toml_datetime 0.5.1";
             features = [ "serde" ];
           }
           {
             name = "toml_edit";
-            packageId = "toml_edit";
+            packageId = "toml_edit 0.18.1";
             optional = true;
             features = [ "serde" ];
           }
@@ -13935,7 +13935,54 @@ rec {
         };
         resolvedDefaultFeatures = [ "default" "display" "parse" ];
       };
-      "toml_datetime" = rec {
+      "toml 0.8.15" = rec {
+        crateName = "toml";
+        version = "0.8.15";
+        edition = "2021";
+        sha256 = "0a2wk4xjdqw59w0jjia3b7p3yinv7wlv68xfq338hwvmpyqalb5c";
+        authors = [
+          "Alex Crichton <alex@alexcrichton.com>"
+        ];
+        dependencies = [
+          {
+            name = "serde";
+            packageId = "serde";
+          }
+          {
+            name = "serde_spanned";
+            packageId = "serde_spanned";
+            features = [ "serde" ];
+          }
+          {
+            name = "toml_datetime";
+            packageId = "toml_datetime 0.6.6";
+            features = [ "serde" ];
+          }
+          {
+            name = "toml_edit";
+            packageId = "toml_edit 0.22.16";
+            optional = true;
+            usesDefaultFeatures = false;
+            features = [ "serde" ];
+          }
+        ];
+        devDependencies = [
+          {
+            name = "serde";
+            packageId = "serde";
+            features = [ "derive" ];
+          }
+        ];
+        features = {
+          "default" = [ "parse" "display" ];
+          "display" = [ "dep:toml_edit" "toml_edit?/display" ];
+          "indexmap" = [ "dep:indexmap" ];
+          "parse" = [ "dep:toml_edit" "toml_edit?/parse" ];
+          "preserve_order" = [ "indexmap" ];
+        };
+        resolvedDefaultFeatures = [ "default" "display" "parse" ];
+      };
+      "toml_datetime 0.5.1" = rec {
         crateName = "toml_datetime";
         version = "0.5.1";
         edition = "2021";
@@ -13955,7 +14002,27 @@ rec {
         };
         resolvedDefaultFeatures = [ "serde" ];
       };
-      "toml_edit" = rec {
+      "toml_datetime 0.6.6" = rec {
+        crateName = "toml_datetime";
+        version = "0.6.6";
+        edition = "2021";
+        sha256 = "1grcrr3gh7id3cy3j700kczwwfbn04p5ncrrj369prjaj9bgvbab";
+        authors = [
+          "Alex Crichton <alex@alexcrichton.com>"
+        ];
+        dependencies = [
+          {
+            name = "serde";
+            packageId = "serde";
+            optional = true;
+          }
+        ];
+        features = {
+          "serde" = [ "dep:serde" ];
+        };
+        resolvedDefaultFeatures = [ "serde" ];
+      };
+      "toml_edit 0.18.1" = rec {
         crateName = "toml_edit";
         version = "0.18.1";
         edition = "2021";
@@ -13986,7 +14053,7 @@ rec {
           }
           {
             name = "toml_datetime";
-            packageId = "toml_datetime";
+            packageId = "toml_datetime 0.5.1";
           }
         ];
         features = {
@@ -13996,6 +14063,50 @@ rec {
         };
         resolvedDefaultFeatures = [ "default" "serde" ];
       };
+      "toml_edit 0.22.16" = rec {
+        crateName = "toml_edit";
+        version = "0.22.16";
+        edition = "2021";
+        sha256 = "1207pygznx57j9hrkmmfiq8mxjkba5c7g1yfjk4ij8hmir8kv3r7";
+        authors = [
+          "Andronik Ordian <write@reusable.software>"
+          "Ed Page <eopage@gmail.com>"
+        ];
+        dependencies = [
+          {
+            name = "indexmap";
+            packageId = "indexmap 2.1.0";
+            features = [ "std" ];
+          }
+          {
+            name = "serde";
+            packageId = "serde";
+            optional = true;
+          }
+          {
+            name = "serde_spanned";
+            packageId = "serde_spanned";
+            optional = true;
+            features = [ "serde" ];
+          }
+          {
+            name = "toml_datetime";
+            packageId = "toml_datetime 0.6.6";
+          }
+          {
+            name = "winnow";
+            packageId = "winnow";
+            optional = true;
+          }
+        ];
+        features = {
+          "default" = [ "parse" "display" ];
+          "parse" = [ "dep:winnow" ];
+          "perf" = [ "dep:kstring" ];
+          "serde" = [ "dep:serde" "toml_datetime/serde" "dep:serde_spanned" ];
+        };
+        resolvedDefaultFeatures = [ "display" "parse" "serde" ];
+      };
       "tonic 0.11.0" = rec {
         crateName = "tonic";
         version = "0.11.0";
@@ -15887,7 +15998,7 @@ rec {
           }
           {
             name = "toml";
-            packageId = "toml";
+            packageId = "toml 0.6.0";
           }
           {
             name = "tvix-eval-builtin-macros";
@@ -16307,6 +16418,11 @@ rec {
             features = [ "io" "io-util" "compat" ];
           }
           {
+            name = "toml";
+            packageId = "toml 0.8.15";
+            optional = true;
+          }
+          {
             name = "tonic";
             packageId = "tonic 0.12.1";
             features = [ "tls" "tls-roots" ];
@@ -16388,11 +16504,12 @@ rec {
           "default" = [ "cloud" "fuse" "otlp" "tonic-reflection" ];
           "fuse" = [ "tvix-castore/fuse" ];
           "otlp" = [ "tvix-tracing/otlp" ];
+          "toml" = [ "dep:toml" ];
           "tonic-reflection" = [ "dep:tonic-reflection" "tvix-castore/tonic-reflection" ];
           "tracy" = [ "tvix-tracing/tracy" ];
           "virtiofs" = [ "tvix-castore/virtiofs" ];
         };
-        resolvedDefaultFeatures = [ "cloud" "default" "fuse" "integration" "otlp" "tonic-reflection" "tracy" "virtiofs" ];
+        resolvedDefaultFeatures = [ "cloud" "default" "fuse" "integration" "otlp" "toml" "tonic-reflection" "tracy" "virtiofs" "xp-store-composition" ];
       };
       "tvix-tracing" = rec {
         crateName = "tvix-tracing";
@@ -19596,6 +19713,28 @@ rec {
         ];
 
       };
+      "winnow" = rec {
+        crateName = "winnow";
+        version = "0.6.14";
+        edition = "2021";
+        sha256 = "0bz2nmp7mqbz7z5z41p9gdimcwsmrmg4gnbj94dkqykn5l5c8kip";
+        dependencies = [
+          {
+            name = "memchr";
+            packageId = "memchr";
+            optional = true;
+            usesDefaultFeatures = false;
+          }
+        ];
+        features = {
+          "debug" = [ "std" "dep:anstream" "dep:anstyle" "dep:is-terminal" "dep:terminal_size" ];
+          "default" = [ "std" ];
+          "simd" = [ "dep:memchr" ];
+          "std" = [ "alloc" "memchr?/std" ];
+          "unstable-doc" = [ "alloc" "std" "simd" "unstable-recover" ];
+        };
+        resolvedDefaultFeatures = [ "alloc" "default" "std" ];
+      };
       "winreg" = rec {
         crateName = "winreg";
         version = "0.52.0";
diff --git a/tvix/store/Cargo.toml b/tvix/store/Cargo.toml
index f1d88ba085ea..e9c8b1fc5356 100644
--- a/tvix/store/Cargo.toml
+++ b/tvix/store/Cargo.toml
@@ -43,6 +43,7 @@ tvix-tracing = { path = "../tracing", features = ["tonic", "reqwest"] }
 tracing = "0.1.40"
 tracing-indicatif = "0.3.6"
 hyper-util = "0.1.6"
+toml = { version = "0.8.15", optional = true }
 
 [dependencies.tonic-reflection]
 optional = true
@@ -74,6 +75,7 @@ otlp = ["tvix-tracing/otlp"]
 tonic-reflection = ["dep:tonic-reflection", "tvix-castore/tonic-reflection"]
 tracy = ["tvix-tracing/tracy"]
 virtiofs = ["tvix-castore/virtiofs"]
+xp-store-composition = []
 # Whether to run the integration tests.
 # Requires the following packages in $PATH:
 # cbtemulator, google-cloud-bigtable-tool
diff --git a/tvix/store/src/bin/tvix-store.rs b/tvix/store/src/bin/tvix-store.rs
index fd85b8bd36f3..5401f67b3f7e 100644
--- a/tvix/store/src/bin/tvix-store.rs
+++ b/tvix/store/src/bin/tvix-store.rs
@@ -70,12 +70,6 @@ enum Commands {
 
         #[clap(flatten)]
         service_addrs: ServiceUrls,
-
-        /// URL to a PathInfoService that's considered "remote".
-        /// If set, the other one is considered "local", and a "cache" for the
-        /// "remote" one.
-        #[arg(long, env)]
-        remote_path_info_service_addr: Option<String>,
     },
     /// Imports a list of paths into the store, print the store path for each of them.
     Import {
@@ -167,37 +161,10 @@ async fn run_cli(cli: Cli) -> Result<(), Box<dyn std::error::Error + Send + Sync
         Commands::Daemon {
             listen_args,
             service_addrs,
-            remote_path_info_service_addr,
         } => {
             // initialize stores
-            let mut configs = tvix_store::utils::addrs_to_configs(service_addrs)?;
-
-            // if remote_path_info_service_addr has been specified,
-            // update path_info_service to point to a cache combining the two.
-            if let Some(addr) = remote_path_info_service_addr {
-                use tvix_store::composition::{with_registry, DeserializeWithRegistry, REG};
-                use tvix_store::pathinfoservice::CachePathInfoServiceConfig;
-
-                let remote_url = url::Url::parse(&addr)?;
-                let remote_config = with_registry(&REG, || remote_url.try_into())?;
-
-                let local = configs.pathinfoservices.insert(
-                    "default".into(),
-                    DeserializeWithRegistry(Box::new(CachePathInfoServiceConfig {
-                        near: "local".into(),
-                        far: "remote".into(),
-                    })),
-                );
-                configs
-                    .pathinfoservices
-                    .insert("local".into(), local.unwrap());
-                configs
-                    .pathinfoservices
-                    .insert("remote".into(), remote_config);
-            }
-
             let (blob_service, directory_service, path_info_service, nar_calculation_service) =
-                tvix_store::utils::construct_services_from_configs(configs).await?;
+                tvix_store::utils::construct_services(service_addrs).await?;
 
             let mut server = Server::builder().layer(
                 ServiceBuilder::new()
diff --git a/tvix/store/src/utils.rs b/tvix/store/src/utils.rs
index 8f97651a03a5..4b07b35d694c 100644
--- a/tvix/store/src/utils.rs
+++ b/tvix/store/src/utils.rs
@@ -47,6 +47,13 @@ pub struct ServiceUrls {
 
     #[arg(long, env, default_value = "sled:///var/lib/tvix-store/pathinfo.sled")]
     path_info_service_addr: String,
+
+    /// Path to a TOML file describing the way the services should be composed
+    /// Experimental because the format is not final.
+    /// If specified, the other service addrs are ignored.
+    #[cfg(feature = "xp-store-composition")]
+    #[arg(long, env)]
+    experimental_store_composition: Option<String>,
 }
 
 /// like ServiceUrls, but with different clap defaults
@@ -60,6 +67,10 @@ pub struct ServiceUrlsGrpc {
 
     #[arg(long, env, default_value = "grpc+http://[::1]:8000")]
     path_info_service_addr: String,
+
+    #[cfg(feature = "xp-store-composition")]
+    #[arg(long, env)]
+    experimental_store_composition: Option<String>,
 }
 
 /// like ServiceUrls, but with different clap defaults
@@ -73,6 +84,10 @@ pub struct ServiceUrlsMemory {
 
     #[arg(long, env, default_value = "memory://")]
     path_info_service_addr: String,
+
+    #[cfg(feature = "xp-store-composition")]
+    #[arg(long, env)]
+    experimental_store_composition: Option<String>,
 }
 
 impl From<ServiceUrlsGrpc> for ServiceUrls {
@@ -81,6 +96,8 @@ impl From<ServiceUrlsGrpc> for ServiceUrls {
             blob_service_addr: urls.blob_service_addr,
             directory_service_addr: urls.directory_service_addr,
             path_info_service_addr: urls.path_info_service_addr,
+            #[cfg(feature = "xp-store-composition")]
+            experimental_store_composition: urls.experimental_store_composition,
         }
     }
 }
@@ -91,14 +108,23 @@ impl From<ServiceUrlsMemory> for ServiceUrls {
             blob_service_addr: urls.blob_service_addr,
             directory_service_addr: urls.directory_service_addr,
             path_info_service_addr: urls.path_info_service_addr,
+            #[cfg(feature = "xp-store-composition")]
+            experimental_store_composition: urls.experimental_store_composition,
         }
     }
 }
 
-pub fn addrs_to_configs(
+pub async fn addrs_to_configs(
     urls: impl Into<ServiceUrls>,
 ) -> Result<CompositionConfigs, Box<dyn std::error::Error + Send + Sync>> {
     let urls: ServiceUrls = urls.into();
+
+    #[cfg(feature = "xp-store-composition")]
+    if let Some(conf_path) = urls.experimental_store_composition {
+        let conf_text = tokio::fs::read_to_string(conf_path).await?;
+        return Ok(with_registry(&REG, || toml::from_str(&conf_text))?);
+    }
+
     let mut configs: CompositionConfigs = Default::default();
 
     let blob_service_url = Url::parse(&urls.blob_service_addr)?;
@@ -133,7 +159,7 @@ pub async fn construct_services(
     ),
     Box<dyn std::error::Error + Send + Sync>,
 > {
-    let configs = addrs_to_configs(urls)?;
+    let configs = addrs_to_configs(urls).await?;
     construct_services_from_configs(configs).await
 }