feat(tvix): add experimental-store-composition option r/8400

Change-Id: I61661fbb0e77ce3c00c2a467dfabdf3fc77d8575
Reviewed-on: https://cl.tvl.fyi/c/depot/+/12011
Autosubmit: yuka <yuka@yuka.dev>
Tested-by: BuildkiteCI
Reviewed-by: flokli <flokli@flokli.de>

3 files changed, 31 insertions, 36 deletions

diff --git a/tvix/store/Cargo.toml b/tvix/store/Cargo.toml
index f1d88ba085ea..e9c8b1fc5356 100644
--- a/tvix/store/Cargo.toml
+++ b/tvix/store/Cargo.toml
@@ -43,6 +43,7 @@ tvix-tracing = { path = "../tracing", features = ["tonic", "reqwest"] }
 tracing = "0.1.40"
 tracing-indicatif = "0.3.6"
 hyper-util = "0.1.6"
+toml = { version = "0.8.15", optional = true }
 
 [dependencies.tonic-reflection]
 optional = true
@@ -74,6 +75,7 @@ otlp = ["tvix-tracing/otlp"]
 tonic-reflection = ["dep:tonic-reflection", "tvix-castore/tonic-reflection"]
 tracy = ["tvix-tracing/tracy"]
 virtiofs = ["tvix-castore/virtiofs"]
+xp-store-composition = []
 # Whether to run the integration tests.
 # Requires the following packages in $PATH:
 # cbtemulator, google-cloud-bigtable-tool
diff --git a/tvix/store/src/bin/tvix-store.rs b/tvix/store/src/bin/tvix-store.rs
index fd85b8bd36f3..5401f67b3f7e 100644
--- a/tvix/store/src/bin/tvix-store.rs
+++ b/tvix/store/src/bin/tvix-store.rs
@@ -70,12 +70,6 @@ enum Commands {
 
         #[clap(flatten)]
         service_addrs: ServiceUrls,
-
-        /// URL to a PathInfoService that's considered "remote".
-        /// If set, the other one is considered "local", and a "cache" for the
-        /// "remote" one.
-        #[arg(long, env)]
-        remote_path_info_service_addr: Option<String>,
     },
     /// Imports a list of paths into the store, print the store path for each of them.
     Import {
@@ -167,37 +161,10 @@ async fn run_cli(cli: Cli) -> Result<(), Box<dyn std::error::Error + Send + Sync
         Commands::Daemon {
             listen_args,
             service_addrs,
-            remote_path_info_service_addr,
         } => {
             // initialize stores
-            let mut configs = tvix_store::utils::addrs_to_configs(service_addrs)?;
-
-            // if remote_path_info_service_addr has been specified,
-            // update path_info_service to point to a cache combining the two.
-            if let Some(addr) = remote_path_info_service_addr {
-                use tvix_store::composition::{with_registry, DeserializeWithRegistry, REG};
-                use tvix_store::pathinfoservice::CachePathInfoServiceConfig;
-
-                let remote_url = url::Url::parse(&addr)?;
-                let remote_config = with_registry(&REG, || remote_url.try_into())?;
-
-                let local = configs.pathinfoservices.insert(
-                    "default".into(),
-                    DeserializeWithRegistry(Box::new(CachePathInfoServiceConfig {
-                        near: "local".into(),
-                        far: "remote".into(),
-                    })),
-                );
-                configs
-                    .pathinfoservices
-                    .insert("local".into(), local.unwrap());
-                configs
-                    .pathinfoservices
-                    .insert("remote".into(), remote_config);
-            }
-
             let (blob_service, directory_service, path_info_service, nar_calculation_service) =
-                tvix_store::utils::construct_services_from_configs(configs).await?;
+                tvix_store::utils::construct_services(service_addrs).await?;
 
             let mut server = Server::builder().layer(
                 ServiceBuilder::new()
diff --git a/tvix/store/src/utils.rs b/tvix/store/src/utils.rs
index 8f97651a03a5..4b07b35d694c 100644
--- a/tvix/store/src/utils.rs
+++ b/tvix/store/src/utils.rs
@@ -47,6 +47,13 @@ pub struct ServiceUrls {
 
     #[arg(long, env, default_value = "sled:///var/lib/tvix-store/pathinfo.sled")]
     path_info_service_addr: String,
+
+    /// Path to a TOML file describing the way the services should be composed
+    /// Experimental because the format is not final.
+    /// If specified, the other service addrs are ignored.
+    #[cfg(feature = "xp-store-composition")]
+    #[arg(long, env)]
+    experimental_store_composition: Option<String>,
 }
 
 /// like ServiceUrls, but with different clap defaults
@@ -60,6 +67,10 @@ pub struct ServiceUrlsGrpc {
 
     #[arg(long, env, default_value = "grpc+http://[::1]:8000")]
     path_info_service_addr: String,
+
+    #[cfg(feature = "xp-store-composition")]
+    #[arg(long, env)]
+    experimental_store_composition: Option<String>,
 }
 
 /// like ServiceUrls, but with different clap defaults
@@ -73,6 +84,10 @@ pub struct ServiceUrlsMemory {
 
     #[arg(long, env, default_value = "memory://")]
     path_info_service_addr: String,
+
+    #[cfg(feature = "xp-store-composition")]
+    #[arg(long, env)]
+    experimental_store_composition: Option<String>,
 }
 
 impl From<ServiceUrlsGrpc> for ServiceUrls {
@@ -81,6 +96,8 @@ impl From<ServiceUrlsGrpc> for ServiceUrls {
             blob_service_addr: urls.blob_service_addr,
             directory_service_addr: urls.directory_service_addr,
             path_info_service_addr: urls.path_info_service_addr,
+            #[cfg(feature = "xp-store-composition")]
+            experimental_store_composition: urls.experimental_store_composition,
         }
     }
 }
@@ -91,14 +108,23 @@ impl From<ServiceUrlsMemory> for ServiceUrls {
             blob_service_addr: urls.blob_service_addr,
             directory_service_addr: urls.directory_service_addr,
             path_info_service_addr: urls.path_info_service_addr,
+            #[cfg(feature = "xp-store-composition")]
+            experimental_store_composition: urls.experimental_store_composition,
         }
     }
 }
 
-pub fn addrs_to_configs(
+pub async fn addrs_to_configs(
     urls: impl Into<ServiceUrls>,
 ) -> Result<CompositionConfigs, Box<dyn std::error::Error + Send + Sync>> {
     let urls: ServiceUrls = urls.into();
+
+    #[cfg(feature = "xp-store-composition")]
+    if let Some(conf_path) = urls.experimental_store_composition {
+        let conf_text = tokio::fs::read_to_string(conf_path).await?;
+        return Ok(with_registry(&REG, || toml::from_str(&conf_text))?);
+    }
+
     let mut configs: CompositionConfigs = Default::default();
 
     let blob_service_url = Url::parse(&urls.blob_service_addr)?;
@@ -133,7 +159,7 @@ pub async fn construct_services(
     ),
     Box<dyn std::error::Error + Send + Sync>,
 > {
-    let configs = addrs_to_configs(urls)?;
+    let configs = addrs_to_configs(urls).await?;
     construct_services_from_configs(configs).await
 }