diff options
| -rw-r--r-- | tvix/Cargo.lock | 56 | ||||
| -rw-r--r-- | tvix/Cargo.nix | 159 | ||||
| -rw-r--r-- | tvix/store/Cargo.toml | 2 | ||||
| -rw-r--r-- | tvix/store/src/bin/tvix-store.rs | 35 | ||||
| -rw-r--r-- | tvix/store/src/utils.rs | 30 |
5 files changed, 230 insertions, 52 deletions
diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock index d41c678144d4..2c67116cd83c 100644 --- a/tvix/Cargo.lock +++ b/tvix/Cargo.lock @@ -3768,9 +3768,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" dependencies = [ "serde", ] @@ -4377,8 +4377,20 @@ checksum = "4fb9d890e4dc9298b70f740f615f2e05b9db37dce531f6b24fb77ac993f9f217" dependencies = [ "serde", "serde_spanned", - "toml_datetime", - "toml_edit", + "toml_datetime 0.5.1", + "toml_edit 0.18.1", +] + +[[package]] +name = "toml" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2caab0bf757388c6c0ae23b3293fdb463fee59434529014f85e3263b995c28" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime 0.6.6", + "toml_edit 0.22.16", ] [[package]] @@ -4391,6 +4403,15 @@ dependencies = [ ] [[package]] +name = "toml_datetime" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +dependencies = [ + "serde", +] + +[[package]] name = "toml_edit" version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4400,7 +4421,20 @@ dependencies = [ "nom8", "serde", "serde_spanned", - "toml_datetime", + "toml_datetime 0.5.1", +] + +[[package]] +name = "toml_edit" +version = "0.22.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "278f3d518e152219c994ce877758516bca5e118eaed6996192a774fb9fbf0788" +dependencies = [ + "indexmap 2.1.0", + "serde", + "serde_spanned", + "toml_datetime 0.6.6", + "winnow", ] [[package]] @@ -4857,7 +4891,7 @@ dependencies = [ "tempfile", "test-strategy", "tikv-jemallocator", - "toml", + "toml 0.6.0", "tvix-eval-builtin-macros", ] @@ -4964,6 +4998,7 @@ dependencies = [ "tokio-retry", "tokio-stream", "tokio-util", + "toml 0.8.15", "tonic 0.12.1", "tonic-build 0.12.1", "tonic-reflection", @@ -5612,6 +5647,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] +name = "winnow" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "374ec40a2d767a3c1b4972d9475ecd557356637be906f2cb3f7fe17a6eb5e22f" +dependencies = [ + "memchr", +] + +[[package]] name = "winreg" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix index bb2a1dd62979..640880d4910c 100644 --- a/tvix/Cargo.nix +++ b/tvix/Cargo.nix @@ -11982,9 +11982,9 @@ rec { }; "serde_spanned" = rec { crateName = "serde_spanned"; - version = "0.6.5"; + version = "0.6.6"; edition = "2021"; - sha256 = "1hgh6s3jjwyzhfk3xwb6pnnr1misq9nflwq0f026jafi37s24dpb"; + sha256 = "1839b6m5p9ijjmcwamiya2r612ks2vg6w2pp95yg76lr3zh79rkr"; dependencies = [ { name = "serde"; @@ -13889,7 +13889,7 @@ rec { }; resolvedDefaultFeatures = [ "codec" "compat" "default" "futures-io" "io" "io-util" "net" "tracing" ]; }; - "toml" = rec { + "toml 0.6.0" = rec { crateName = "toml"; version = "0.6.0"; edition = "2021"; @@ -13909,12 +13909,12 @@ rec { } { name = "toml_datetime"; - packageId = "toml_datetime"; + packageId = "toml_datetime 0.5.1"; features = [ "serde" ]; } { name = "toml_edit"; - packageId = "toml_edit"; + packageId = "toml_edit 0.18.1"; optional = true; features = [ "serde" ]; } @@ -13935,7 +13935,54 @@ rec { }; resolvedDefaultFeatures = [ "default" "display" "parse" ]; }; - "toml_datetime" = rec { + "toml 0.8.15" = rec { + crateName = "toml"; + version = "0.8.15"; + edition = "2021"; + sha256 = "0a2wk4xjdqw59w0jjia3b7p3yinv7wlv68xfq338hwvmpyqalb5c"; + authors = [ + "Alex Crichton <alex@alexcrichton.com>" + ]; + dependencies = [ + { + name = "serde"; + packageId = "serde"; + } + { + name = "serde_spanned"; + packageId = "serde_spanned"; + features = [ "serde" ]; + } + { + name = "toml_datetime"; + packageId = "toml_datetime 0.6.6"; + features = [ "serde" ]; + } + { + name = "toml_edit"; + packageId = "toml_edit 0.22.16"; + optional = true; + usesDefaultFeatures = false; + features = [ "serde" ]; + } + ]; + devDependencies = [ + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + ]; + features = { + "default" = [ "parse" "display" ]; + "display" = [ "dep:toml_edit" "toml_edit?/display" ]; + "indexmap" = [ "dep:indexmap" ]; + "parse" = [ "dep:toml_edit" "toml_edit?/parse" ]; + "preserve_order" = [ "indexmap" ]; + }; + resolvedDefaultFeatures = [ "default" "display" "parse" ]; + }; + "toml_datetime 0.5.1" = rec { crateName = "toml_datetime"; version = "0.5.1"; edition = "2021"; @@ -13955,7 +14002,27 @@ rec { }; resolvedDefaultFeatures = [ "serde" ]; }; - "toml_edit" = rec { + "toml_datetime 0.6.6" = rec { + crateName = "toml_datetime"; + version = "0.6.6"; + edition = "2021"; + sha256 = "1grcrr3gh7id3cy3j700kczwwfbn04p5ncrrj369prjaj9bgvbab"; + authors = [ + "Alex Crichton <alex@alexcrichton.com>" + ]; + dependencies = [ + { + name = "serde"; + packageId = "serde"; + optional = true; + } + ]; + features = { + "serde" = [ "dep:serde" ]; + }; + resolvedDefaultFeatures = [ "serde" ]; + }; + "toml_edit 0.18.1" = rec { crateName = "toml_edit"; version = "0.18.1"; edition = "2021"; @@ -13986,7 +14053,7 @@ rec { } { name = "toml_datetime"; - packageId = "toml_datetime"; + packageId = "toml_datetime 0.5.1"; } ]; features = { @@ -13996,6 +14063,50 @@ rec { }; resolvedDefaultFeatures = [ "default" "serde" ]; }; + "toml_edit 0.22.16" = rec { + crateName = "toml_edit"; + version = "0.22.16"; + edition = "2021"; + sha256 = "1207pygznx57j9hrkmmfiq8mxjkba5c7g1yfjk4ij8hmir8kv3r7"; + authors = [ + "Andronik Ordian <write@reusable.software>" + "Ed Page <eopage@gmail.com>" + ]; + dependencies = [ + { + name = "indexmap"; + packageId = "indexmap 2.1.0"; + features = [ "std" ]; + } + { + name = "serde"; + packageId = "serde"; + optional = true; + } + { + name = "serde_spanned"; + packageId = "serde_spanned"; + optional = true; + features = [ "serde" ]; + } + { + name = "toml_datetime"; + packageId = "toml_datetime 0.6.6"; + } + { + name = "winnow"; + packageId = "winnow"; + optional = true; + } + ]; + features = { + "default" = [ "parse" "display" ]; + "parse" = [ "dep:winnow" ]; + "perf" = [ "dep:kstring" ]; + "serde" = [ "dep:serde" "toml_datetime/serde" "dep:serde_spanned" ]; + }; + resolvedDefaultFeatures = [ "display" "parse" "serde" ]; + }; "tonic 0.11.0" = rec { crateName = "tonic"; version = "0.11.0"; @@ -15887,7 +15998,7 @@ rec { } { name = "toml"; - packageId = "toml"; + packageId = "toml 0.6.0"; } { name = "tvix-eval-builtin-macros"; @@ -16307,6 +16418,11 @@ rec { features = [ "io" "io-util" "compat" ]; } { + name = "toml"; + packageId = "toml 0.8.15"; + optional = true; + } + { name = "tonic"; packageId = "tonic 0.12.1"; features = [ "tls" "tls-roots" ]; @@ -16388,11 +16504,12 @@ rec { "default" = [ "cloud" "fuse" "otlp" "tonic-reflection" ]; "fuse" = [ "tvix-castore/fuse" ]; "otlp" = [ "tvix-tracing/otlp" ]; + "toml" = [ "dep:toml" ]; "tonic-reflection" = [ "dep:tonic-reflection" "tvix-castore/tonic-reflection" ]; "tracy" = [ "tvix-tracing/tracy" ]; "virtiofs" = [ "tvix-castore/virtiofs" ]; }; - resolvedDefaultFeatures = [ "cloud" "default" "fuse" "integration" "otlp" "tonic-reflection" "tracy" "virtiofs" ]; + resolvedDefaultFeatures = [ "cloud" "default" "fuse" "integration" "otlp" "toml" "tonic-reflection" "tracy" "virtiofs" "xp-store-composition" ]; }; "tvix-tracing" = rec { crateName = "tvix-tracing"; @@ -19596,6 +19713,28 @@ rec { ]; }; + "winnow" = rec { + crateName = "winnow"; + version = "0.6.14"; + edition = "2021"; + sha256 = "0bz2nmp7mqbz7z5z41p9gdimcwsmrmg4gnbj94dkqykn5l5c8kip"; + dependencies = [ + { + name = "memchr"; + packageId = "memchr"; + optional = true; + usesDefaultFeatures = false; + } + ]; + features = { + "debug" = [ "std" "dep:anstream" "dep:anstyle" "dep:is-terminal" "dep:terminal_size" ]; + "default" = [ "std" ]; + "simd" = [ "dep:memchr" ]; + "std" = [ "alloc" "memchr?/std" ]; + "unstable-doc" = [ "alloc" "std" "simd" "unstable-recover" ]; + }; + resolvedDefaultFeatures = [ "alloc" "default" "std" ]; + }; "winreg" = rec { crateName = "winreg"; version = "0.52.0"; diff --git a/tvix/store/Cargo.toml b/tvix/store/Cargo.toml index f1d88ba085ea..e9c8b1fc5356 100644 --- a/tvix/store/Cargo.toml +++ b/tvix/store/Cargo.toml @@ -43,6 +43,7 @@ tvix-tracing = { path = "../tracing", features = ["tonic", "reqwest"] } tracing = "0.1.40" tracing-indicatif = "0.3.6" hyper-util = "0.1.6" +toml = { version = "0.8.15", optional = true } [dependencies.tonic-reflection] optional = true @@ -74,6 +75,7 @@ otlp = ["tvix-tracing/otlp"] tonic-reflection = ["dep:tonic-reflection", "tvix-castore/tonic-reflection"] tracy = ["tvix-tracing/tracy"] virtiofs = ["tvix-castore/virtiofs"] +xp-store-composition = [] # Whether to run the integration tests. # Requires the following packages in $PATH: # cbtemulator, google-cloud-bigtable-tool diff --git a/tvix/store/src/bin/tvix-store.rs b/tvix/store/src/bin/tvix-store.rs index fd85b8bd36f3..5401f67b3f7e 100644 --- a/tvix/store/src/bin/tvix-store.rs +++ b/tvix/store/src/bin/tvix-store.rs @@ -70,12 +70,6 @@ enum Commands { #[clap(flatten)] service_addrs: ServiceUrls, - - /// URL to a PathInfoService that's considered "remote". - /// If set, the other one is considered "local", and a "cache" for the - /// "remote" one. - #[arg(long, env)] - remote_path_info_service_addr: Option<String>, }, /// Imports a list of paths into the store, print the store path for each of them. Import { @@ -167,37 +161,10 @@ async fn run_cli(cli: Cli) -> Result<(), Box<dyn std::error::Error + Send + Sync Commands::Daemon { listen_args, service_addrs, - remote_path_info_service_addr, } => { // initialize stores - let mut configs = tvix_store::utils::addrs_to_configs(service_addrs)?; - - // if remote_path_info_service_addr has been specified, - // update path_info_service to point to a cache combining the two. - if let Some(addr) = remote_path_info_service_addr { - use tvix_store::composition::{with_registry, DeserializeWithRegistry, REG}; - use tvix_store::pathinfoservice::CachePathInfoServiceConfig; - - let remote_url = url::Url::parse(&addr)?; - let remote_config = with_registry(®, || remote_url.try_into())?; - - let local = configs.pathinfoservices.insert( - "default".into(), - DeserializeWithRegistry(Box::new(CachePathInfoServiceConfig { - near: "local".into(), - far: "remote".into(), - })), - ); - configs - .pathinfoservices - .insert("local".into(), local.unwrap()); - configs - .pathinfoservices - .insert("remote".into(), remote_config); - } - let (blob_service, directory_service, path_info_service, nar_calculation_service) = - tvix_store::utils::construct_services_from_configs(configs).await?; + tvix_store::utils::construct_services(service_addrs).await?; let mut server = Server::builder().layer( ServiceBuilder::new() diff --git a/tvix/store/src/utils.rs b/tvix/store/src/utils.rs index 8f97651a03a5..4b07b35d694c 100644 --- a/tvix/store/src/utils.rs +++ b/tvix/store/src/utils.rs @@ -47,6 +47,13 @@ pub struct ServiceUrls { #[arg(long, env, default_value = "sled:///var/lib/tvix-store/pathinfo.sled")] path_info_service_addr: String, + + /// Path to a TOML file describing the way the services should be composed + /// Experimental because the format is not final. + /// If specified, the other service addrs are ignored. + #[cfg(feature = "xp-store-composition")] + #[arg(long, env)] + experimental_store_composition: Option<String>, } /// like ServiceUrls, but with different clap defaults @@ -60,6 +67,10 @@ pub struct ServiceUrlsGrpc { #[arg(long, env, default_value = "grpc+http://[::1]:8000")] path_info_service_addr: String, + + #[cfg(feature = "xp-store-composition")] + #[arg(long, env)] + experimental_store_composition: Option<String>, } /// like ServiceUrls, but with different clap defaults @@ -73,6 +84,10 @@ pub struct ServiceUrlsMemory { #[arg(long, env, default_value = "memory://")] path_info_service_addr: String, + + #[cfg(feature = "xp-store-composition")] + #[arg(long, env)] + experimental_store_composition: Option<String>, } impl From<ServiceUrlsGrpc> for ServiceUrls { @@ -81,6 +96,8 @@ impl From<ServiceUrlsGrpc> for ServiceUrls { blob_service_addr: urls.blob_service_addr, directory_service_addr: urls.directory_service_addr, path_info_service_addr: urls.path_info_service_addr, + #[cfg(feature = "xp-store-composition")] + experimental_store_composition: urls.experimental_store_composition, } } } @@ -91,14 +108,23 @@ impl From<ServiceUrlsMemory> for ServiceUrls { blob_service_addr: urls.blob_service_addr, directory_service_addr: urls.directory_service_addr, path_info_service_addr: urls.path_info_service_addr, + #[cfg(feature = "xp-store-composition")] + experimental_store_composition: urls.experimental_store_composition, } } } -pub fn addrs_to_configs( +pub async fn addrs_to_configs( urls: impl Into<ServiceUrls>, ) -> Result<CompositionConfigs, Box<dyn std::error::Error + Send + Sync>> { let urls: ServiceUrls = urls.into(); + + #[cfg(feature = "xp-store-composition")] + if let Some(conf_path) = urls.experimental_store_composition { + let conf_text = tokio::fs::read_to_string(conf_path).await?; + return Ok(with_registry(®, || toml::from_str(&conf_text))?); + } + let mut configs: CompositionConfigs = Default::default(); let blob_service_url = Url::parse(&urls.blob_service_addr)?; @@ -133,7 +159,7 @@ pub async fn construct_services( ), Box<dyn std::error::Error + Send + Sync>, > { - let configs = addrs_to_configs(urls)?; + let configs = addrs_to_configs(urls).await?; construct_services_from_configs(configs).await } |