From 20513e7a52fc81f05cf31c183203e1929ee13464 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Wed, 17 Apr 2024 19:44:07 +0300 Subject: feat(tvix/store/bin): add progress bar infrastructure This adds the tracing-indicatif crate, and configures it as a layer in our tracing_subscriber pipeline to emit progress for every span that's configured so. It also moves from using std::io::stderr to write logs to using their writer, to avoid clobbering output. Progress bar styles are defined in a lazy_static, moving this into a general tracing is left for later. This adds some usage of this to the `imports` and `copy` commands. The output can still be improved a bit - we should probably split each task up into a smaller (instrumented) helper functions, so we can create a progress bar for each task. Change-Id: I59a1915aa4e0caa89c911632dec59c4cbeba1b89 Reviewed-on: https://cl.tvl.fyi/c/depot/+/11747 Reviewed-by: flokli Reviewed-by: Simon Hauser Tested-by: BuildkiteCI Autosubmit: flokli --- tvix/Cargo.lock | 95 ++++++++++++++ tvix/Cargo.nix | 270 ++++++++++++++++++++++++++++++++++++++- tvix/castore/Cargo.toml | 2 + tvix/castore/src/import/fs.rs | 15 ++- tvix/castore/src/import/mod.rs | 6 +- tvix/glue/Cargo.toml | 1 + tvix/store/Cargo.toml | 2 + tvix/store/src/bin/tvix-store.rs | 80 +++++++++--- 8 files changed, 452 insertions(+), 19 deletions(-) diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock index fa2c2c3dcc52..240ca7137d09 100644 --- a/tvix/Cargo.lock +++ b/tvix/Cargo.lock @@ -677,6 +677,19 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.52.0", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -1007,6 +1020,12 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.33" @@ -1664,6 +1683,20 @@ dependencies = [ "serde", ] +[[package]] +name = "indicatif" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", + "vt100", +] + [[package]] name = "instant" version = "0.1.12" @@ -2137,6 +2170,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.32.2" @@ -2476,6 +2515,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + [[package]] name = "powerfmt" version = "0.2.0" @@ -3980,6 +4025,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "tracing-indicatif" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069580424efe11d97c3fef4197fa98c004fa26672cc71ad8770d224e23b1951d" +dependencies = [ + "indicatif", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "tracing-log" version = "0.2.0" @@ -4073,6 +4130,7 @@ dependencies = [ "fuse-backend-rs", "futures", "hex-literal", + "indicatif", "lazy_static", "libc", "object_store", @@ -4099,6 +4157,7 @@ dependencies = [ "tonic-reflection", "tower", "tracing", + "tracing-indicatif", "url", "vhost", "vhost-user-backend", @@ -4210,6 +4269,7 @@ dependencies = [ "tokio-tar", "tokio-util", "tracing", + "tracing-indicatif", "tvix-build", "tvix-castore", "tvix-eval", @@ -4244,6 +4304,7 @@ dependencies = [ "count-write", "data-encoding", "futures", + "indicatif", "lazy_static", "lru", "nix-compat", @@ -4275,6 +4336,7 @@ dependencies = [ "tonic-reflection", "tower", "tracing", + "tracing-indicatif", "tracing-opentelemetry", "tracing-subscriber", "tvix-castore", @@ -4488,6 +4550,39 @@ dependencies = [ "libc", ] +[[package]] +name = "vt100" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84cd863bf0db7e392ba3bd04994be3473491b31e66340672af5d11943c6274de" +dependencies = [ + "itoa", + "log", + "unicode-width", + "vte", +] + +[[package]] +name = "vte" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" +dependencies = [ + "arrayvec", + "utf8parse", + "vte_generate_state_changes", +] + +[[package]] +name = "vte_generate_state_changes" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d257817081c7dffcdbab24b9e62d2def62e2ff7d00b1c20062551e6cccc145ff" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "wait-timeout" version = "0.2.0" diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix index c25ff06dc0ea..6f0c31f2c504 100644 --- a/tvix/Cargo.nix +++ b/tvix/Cargo.nix @@ -2126,6 +2126,47 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; + "console" = rec { + crateName = "console"; + version = "0.15.8"; + edition = "2018"; + sha256 = "1sz4nl9nz8pkmapqni6py7jxzi7nzqjxzb3ya4kxvmkb0zy867qf"; + authors = [ + "Armin Ronacher " + ]; + dependencies = [ + { + name = "encode_unicode"; + packageId = "encode_unicode"; + target = { target, features }: (target."windows" or false); + } + { + name = "lazy_static"; + packageId = "lazy_static"; + } + { + name = "libc"; + packageId = "libc"; + } + { + name = "unicode-width"; + packageId = "unicode-width"; + optional = true; + } + { + name = "windows-sys"; + packageId = "windows-sys 0.52.0"; + target = { target, features }: (target."windows" or false); + features = [ "Win32_Foundation" "Win32_System_Console" "Win32_Storage_FileSystem" "Win32_UI_Input_KeyboardAndMouse" ]; + } + ]; + features = { + "default" = [ "unicode-width" "ansi-parsing" ]; + "unicode-width" = [ "dep:unicode-width" ]; + "windows-console-colors" = [ "ansi-parsing" ]; + }; + resolvedDefaultFeatures = [ "ansi-parsing" "unicode-width" ]; + }; "const-oid" = rec { crateName = "const-oid"; version = "0.9.6"; @@ -3072,6 +3113,21 @@ rec { }; resolvedDefaultFeatures = [ "default" "use_std" ]; }; + "encode_unicode" = rec { + crateName = "encode_unicode"; + version = "0.3.6"; + edition = "2015"; + sha256 = "07w3vzrhxh9lpjgsg2y5bwzfar2aq35mdznvcp3zjl0ssj7d4mx3"; + authors = [ + "Torbjørn Birch Moltu " + ]; + features = { + "ascii" = [ "dep:ascii" ]; + "clippy" = [ "dep:clippy" ]; + "default" = [ "std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "encoding_rs" = rec { crateName = "encoding_rs"; version = "0.8.33"; @@ -5057,6 +5113,55 @@ rec { }; resolvedDefaultFeatures = [ "default" "serde" "std" ]; }; + "indicatif" = rec { + crateName = "indicatif"; + version = "0.17.8"; + edition = "2021"; + sha256 = "18xyqxw9i5x4sbpzckhfz3nm984iq9r7nbi2lk76nz888n7mlfkn"; + dependencies = [ + { + name = "console"; + packageId = "console"; + usesDefaultFeatures = false; + features = [ "ansi-parsing" ]; + } + { + name = "instant"; + packageId = "instant"; + target = { target, features }: ("wasm32" == target."arch" or null); + } + { + name = "number_prefix"; + packageId = "number_prefix"; + } + { + name = "portable-atomic"; + packageId = "portable-atomic"; + } + { + name = "unicode-width"; + packageId = "unicode-width"; + optional = true; + } + { + name = "vt100"; + packageId = "vt100"; + optional = true; + } + ]; + features = { + "default" = [ "unicode-width" "console/unicode-width" ]; + "futures" = [ "dep:futures-core" ]; + "improved_unicode" = [ "unicode-segmentation" "unicode-width" "console/unicode-width" ]; + "in_memory" = [ "vt100" ]; + "rayon" = [ "dep:rayon" ]; + "tokio" = [ "dep:tokio" ]; + "unicode-segmentation" = [ "dep:unicode-segmentation" ]; + "unicode-width" = [ "dep:unicode-width" ]; + "vt100" = [ "dep:vt100" ]; + }; + resolvedDefaultFeatures = [ "default" "in_memory" "unicode-width" "vt100" ]; + }; "instant" = rec { crateName = "instant"; version = "0.1.12"; @@ -6440,6 +6545,19 @@ rec { ]; }; + "number_prefix" = rec { + crateName = "number_prefix"; + version = "0.4.0"; + edition = "2015"; + sha256 = "1wvh13wvlajqxkb1filsfzbrnq0vrmrw298v2j3sy82z1rm282w3"; + authors = [ + "Benjamin Sago " + ]; + features = { + "default" = [ "std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "object" = rec { crateName = "object"; version = "0.32.2"; @@ -7535,6 +7653,18 @@ rec { ]; }; + "portable-atomic" = rec { + crateName = "portable-atomic"; + version = "1.6.0"; + edition = "2018"; + sha256 = "1h77x9qx7pns0d66vdrmdbmwpi7586h7ysnkdnhrn5mwi2cyyw3i"; + features = { + "critical-section" = [ "dep:critical-section" ]; + "default" = [ "fallback" ]; + "serde" = [ "dep:serde" ]; + }; + resolvedDefaultFeatures = [ "default" "fallback" ]; + }; "powerfmt" = rec { crateName = "powerfmt"; version = "0.2.0"; @@ -12414,6 +12544,33 @@ rec { }; resolvedDefaultFeatures = [ "default" "pin-project" "std" "std-future" ]; }; + "tracing-indicatif" = rec { + crateName = "tracing-indicatif"; + version = "0.3.6"; + edition = "2021"; + sha256 = "07cmn4ilw8hdfzc1mirccwkgl160k3x9fhgg7xydj4gy9r181586"; + libName = "tracing_indicatif"; + dependencies = [ + { + name = "indicatif"; + packageId = "indicatif"; + features = [ "in_memory" ]; + } + { + name = "tracing"; + packageId = "tracing"; + } + { + name = "tracing-core"; + packageId = "tracing-core"; + } + { + name = "tracing-subscriber"; + packageId = "tracing-subscriber"; + } + ]; + + }; "tracing-log" = rec { crateName = "tracing-log"; version = "0.2.0"; @@ -12828,6 +12985,10 @@ rec { name = "futures"; packageId = "futures"; } + { + name = "indicatif"; + packageId = "indicatif"; + } { name = "lazy_static"; packageId = "lazy_static"; @@ -12915,6 +13076,10 @@ rec { name = "tracing"; packageId = "tracing"; } + { + name = "tracing-indicatif"; + packageId = "tracing-indicatif"; + } { name = "url"; packageId = "url"; @@ -13386,6 +13551,10 @@ rec { name = "tracing"; packageId = "tracing"; } + { + name = "tracing-indicatif"; + packageId = "tracing-indicatif"; + } { name = "tvix-build"; packageId = "tvix-build"; @@ -13548,6 +13717,10 @@ rec { name = "futures"; packageId = "futures"; } + { + name = "indicatif"; + packageId = "indicatif"; + } { name = "lazy_static"; packageId = "lazy_static"; @@ -13662,6 +13835,10 @@ rec { name = "tracing"; packageId = "tracing"; } + { + name = "tracing-indicatif"; + packageId = "tracing-indicatif"; + } { name = "tracing-opentelemetry"; packageId = "tracing-opentelemetry"; @@ -14247,6 +14424,97 @@ rec { "with-serde" = [ "serde" "serde_derive" ]; }; }; + "vt100" = rec { + crateName = "vt100"; + version = "0.15.2"; + edition = "2021"; + sha256 = "1pklc8y984axmxr0cd363srr2d27wd5rj15xlcmkjznvy0xqdkc4"; + authors = [ + "Jesse Luehrs " + ]; + dependencies = [ + { + name = "itoa"; + packageId = "itoa"; + } + { + name = "log"; + packageId = "log"; + } + { + name = "unicode-width"; + packageId = "unicode-width"; + } + { + name = "vte"; + packageId = "vte"; + } + ]; + devDependencies = [ + { + name = "vte"; + packageId = "vte"; + } + ]; + + }; + "vte" = rec { + crateName = "vte"; + version = "0.11.1"; + edition = "2021"; + sha256 = "15r1ff4j8ndqj9vsyil3wqwxhhl7jsz5g58f31n0h1wlpxgjn0pm"; + authors = [ + "Joe Wilm " + "Christian Duerr " + ]; + dependencies = [ + { + name = "arrayvec"; + packageId = "arrayvec"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "utf8parse"; + packageId = "utf8parse"; + } + { + name = "vte_generate_state_changes"; + packageId = "vte_generate_state_changes"; + } + ]; + features = { + "ansi" = [ "log" ]; + "arrayvec" = [ "dep:arrayvec" ]; + "default" = [ "no_std" ]; + "log" = [ "dep:log" ]; + "nightly" = [ "utf8parse/nightly" ]; + "no_std" = [ "arrayvec" ]; + "serde" = [ "dep:serde" ]; + }; + resolvedDefaultFeatures = [ "arrayvec" "default" "no_std" ]; + }; + "vte_generate_state_changes" = rec { + crateName = "vte_generate_state_changes"; + version = "0.1.1"; + edition = "2018"; + sha256 = "1zs5q766q7jmc80c5c80gpzy4qpg5lnydf94mgdzrpy7h5q82myj"; + procMacro = true; + authors = [ + "Christian Duerr " + ]; + dependencies = [ + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + ]; + + }; "wait-timeout" = rec { crateName = "wait-timeout"; version = "0.2.0"; @@ -15725,7 +15993,7 @@ rec { "Win32_Web" = [ "Win32" ]; "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; }; - resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Win32" "Win32_Foundation" "Win32_NetworkManagement" "Win32_NetworkManagement_IpHelper" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Com" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_LibraryLoader" "Win32_System_Memory" "Win32_System_Threading" "Win32_System_WindowsProgramming" "Win32_UI" "Win32_UI_Shell" "default" ]; + resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Win32" "Win32_Foundation" "Win32_NetworkManagement" "Win32_NetworkManagement_IpHelper" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Com" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_LibraryLoader" "Win32_System_Memory" "Win32_System_Threading" "Win32_System_WindowsProgramming" "Win32_UI" "Win32_UI_Input" "Win32_UI_Input_KeyboardAndMouse" "Win32_UI_Shell" "default" ]; }; "windows-targets 0.48.5" = rec { crateName = "windows-targets"; diff --git a/tvix/castore/Cargo.toml b/tvix/castore/Cargo.toml index f80c5813c141..40d4f24d9f75 100644 --- a/tvix/castore/Cargo.toml +++ b/tvix/castore/Cargo.toml @@ -14,6 +14,7 @@ data-encoding = "2.3.3" digest = "0.10.7" fastcdc = { version = "3.1.0", features = ["tokio"] } futures = "0.3.30" +indicatif = "0.17.8" lazy_static = "1.4.0" object_store = { version = "0.9.1", features = ["http"] } parking_lot = "0.12.1" @@ -28,6 +29,7 @@ tokio = { version = "1.32.0", features = ["fs", "macros", "net", "rt", "rt-multi tonic = "0.11.0" tower = "0.4.13" tracing = "0.1.37" +tracing-indicatif = "0.3.6" url = "2.4.0" walkdir = "2.4.0" zstd = "0.13.0" diff --git a/tvix/castore/src/import/fs.rs b/tvix/castore/src/import/fs.rs index 9d3ecfe6ab7a..265d7723554b 100644 --- a/tvix/castore/src/import/fs.rs +++ b/tvix/castore/src/import/fs.rs @@ -7,6 +7,8 @@ use std::os::unix::ffi::OsStringExt; use std::os::unix::fs::MetadataExt; use std::os::unix::fs::PermissionsExt; use tracing::instrument; +use tracing::Span; +use tracing_indicatif::span_ext::IndicatifSpanExt; use walkdir::DirEntry; use walkdir::WalkDir; @@ -37,6 +39,7 @@ where BS: BlobService + Clone, DS: DirectoryService, { + Span::current().pb_start(); let iter = WalkDir::new(path.as_ref()) .follow_links(false) .follow_root_links(false) @@ -44,7 +47,17 @@ where .into_iter(); let entries = dir_entries_to_ingestion_stream(blob_service, iter, path.as_ref()); - ingest_entries(directory_service, entries).await + ingest_entries( + directory_service, + entries.inspect(|e| { + if let Ok(e) = e { + let s = Span::current(); + s.pb_inc(1); + s.pb_set_message(&format!("Ingesting {}", e.path())); + } + }), + ) + .await } /// Converts an iterator of [walkdir::DirEntry]s into a stream of ingestion entries. diff --git a/tvix/castore/src/import/mod.rs b/tvix/castore/src/import/mod.rs index 4223fe538756..c57c5bcada20 100644 --- a/tvix/castore/src/import/mod.rs +++ b/tvix/castore/src/import/mod.rs @@ -14,6 +14,8 @@ use crate::proto::FileNode; use crate::proto::SymlinkNode; use crate::B3Digest; use futures::{Stream, StreamExt}; +use tracing::Span; +use tracing_indicatif::span_ext::IndicatifSpanExt; use tracing::Level; @@ -44,7 +46,7 @@ pub mod fs; /// map and upload it to the [DirectoryService] through a lazily created [DirectoryPutter]. /// /// On success, returns the root node. -#[instrument(skip_all, ret(level = Level::TRACE), err)] +#[instrument(skip_all, fields(indicatif.pb_show=1), ret(level = Level::TRACE), err)] pub async fn ingest_entries( directory_service: DS, mut entries: S, @@ -58,6 +60,8 @@ where let mut directories: HashMap = HashMap::default(); let mut maybe_directory_putter: Option> = None; + Span::current().pb_start(); + let root_node = loop { let mut entry = entries .next() diff --git a/tvix/glue/Cargo.toml b/tvix/glue/Cargo.toml index 0afdefeaaa0e..e51f8ade895a 100644 --- a/tvix/glue/Cargo.toml +++ b/tvix/glue/Cargo.toml @@ -26,6 +26,7 @@ serde = "1.0.195" serde_json = "1.0" sha2 = "0.10.8" sha1 = "0.10.6" +tracing-indicatif = "0.3.6" md-5 = "0.10.6" url = "2.4.0" walkdir = "2.4.0" diff --git a/tvix/store/Cargo.toml b/tvix/store/Cargo.toml index 4727f43f78ff..28dad300fe20 100644 --- a/tvix/store/Cargo.toml +++ b/tvix/store/Cargo.toml @@ -43,6 +43,8 @@ walkdir = "2.4.0" reqwest = { version = "0.11.22", features = ["rustls-tls-native-roots", "stream"], default-features = false } lru = "0.12.3" parking_lot = "0.12.2" +indicatif = "0.17.8" +tracing-indicatif = "0.3.6" [dependencies.tonic-reflection] optional = true diff --git a/tvix/store/src/bin/tvix-store.rs b/tvix/store/src/bin/tvix-store.rs index 906d0ab5206a..2a2d6fe6f7d0 100644 --- a/tvix/store/src/bin/tvix-store.rs +++ b/tvix/store/src/bin/tvix-store.rs @@ -4,6 +4,7 @@ use clap::Subcommand; use futures::future::try_join_all; use futures::StreamExt; use futures::TryStreamExt; +use indicatif::ProgressStyle; use nix_compat::path_info::ExportedPathInfo; use serde::Deserialize; use serde::Serialize; @@ -14,10 +15,13 @@ use tokio_listener::SystemOptions; use tokio_listener::UserOptions; use tonic::transport::Server; use tracing::info; +use tracing::info_span; +use tracing::instrument; use tracing::Level; -use tracing_subscriber::EnvFilter; -use tracing_subscriber::Layer; -use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; +use tracing::Span; +use tracing_indicatif::filter::IndicatifFilter; +use tracing_indicatif::{span_ext::IndicatifSpanExt, IndicatifLayer}; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer}; use tvix_castore::import::fs::ingest_path; use tvix_store::nar::NarCalculationService; use tvix_store::proto::NarInfo; @@ -31,6 +35,20 @@ use tvix_store::pathinfoservice::PathInfoService; use tvix_store::proto::path_info_service_server::PathInfoServiceServer; use tvix_store::proto::GRPCPathInfoServiceWrapper; +use lazy_static::lazy_static; + +// FUTUREWORK: move this to tracing crate +lazy_static! { + pub static ref PB_PROGRESS_STYLE: ProgressStyle = ProgressStyle::with_template( + "{span_child_prefix}{bar:30} {wide_msg} [{elapsed_precise}] {pos:>7}/{len:7}" + ) + .expect("invalid progress template"); + pub static ref PB_SPINNER_STYLE: ProgressStyle = ProgressStyle::with_template( + "{span_child_prefix}{spinner} {wide_msg} [{elapsed_precise}] {pos:>7}/{len:7}" + ) + .expect("invalid progress template"); +} + #[cfg(any(feature = "fuse", feature = "virtiofs"))] use tvix_store::pathinfoservice::make_fs; @@ -212,24 +230,32 @@ fn default_threads() -> usize { } #[tokio::main] +#[instrument(fields(indicatif.pb_show=1))] async fn main() -> Result<(), Box> { let cli = Cli::parse(); // configure log settings let level = cli.log_level.unwrap_or(Level::INFO); + let indicatif_layer = IndicatifLayer::new().with_progress_style(PB_SPINNER_STYLE.clone()); + // Set up the tracing subscriber. - let subscriber = tracing_subscriber::registry().with( - tracing_subscriber::fmt::Layer::new() - .with_writer(std::io::stderr) - .compact() - .with_filter( - EnvFilter::builder() - .with_default_directive(level.into()) - .from_env() - .expect("invalid RUST_LOG"), - ), - ); + let subscriber = tracing_subscriber::registry() + .with( + tracing_subscriber::fmt::Layer::new() + .with_writer(indicatif_layer.get_stderr_writer()) + .compact() + .with_filter( + EnvFilter::builder() + .with_default_directive(level.into()) + .from_env() + .expect("invalid RUST_LOG"), + ), + ) + .with(indicatif_layer.with_filter( + // only show progress for spans with indicatif.pb_show field being set + IndicatifFilter::new(false), + )); // Add the otlp layer (when otlp is enabled, and it's not disabled in the CLI) // then init the registry. @@ -355,9 +381,19 @@ async fn main() -> Result<(), Box> { let nar_calculation_service: Arc = nar_calculation_service.into(); + let root_span = { + let s = Span::current(); + s.pb_set_style(&PB_PROGRESS_STYLE); + s.pb_set_message("Importing paths"); + s.pb_set_length(paths.len() as u64); + s.pb_start(); + s + }; + let tasks = paths .into_iter() .map(|path| { + let paths_span = root_span.clone(); tokio::task::spawn({ let blob_service = blob_service.clone(); let directory_service = directory_service.clone(); @@ -380,6 +416,7 @@ async fn main() -> Result<(), Box> { println!("{}", output_path.to_absolute_path()); } } + paths_span.pb_inc(1); } }) }) @@ -416,15 +453,26 @@ async fn main() -> Result<(), Box> { // Arc the PathInfoService, as we clone it . let path_info_service: Arc = path_info_service.into(); + let lookups_span = info_span!( + "lookup pathinfos", + "indicatif.pb_show" = tracing::field::Empty + ); + lookups_span.pb_set_length(reference_graph.closure.len() as u64); + lookups_span.pb_set_style(&PB_PROGRESS_STYLE); + lookups_span.pb_start(); + // From our reference graph, lookup all pathinfos that might exist. let elems: Vec<_> = futures::stream::iter(reference_graph.closure) .map(|elem| { let path_info_service = path_info_service.clone(); async move { - path_info_service + let resp = path_info_service .get(*elem.path.digest()) .await - .map(|resp| (elem, resp)) + .map(|resp| (elem, resp)); + + Span::current().pb_inc(1); + resp } }) .buffer_unordered(50) -- cgit 1.4.1