From 84a82f6f41b422f7f76e44e3705ee509b6c6eaf6 Mon Sep 17 00:00:00 2001 From: edef Date: Sat, 19 Oct 2024 19:23:56 +0000 Subject: feat(users/edef/weave): use tracing_indicatif for progress Progress bars :3 Change-Id: I770d0f8381521b6efc8b38c0db4d59c771887fee Reviewed-on: https://cl.tvl.fyi/c/depot/+/12673 Tested-by: BuildkiteCI Reviewed-by: flokli --- users/edef/weave/Cargo.lock | 235 +++++++++++- users/edef/weave/Cargo.nix | 688 +++++++++++++++++++++++++++++++++++- users/edef/weave/Cargo.toml | 3 + users/edef/weave/src/bin/swizzle.rs | 79 +++-- users/edef/weave/src/lib.rs | 10 +- users/edef/weave/src/main.rs | 172 +++++---- 6 files changed, 1072 insertions(+), 115 deletions(-) (limited to 'users') diff --git a/users/edef/weave/Cargo.lock b/users/edef/weave/Cargo.lock index fb7a02fb1340..734f12d4e699 100644 --- a/users/edef/weave/Cargo.lock +++ b/users/edef/weave/Cargo.lock @@ -99,6 +99,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf7d0a018de4f6aa429b9d33d69edf69072b1c5b1cb8d3e4a5f7ef898fc3eb76" +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "arrow-format" version = "0.8.1" @@ -233,7 +239,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" dependencies = [ "memchr", - "regex-automata", + "regex-automata 0.4.8", "serde", ] @@ -316,6 +322,19 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "console" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -510,6 +529,12 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "enum-primitive-derive" version = "0.3.0" @@ -780,6 +805,29 @@ dependencies = [ "hashbrown 0.15.0", ] +[[package]] +name = "indicatif" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", + "vt100", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "itoa" version = "1.0.11" @@ -804,6 +852,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.159" @@ -861,6 +915,15 @@ dependencies = [ "libc", ] +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + [[package]] name = "memchr" version = "2.7.4" @@ -996,6 +1059,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1006,6 +1079,12 @@ dependencies = [ "libm", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.36.5" @@ -1021,6 +1100,12 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "parking_lot" version = "0.12.3" @@ -1425,6 +1510,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "portable-atomic" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -1529,8 +1620,17 @@ checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -1541,9 +1641,15 @@ checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.5", ] +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.8.5" @@ -1653,6 +1759,15 @@ dependencies = [ "digest", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.3.0" @@ -1868,6 +1983,16 @@ dependencies = [ "syn 2.0.79", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "tokio" version = "1.40.0" @@ -1937,6 +2062,60 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-indicatif" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069580424efe11d97c3fef4197fa98c004fa26672cc71ad8770d224e23b1951d" +dependencies = [ + "indicatif", + "tracing", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "tvix-tracing" +version = "0.1.0" +dependencies = [ + "indicatif", + "thiserror", + "tokio", + "tracing", + "tracing-indicatif", + "tracing-subscriber", ] [[package]] @@ -1957,12 +2136,57 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vt100" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84cd863bf0db7e392ba3bd04994be3473491b31e66340672af5d11943c6274de" +dependencies = [ + "itoa", + "log", + "unicode-width", + "vte", +] + +[[package]] +name = "vte" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" +dependencies = [ + "arrayvec", + "utf8parse", + "vte_generate_state_changes", +] + +[[package]] +name = "vte_generate_state_changes" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2035,6 +2259,9 @@ dependencies = [ "rayon", "rustc-hash", "safer_owning_ref", + "tracing", + "tracing-indicatif", + "tvix-tracing", ] [[package]] diff --git a/users/edef/weave/Cargo.nix b/users/edef/weave/Cargo.nix index 0fd5eeed1724..b10bdfb7688c 100644 --- a/users/edef/weave/Cargo.nix +++ b/users/edef/weave/Cargo.nix @@ -329,6 +329,21 @@ rec { libName = "array_init_cursor"; }; + "arrayvec" = rec { + crateName = "arrayvec"; + version = "0.7.6"; + edition = "2018"; + sha256 = "0l1fz4ccgv6pm609rif37sl5nv5k6lbzi7kkppgzqzh1vwix20kw"; + authors = [ + "bluss" + ]; + features = { + "borsh" = [ "dep:borsh" ]; + "default" = [ "std" ]; + "serde" = [ "dep:serde" ]; + "zeroize" = [ "dep:zeroize" ]; + }; + }; "arrow-format" = rec { crateName = "arrow-format"; version = "0.8.1"; @@ -692,7 +707,7 @@ rec { } { name = "regex-automata"; - packageId = "regex-automata"; + packageId = "regex-automata 0.4.8"; optional = true; usesDefaultFeatures = false; features = [ "dfa-search" ]; @@ -953,6 +968,47 @@ rec { }; resolvedDefaultFeatures = [ "crossterm" "tty" ]; }; + "console" = rec { + crateName = "console"; + version = "0.15.8"; + edition = "2018"; + sha256 = "1sz4nl9nz8pkmapqni6py7jxzi7nzqjxzb3ya4kxvmkb0zy867qf"; + authors = [ + "Armin Ronacher " + ]; + dependencies = [ + { + name = "encode_unicode"; + packageId = "encode_unicode"; + target = { target, features }: (target."windows" or false); + } + { + name = "lazy_static"; + packageId = "lazy_static"; + } + { + name = "libc"; + packageId = "libc"; + } + { + name = "unicode-width"; + packageId = "unicode-width"; + optional = true; + } + { + name = "windows-sys"; + packageId = "windows-sys"; + target = { target, features }: (target."windows" or false); + features = [ "Win32_Foundation" "Win32_System_Console" "Win32_Storage_FileSystem" "Win32_UI_Input_KeyboardAndMouse" ]; + } + ]; + features = { + "default" = [ "unicode-width" "ansi-parsing" ]; + "unicode-width" = [ "dep:unicode-width" ]; + "windows-console-colors" = [ "ansi-parsing" ]; + }; + resolvedDefaultFeatures = [ "ansi-parsing" "unicode-width" ]; + }; "const-oid" = rec { crateName = "const-oid"; version = "0.9.6"; @@ -1531,6 +1587,21 @@ rec { }; resolvedDefaultFeatures = [ "default" "use_std" ]; }; + "encode_unicode" = rec { + crateName = "encode_unicode"; + version = "0.3.6"; + edition = "2015"; + sha256 = "07w3vzrhxh9lpjgsg2y5bwzfar2aq35mdznvcp3zjl0ssj7d4mx3"; + authors = [ + "Torbjørn Birch Moltu " + ]; + features = { + "ascii" = [ "dep:ascii" ]; + "clippy" = [ "dep:clippy" ]; + "default" = [ "std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "enum-primitive-derive" = rec { crateName = "enum-primitive-derive"; version = "0.3.0"; @@ -2307,6 +2378,77 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; + "indicatif" = rec { + crateName = "indicatif"; + version = "0.17.8"; + edition = "2021"; + sha256 = "18xyqxw9i5x4sbpzckhfz3nm984iq9r7nbi2lk76nz888n7mlfkn"; + dependencies = [ + { + name = "console"; + packageId = "console"; + usesDefaultFeatures = false; + features = [ "ansi-parsing" ]; + } + { + name = "instant"; + packageId = "instant"; + target = { target, features }: ("wasm32" == target."arch" or null); + } + { + name = "number_prefix"; + packageId = "number_prefix"; + } + { + name = "portable-atomic"; + packageId = "portable-atomic"; + } + { + name = "unicode-width"; + packageId = "unicode-width"; + optional = true; + } + { + name = "vt100"; + packageId = "vt100"; + optional = true; + } + ]; + features = { + "default" = [ "unicode-width" "console/unicode-width" ]; + "futures" = [ "dep:futures-core" ]; + "improved_unicode" = [ "unicode-segmentation" "unicode-width" "console/unicode-width" ]; + "in_memory" = [ "vt100" ]; + "rayon" = [ "dep:rayon" ]; + "tokio" = [ "dep:tokio" ]; + "unicode-segmentation" = [ "dep:unicode-segmentation" ]; + "unicode-width" = [ "dep:unicode-width" ]; + "vt100" = [ "dep:vt100" ]; + }; + resolvedDefaultFeatures = [ "default" "in_memory" "unicode-width" "vt100" ]; + }; + "instant" = rec { + crateName = "instant"; + version = "0.1.13"; + edition = "2018"; + sha256 = "08h27kzvb5jw74mh0ajv0nv9ggwvgqm8ynjsn2sa9jsks4cjh970"; + authors = [ + "sebcrozet " + ]; + dependencies = [ + { + name = "cfg-if"; + packageId = "cfg-if"; + } + ]; + features = { + "js-sys" = [ "dep:js-sys" ]; + "stdweb" = [ "dep:stdweb" ]; + "wasm-bindgen" = [ "js-sys" "wasm-bindgen_rs" "web-sys" ]; + "wasm-bindgen_rs" = [ "dep:wasm-bindgen_rs" ]; + "web-sys" = [ "dep:web-sys" ]; + }; + }; "itoa" = rec { crateName = "itoa"; version = "1.0.11"; @@ -2353,6 +2495,19 @@ rec { ]; }; + "lazy_static" = rec { + crateName = "lazy_static"; + version = "1.5.0"; + edition = "2015"; + sha256 = "1zk6dqqni0193xg6iijh7i3i44sryglwgvx20spdvwk3r6sbrlmv"; + authors = [ + "Marvin Löbel " + ]; + features = { + "spin" = [ "dep:spin" ]; + "spin_no_std" = [ "spin" ]; + }; + }; "libc" = rec { crateName = "libc"; version = "0.2.159"; @@ -2459,6 +2614,7 @@ rec { "sval_ref" = [ "dep:sval_ref" ]; "value-bag" = [ "dep:value-bag" ]; }; + resolvedDefaultFeatures = [ "std" ]; }; "lz4" = rec { crateName = "lz4"; @@ -2504,6 +2660,22 @@ rec { } ]; + }; + "matchers" = rec { + crateName = "matchers"; + version = "0.1.0"; + edition = "2018"; + sha256 = "0n2mbk7lg2vf962c8xwzdq96yrc9i0p8dbmm4wa1nnkcp1dhfqw2"; + authors = [ + "Eliza Weisman " + ]; + dependencies = [ + { + name = "regex-automata"; + packageId = "regex-automata 0.1.10"; + } + ]; + }; "memchr" = rec { crateName = "memchr"; @@ -2923,6 +3095,35 @@ rec { }; resolvedDefaultFeatures = [ "default" "user" ]; }; + "nu-ansi-term" = rec { + crateName = "nu-ansi-term"; + version = "0.46.0"; + edition = "2018"; + sha256 = "115sywxh53p190lyw97alm14nc004qj5jm5lvdj608z84rbida3p"; + libName = "nu_ansi_term"; + authors = [ + "ogham@bsago.me" + "Ryan Scheel (Havvy) " + "Josh Triplett " + "The Nushell Project Developers" + ]; + dependencies = [ + { + name = "overload"; + packageId = "overload"; + } + { + name = "winapi"; + packageId = "winapi"; + target = { target, features }: ("windows" == target."os" or null); + features = [ "consoleapi" "errhandlingapi" "fileapi" "handleapi" "processenv" ]; + } + ]; + features = { + "derive_serde_style" = [ "serde" ]; + "serde" = [ "dep:serde" ]; + }; + }; "num-traits" = rec { crateName = "num-traits"; version = "0.2.19"; @@ -2951,6 +3152,19 @@ rec { }; resolvedDefaultFeatures = [ "default" "libm" "std" ]; }; + "number_prefix" = rec { + crateName = "number_prefix"; + version = "0.4.0"; + edition = "2015"; + sha256 = "1wvh13wvlajqxkb1filsfzbrnq0vrmrw298v2j3sy82z1rm282w3"; + authors = [ + "Benjamin Sago " + ]; + features = { + "default" = [ "std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "object" = rec { crateName = "object"; version = "0.36.5"; @@ -3004,6 +3218,16 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "race" "std" ]; }; + "overload" = rec { + crateName = "overload"; + version = "0.1.1"; + edition = "2018"; + sha256 = "0fdgbaqwknillagy1xq7xfgv60qdbk010diwl7s1p0qx7hb16n5i"; + authors = [ + "Daniel Salvadori " + ]; + + }; "parking_lot" = rec { crateName = "parking_lot"; version = "0.12.3"; @@ -5024,6 +5248,19 @@ rec { }; resolvedDefaultFeatures = [ "sysinfo" ]; }; + "portable-atomic" = rec { + crateName = "portable-atomic"; + version = "1.9.0"; + edition = "2018"; + sha256 = "1cmd87qj90panwsi350djb8lsxdryqkkxmimjcz7a1nsysini76c"; + libName = "portable_atomic"; + features = { + "critical-section" = [ "dep:critical-section" ]; + "default" = [ "fallback" ]; + "serde" = [ "dep:serde" ]; + }; + resolvedDefaultFeatures = [ "default" "fallback" ]; + }; "ppv-lite86" = rec { crateName = "ppv-lite86"; version = "0.2.20"; @@ -5319,13 +5556,13 @@ rec { } { name = "regex-automata"; - packageId = "regex-automata"; + packageId = "regex-automata 0.4.8"; usesDefaultFeatures = false; features = [ "alloc" "syntax" "meta" "nfa-pikevm" ]; } { name = "regex-syntax"; - packageId = "regex-syntax"; + packageId = "regex-syntax 0.8.5"; usesDefaultFeatures = false; } ]; @@ -5353,7 +5590,32 @@ rec { }; resolvedDefaultFeatures = [ "default" "perf" "perf-backtrack" "perf-cache" "perf-dfa" "perf-inline" "perf-literal" "perf-onepass" "std" "unicode" "unicode-age" "unicode-bool" "unicode-case" "unicode-gencat" "unicode-perl" "unicode-script" "unicode-segment" ]; }; - "regex-automata" = rec { + "regex-automata 0.1.10" = rec { + crateName = "regex-automata"; + version = "0.1.10"; + edition = "2015"; + sha256 = "0ci1hvbzhrfby5fdpf4ganhf7kla58acad9i1ff1p34dzdrhs8vc"; + libName = "regex_automata"; + authors = [ + "Andrew Gallant " + ]; + dependencies = [ + { + name = "regex-syntax"; + packageId = "regex-syntax 0.6.29"; + optional = true; + } + ]; + features = { + "default" = [ "std" ]; + "fst" = [ "dep:fst" ]; + "regex-syntax" = [ "dep:regex-syntax" ]; + "std" = [ "regex-syntax" ]; + "transducer" = [ "std" "fst" ]; + }; + resolvedDefaultFeatures = [ "default" "regex-syntax" "std" ]; + }; + "regex-automata 0.4.8" = rec { crateName = "regex-automata"; version = "0.4.8"; edition = "2021"; @@ -5378,7 +5640,7 @@ rec { } { name = "regex-syntax"; - packageId = "regex-syntax"; + packageId = "regex-syntax 0.8.5"; optional = true; usesDefaultFeatures = false; } @@ -5414,7 +5676,22 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "dfa-onepass" "dfa-search" "hybrid" "meta" "nfa-backtrack" "nfa-pikevm" "nfa-thompson" "perf-inline" "perf-literal" "perf-literal-multisubstring" "perf-literal-substring" "std" "syntax" "unicode" "unicode-age" "unicode-bool" "unicode-case" "unicode-gencat" "unicode-perl" "unicode-script" "unicode-segment" "unicode-word-boundary" ]; }; - "regex-syntax" = rec { + "regex-syntax 0.6.29" = rec { + crateName = "regex-syntax"; + version = "0.6.29"; + edition = "2018"; + sha256 = "1qgj49vm6y3zn1hi09x91jvgkl2b1fiaq402skj83280ggfwcqpi"; + libName = "regex_syntax"; + authors = [ + "The Rust Project Developers" + ]; + features = { + "default" = [ "unicode" ]; + "unicode" = [ "unicode-age" "unicode-bool" "unicode-case" "unicode-gencat" "unicode-perl" "unicode-script" "unicode-segment" ]; + }; + resolvedDefaultFeatures = [ "default" "unicode" "unicode-age" "unicode-bool" "unicode-case" "unicode-gencat" "unicode-perl" "unicode-script" "unicode-segment" ]; + }; + "regex-syntax 0.8.5" = rec { crateName = "regex-syntax"; version = "0.8.5"; edition = "2021"; @@ -5706,6 +5983,25 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; + "sharded-slab" = rec { + crateName = "sharded-slab"; + version = "0.1.7"; + edition = "2018"; + sha256 = "1xipjr4nqsgw34k7a2cgj9zaasl2ds6jwn89886kww93d32a637l"; + libName = "sharded_slab"; + authors = [ + "Eliza Weisman " + ]; + dependencies = [ + { + name = "lazy_static"; + packageId = "lazy_static"; + } + ]; + features = { + "loom" = [ "dep:loom" ]; + }; + }; "shlex" = rec { crateName = "shlex"; version = "1.3.0"; @@ -6248,6 +6544,26 @@ rec { ]; }; + "thread_local" = rec { + crateName = "thread_local"; + version = "1.1.8"; + edition = "2021"; + sha256 = "173i5lyjh011gsimk21np9jn8al18rxsrkjli20a7b8ks2xgk7lb"; + authors = [ + "Amanieu d'Antras " + ]; + dependencies = [ + { + name = "cfg-if"; + packageId = "cfg-if"; + } + { + name = "once_cell"; + packageId = "once_cell"; + } + ]; + features = { }; + }; "tokio" = rec { crateName = "tokio"; version = "1.40.0"; @@ -6457,7 +6773,7 @@ rec { "tracing-attributes" = [ "dep:tracing-attributes" ]; "valuable" = [ "tracing-core/valuable" ]; }; - resolvedDefaultFeatures = [ "attributes" "default" "std" "tracing-attributes" ]; + resolvedDefaultFeatures = [ "attributes" "default" "max_level_trace" "release_max_level_debug" "std" "tracing-attributes" ]; }; "tracing-attributes" = rec { crateName = "tracing-attributes"; @@ -6504,6 +6820,13 @@ rec { packageId = "once_cell"; optional = true; } + { + name = "valuable"; + packageId = "valuable"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (target."tracing_unstable" or false); + } ]; features = { "default" = [ "std" "valuable/std" ]; @@ -6511,7 +6834,224 @@ rec { "std" = [ "once_cell" ]; "valuable" = [ "dep:valuable" ]; }; - resolvedDefaultFeatures = [ "once_cell" "std" ]; + resolvedDefaultFeatures = [ "default" "once_cell" "std" "valuable" ]; + }; + "tracing-indicatif" = rec { + crateName = "tracing-indicatif"; + version = "0.3.6"; + edition = "2021"; + sha256 = "07cmn4ilw8hdfzc1mirccwkgl160k3x9fhgg7xydj4gy9r181586"; + libName = "tracing_indicatif"; + dependencies = [ + { + name = "indicatif"; + packageId = "indicatif"; + features = [ "in_memory" ]; + } + { + name = "tracing"; + packageId = "tracing"; + } + { + name = "tracing-core"; + packageId = "tracing-core"; + } + { + name = "tracing-subscriber"; + packageId = "tracing-subscriber"; + } + ]; + + }; + "tracing-log" = rec { + crateName = "tracing-log"; + version = "0.2.0"; + edition = "2018"; + sha256 = "1hs77z026k730ij1a9dhahzrl0s073gfa2hm5p0fbl0b80gmz1gf"; + libName = "tracing_log"; + authors = [ + "Tokio Contributors " + ]; + dependencies = [ + { + name = "log"; + packageId = "log"; + } + { + name = "once_cell"; + packageId = "once_cell"; + } + { + name = "tracing-core"; + packageId = "tracing-core"; + } + ]; + features = { + "ahash" = [ "dep:ahash" ]; + "default" = [ "log-tracer" "std" ]; + "interest-cache" = [ "lru" "ahash" ]; + "lru" = [ "dep:lru" ]; + "std" = [ "log/std" ]; + }; + resolvedDefaultFeatures = [ "log-tracer" "std" ]; + }; + "tracing-subscriber" = rec { + crateName = "tracing-subscriber"; + version = "0.3.18"; + edition = "2018"; + sha256 = "12vs1bwk4kig1l2qqjbbn2nm5amwiqmkcmnznylzmnfvjy6083xd"; + libName = "tracing_subscriber"; + authors = [ + "Eliza Weisman " + "David Barsky " + "Tokio Contributors " + ]; + dependencies = [ + { + name = "matchers"; + packageId = "matchers"; + optional = true; + } + { + name = "nu-ansi-term"; + packageId = "nu-ansi-term"; + optional = true; + } + { + name = "once_cell"; + packageId = "once_cell"; + optional = true; + } + { + name = "regex"; + packageId = "regex"; + optional = true; + usesDefaultFeatures = false; + features = [ "std" "unicode-case" "unicode-perl" ]; + } + { + name = "sharded-slab"; + packageId = "sharded-slab"; + optional = true; + } + { + name = "smallvec"; + packageId = "smallvec"; + optional = true; + } + { + name = "thread_local"; + packageId = "thread_local"; + optional = true; + } + { + name = "tracing"; + packageId = "tracing"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "tracing-core"; + packageId = "tracing-core"; + usesDefaultFeatures = false; + } + { + name = "tracing-log"; + packageId = "tracing-log"; + optional = true; + usesDefaultFeatures = false; + features = [ "log-tracer" "std" ]; + } + ]; + devDependencies = [ + { + name = "regex"; + packageId = "regex"; + usesDefaultFeatures = false; + features = [ "std" ]; + } + { + name = "tracing"; + packageId = "tracing"; + } + { + name = "tracing-log"; + packageId = "tracing-log"; + } + ]; + features = { + "ansi" = [ "fmt" "nu-ansi-term" ]; + "chrono" = [ "dep:chrono" ]; + "default" = [ "smallvec" "fmt" "ansi" "tracing-log" "std" ]; + "env-filter" = [ "matchers" "regex" "once_cell" "tracing" "std" "thread_local" ]; + "fmt" = [ "registry" "std" ]; + "json" = [ "tracing-serde" "serde" "serde_json" ]; + "local-time" = [ "time/local-offset" ]; + "matchers" = [ "dep:matchers" ]; + "nu-ansi-term" = [ "dep:nu-ansi-term" ]; + "once_cell" = [ "dep:once_cell" ]; + "parking_lot" = [ "dep:parking_lot" ]; + "regex" = [ "dep:regex" ]; + "registry" = [ "sharded-slab" "thread_local" "std" ]; + "serde" = [ "dep:serde" ]; + "serde_json" = [ "dep:serde_json" ]; + "sharded-slab" = [ "dep:sharded-slab" ]; + "smallvec" = [ "dep:smallvec" ]; + "std" = [ "alloc" "tracing-core/std" ]; + "thread_local" = [ "dep:thread_local" ]; + "time" = [ "dep:time" ]; + "tracing" = [ "dep:tracing" ]; + "tracing-log" = [ "dep:tracing-log" ]; + "tracing-serde" = [ "dep:tracing-serde" ]; + "valuable" = [ "tracing-core/valuable" "valuable_crate" "valuable-serde" "tracing-serde/valuable" ]; + "valuable-serde" = [ "dep:valuable-serde" ]; + "valuable_crate" = [ "dep:valuable_crate" ]; + }; + resolvedDefaultFeatures = [ "alloc" "ansi" "default" "env-filter" "fmt" "matchers" "nu-ansi-term" "once_cell" "regex" "registry" "sharded-slab" "smallvec" "std" "thread_local" "tracing" "tracing-log" ]; + }; + "tvix-tracing" = rec { + crateName = "tvix-tracing"; + version = "0.1.0"; + edition = "2021"; + src = lib.cleanSourceWith { filter = sourceFilter; src = ../../../tvix/tracing; }; + libName = "tvix_tracing"; + dependencies = [ + { + name = "indicatif"; + packageId = "indicatif"; + } + { + name = "thiserror"; + packageId = "thiserror"; + } + { + name = "tokio"; + packageId = "tokio"; + features = [ "sync" "rt" ]; + } + { + name = "tracing"; + packageId = "tracing"; + features = [ "max_level_trace" "release_max_level_debug" ]; + } + { + name = "tracing-indicatif"; + packageId = "tracing-indicatif"; + } + { + name = "tracing-subscriber"; + packageId = "tracing-subscriber"; + features = [ "env-filter" ]; + } + ]; + features = { + "axum" = [ "dep:axum" ]; + "otlp" = [ "dep:tracing-opentelemetry" "dep:opentelemetry" "dep:opentelemetry-otlp" "dep:opentelemetry_sdk" "dep:opentelemetry-http" "reqwest-tracing?/opentelemetry_0_22" ]; + "reqwest" = [ "dep:reqwest-tracing" ]; + "tonic" = [ "dep:tonic" "dep:http" ]; + "tracy" = [ "dep:tracing-tracy" ]; + }; + resolvedDefaultFeatures = [ "default" ]; }; "typenum" = rec { crateName = "typenum"; @@ -6558,6 +7098,31 @@ rec { }; resolvedDefaultFeatures = [ "cjk" "default" ]; }; + "utf8parse" = rec { + crateName = "utf8parse"; + version = "0.2.2"; + edition = "2018"; + sha256 = "088807qwjq46azicqwbhlmzwrbkz7l4hpw43sdkdyyk524vdxaq6"; + authors = [ + "Joe Wilm " + "Christian Duerr " + ]; + features = { }; + resolvedDefaultFeatures = [ "default" ]; + }; + "valuable" = rec { + crateName = "valuable"; + version = "0.1.0"; + edition = "2018"; + sha256 = "0v9gp3nkjbl30z0fd56d8mx7w1csk86wwjhfjhr400wh9mfpw2w3"; + features = { + "default" = [ "std" ]; + "derive" = [ "valuable-derive" ]; + "std" = [ "alloc" ]; + "valuable-derive" = [ "dep:valuable-derive" ]; + }; + resolvedDefaultFeatures = [ "alloc" "std" ]; + }; "version_check" = rec { crateName = "version_check"; version = "0.9.5"; @@ -6567,6 +7132,97 @@ rec { "Sergio Benitez " ]; + }; + "vt100" = rec { + crateName = "vt100"; + version = "0.15.2"; + edition = "2021"; + sha256 = "1pklc8y984axmxr0cd363srr2d27wd5rj15xlcmkjznvy0xqdkc4"; + authors = [ + "Jesse Luehrs " + ]; + dependencies = [ + { + name = "itoa"; + packageId = "itoa"; + } + { + name = "log"; + packageId = "log"; + } + { + name = "unicode-width"; + packageId = "unicode-width"; + } + { + name = "vte"; + packageId = "vte"; + } + ]; + devDependencies = [ + { + name = "vte"; + packageId = "vte"; + } + ]; + + }; + "vte" = rec { + crateName = "vte"; + version = "0.11.1"; + edition = "2021"; + sha256 = "15r1ff4j8ndqj9vsyil3wqwxhhl7jsz5g58f31n0h1wlpxgjn0pm"; + authors = [ + "Joe Wilm " + "Christian Duerr " + ]; + dependencies = [ + { + name = "arrayvec"; + packageId = "arrayvec"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "utf8parse"; + packageId = "utf8parse"; + } + { + name = "vte_generate_state_changes"; + packageId = "vte_generate_state_changes"; + } + ]; + features = { + "ansi" = [ "log" ]; + "arrayvec" = [ "dep:arrayvec" ]; + "default" = [ "no_std" ]; + "log" = [ "dep:log" ]; + "nightly" = [ "utf8parse/nightly" ]; + "no_std" = [ "arrayvec" ]; + "serde" = [ "dep:serde" ]; + }; + resolvedDefaultFeatures = [ "arrayvec" "default" "no_std" ]; + }; + "vte_generate_state_changes" = rec { + crateName = "vte_generate_state_changes"; + version = "0.1.2"; + edition = "2018"; + sha256 = "0biwgpcji3w4llz7h4bi8c2rwqchm9gmyr7dnjki1m853gp9ndif"; + procMacro = true; + authors = [ + "Christian Duerr " + ]; + dependencies = [ + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + ]; + }; "wasi" = rec { crateName = "wasi"; @@ -6789,6 +7445,18 @@ rec { name = "safer_owning_ref"; packageId = "safer_owning_ref"; } + { + name = "tracing"; + packageId = "tracing"; + } + { + name = "tracing-indicatif"; + packageId = "tracing-indicatif"; + } + { + name = "tvix-tracing"; + packageId = "tvix-tracing"; + } ]; }; @@ -6815,7 +7483,7 @@ rec { features = { "debug" = [ "impl-debug" ]; }; - resolvedDefaultFeatures = [ "cfg" "consoleapi" "evntrace" "handleapi" "impl-default" "in6addr" "inaddr" "minwinbase" "ntsecapi" "processenv" "synchapi" "winbase" "windef" "winerror" "winioctl" "winuser" ]; + resolvedDefaultFeatures = [ "cfg" "consoleapi" "errhandlingapi" "evntrace" "fileapi" "handleapi" "impl-default" "in6addr" "inaddr" "minwinbase" "ntsecapi" "processenv" "synchapi" "winbase" "windef" "winerror" "winioctl" "winuser" ]; }; "winapi-i686-pc-windows-gnu" = rec { crateName = "winapi-i686-pc-windows-gnu"; @@ -7784,7 +8452,7 @@ rec { "Win32_Web" = [ "Win32" ]; "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; }; - resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Wdk_System" "Wdk_System_IO" "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Com" "Win32_System_IO" "Win32_System_Pipes" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_WindowsProgramming" "Win32_UI" "Win32_UI_Shell" "default" ]; + resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Wdk_System" "Wdk_System_IO" "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Com" "Win32_System_Console" "Win32_System_IO" "Win32_System_Pipes" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_WindowsProgramming" "Win32_UI" "Win32_UI_Input" "Win32_UI_Input_KeyboardAndMouse" "Win32_UI_Shell" "default" ]; }; "windows-targets" = rec { crateName = "windows-targets"; diff --git a/users/edef/weave/Cargo.toml b/users/edef/weave/Cargo.toml index 55b4e187defe..936428526046 100644 --- a/users/edef/weave/Cargo.toml +++ b/users/edef/weave/Cargo.toml @@ -14,6 +14,9 @@ nix-compat = { version = "0.1.0", path = "../../../tvix/nix-compat" } safer_owning_ref = "0.5.0" rayon = "1.8.1" rustc-hash = "2.0.0" +tvix-tracing = { version = "0.1.0", path = "../../../tvix/tracing" } +tracing = "0.1.40" +tracing-indicatif = "0.3.6" [dependencies.polars] version = "0.36.2" diff --git a/users/edef/weave/src/bin/swizzle.rs b/users/edef/weave/src/bin/swizzle.rs index bcea3edfada9..840a53454f57 100644 --- a/users/edef/weave/src/bin/swizzle.rs +++ b/users/edef/weave/src/bin/swizzle.rs @@ -36,17 +36,27 @@ use polars::{ lazy::dsl::{col, SpecialEq}, prelude::*, }; +use tracing::info_span; +use tracing_indicatif::span_ext::IndicatifSpanExt as _; -use weave::{as_fixed_binary, hash64, leak, load_ph_array, DONE, INDEX_NULL}; +use weave::{as_fixed_binary, hash64, leak, load_ph_array, INDEX_NULL}; +#[tracing::instrument] fn main() -> Result<()> { + let _tracing = tvix_tracing::TracingBuilder::default() + .enable_progressbar() + .build()?; + let ph_array: &'static [[u8; 20]] = leak(load_ph_array()?); // TODO(edef): re-parallelise this // We originally parallelised on chunks, but ph_array is only a single chunk, due to how Parquet loading works. // TODO(edef): outline the 64-bit hash prefix? it's an indirection, but it saves ~2G of memory - eprint!("… build index\r"); let ph_map: &'static HashTable<(u64, u32)> = { + let span = info_span!("ph_map", indicatif.pb_show = tracing::field::Empty).entered(); + span.pb_set_message("build index"); + span.pb_start(); + let mut ph_map = HashTable::with_capacity(ph_array.len()); for (offset, item) in ph_array.iter().enumerate() { @@ -57,7 +67,6 @@ fn main() -> Result<()> { &*Box::leak(Box::new(ph_map)) }; - eprintln!("{DONE}"); let ph_to_idx = |key: &[u8; 20]| -> u32 { let hash = hash64(key); @@ -69,35 +78,41 @@ fn main() -> Result<()> { .unwrap_or(INDEX_NULL) }; - eprint!("… swizzle references\r"); - LazyFrame::scan_parquet("narinfo.parquet", ScanArgsParquet::default())? - .with_column( - col("references") - .map( - move |series: Series| -> PolarsResult> { - Ok(Some( - series - .list()? - .apply_to_inner(&|series: Series| -> PolarsResult { - let series = series.binary()?; - let mut out: Vec = Vec::with_capacity(series.len()); - out.extend(as_fixed_binary(series).flatten().map(ph_to_idx)); - Ok(Series::from_vec("reference_idxs", out)) - })? - .into_series(), - )) - }, - SpecialEq::from_type(DataType::List(DataType::UInt32.into())), - ) - .alias("reference_idxs"), - ) - .select([col("reference_idxs")]) - .with_streaming(true) - .sink_parquet( - "narinfo-references.parquet".into(), - ParquetWriteOptions::default(), - )?; - eprintln!("{DONE}"); + { + let span = info_span!("swizzle_refs", indicatif.pb_show = tracing::field::Empty).entered(); + span.pb_set_message("swizzle references"); + span.pb_start(); + + LazyFrame::scan_parquet("narinfo.parquet", ScanArgsParquet::default())? + .with_column( + col("references") + .map( + move |series: Series| -> PolarsResult> { + Ok(Some( + series + .list()? + .apply_to_inner(&|series: Series| -> PolarsResult { + let series = series.binary()?; + let mut out: Vec = Vec::with_capacity(series.len()); + out.extend( + as_fixed_binary(series).flatten().map(ph_to_idx), + ); + Ok(Series::from_vec("reference_idxs", out)) + })? + .into_series(), + )) + }, + SpecialEq::from_type(DataType::List(DataType::UInt32.into())), + ) + .alias("reference_idxs"), + ) + .select([col("reference_idxs")]) + .with_streaming(true) + .sink_parquet( + "narinfo-references.parquet".into(), + ParquetWriteOptions::default(), + )?; + }; Ok(()) } diff --git a/users/edef/weave/src/lib.rs b/users/edef/weave/src/lib.rs index 4ccd566ca52d..ff5bf9232676 100644 --- a/users/edef/weave/src/lib.rs +++ b/users/edef/weave/src/lib.rs @@ -8,6 +8,7 @@ use std::{ slice, sync::Arc, }; +use tracing_indicatif::span_ext::IndicatifSpanExt as _; use polars::{ datatypes::BinaryChunked, @@ -20,7 +21,6 @@ pub type FixedBytes = ArcRef<'static, polars::export::arrow::buffer::Bytes, [[u8; N]]>; pub const INDEX_NULL: u32 = !0; -pub const DONE: &str = "\u{2714}"; /// A terrific hash function, turning 20 bytes of cryptographic hash /// into 8 bytes of cryptographic hash. @@ -42,8 +42,13 @@ pub fn leak(r: OwningRef, T>) -> &T { /// Read a dense `store_path_hash` array from `narinfo.parquet`, /// returning it as an owned [FixedBytes]. +#[tracing::instrument(fields(indicatif.pb_show = tracing::field::Empty))] pub fn load_ph_array() -> Result> { - eprint!("… load store_path_hash\r"); + let span = tracing::Span::current(); + + span.pb_set_message("load store_path_hash"); + span.pb_start(); + // TODO(edef): this could use a further pushdown, since polars is more hindrance than help here // We know this has to fit in memory (we can't mmap it without further encoding constraints), // and we want a single `Vec<[u8; 20]>` of the data. @@ -57,7 +62,6 @@ pub fn load_ph_array() -> Result> { ); u32::try_from(ph_array.len()).expect("dataset exceeds 2^32"); - eprintln!("{DONE}"); Ok(ph_array) } diff --git a/users/edef/weave/src/main.rs b/users/edef/weave/src/main.rs index b86992c279d1..b1dd27d8c1af 100644 --- a/users/edef/weave/src/main.rs +++ b/users/edef/weave/src/main.rs @@ -15,6 +15,8 @@ use std::{ ops::Index, sync::atomic::{AtomicU32, Ordering}, }; +use tracing::{info_span, warn}; +use tracing_indicatif::span_ext::IndicatifSpanExt; use polars::{ datatypes::StaticArray, @@ -23,36 +25,48 @@ use polars::{ prelude::*, }; -use weave::{as_fixed_binary, hash64, DONE, INDEX_NULL}; +use weave::{as_fixed_binary, hash64, INDEX_NULL}; +#[tracing::instrument] fn main() -> Result<()> { - eprint!("… parse roots\r"); - let roots: PathSet32 = as_fixed_binary::<20>( - LazyFrame::scan_parquet("releases.parquet", ScanArgsParquet::default())? - .explode([col("store_path_hash")]) - .select([col("store_path_hash")]) - .collect()? - .column("store_path_hash")? - .binary()?, - ) - .flatten() - .collect(); - eprintln!("{DONE}"); + let _tracing = tvix_tracing::TracingBuilder::default() + .enable_progressbar() + .build()?; + + let roots: PathSet32 = { + let span = info_span!("parse_roots", indicatif.pb_show = tracing::field::Empty).entered(); + span.pb_set_message("parse roots"); + span.pb_start(); + + as_fixed_binary::<20>( + LazyFrame::scan_parquet("releases.parquet", ScanArgsParquet::default())? + .explode([col("store_path_hash")]) + .select([col("store_path_hash")]) + .collect()? + .column("store_path_hash")? + .binary()?, + ) + .flatten() + .collect() + }; { - let ph_array = weave::load_ph_array()?; - - eprint!("… resolve roots\r"); - ph_array.par_iter().enumerate().for_each(|(idx, h)| { - if let Some(idx_slot) = roots.find(h) { - assert_eq!( - idx_slot.swap(idx as u32, Ordering::Relaxed), - INDEX_NULL, - "duplicate entry" - ); - } - }); - eprintln!("{DONE}"); + let span = info_span!("resolve_roots", indicatif.pb_show = tracing::field::Empty).entered(); + span.pb_set_message("resolve roots"); + span.pb_start(); + + weave::load_ph_array()? + .into_par_iter() + .enumerate() + .for_each(|(idx, h)| { + if let Some(idx_slot) = roots.find(h) { + assert_eq!( + idx_slot.swap(idx as u32, Ordering::Relaxed), + INDEX_NULL, + "duplicate entry" + ); + } + }); } let mut todo = FxHashSet::default(); @@ -67,17 +81,28 @@ fn main() -> Result<()> { } todo.insert(idx); } - println!("skipping {unknown_roots} unknown roots"); - } - eprint!("… load reference_idxs\r"); - let ri_array = ParquetReader::new(File::open("narinfo-references.parquet")?) - .finish()? - .column("reference_idxs")? - .list()? - .clone(); + if unknown_roots != 0 { + warn!("skipping {unknown_roots} unknown roots"); + } + } + let ri_array; let ri_array = { + let span = info_span!( + "load_reference_idxs", + indicatif.pb_show = tracing::field::Empty + ) + .entered(); + span.pb_set_message("load reference_idxs"); + span.pb_start(); + + ri_array = ParquetReader::new(File::open("narinfo-references.parquet")?) + .finish()? + .column("reference_idxs")? + .list()? + .clone(); + ChunkedList::new(ri_array.downcast_iter().map(|chunk| { ( chunk.offsets(), @@ -91,48 +116,63 @@ fn main() -> Result<()> { ) })) }; - eprintln!("{DONE}"); let mut seen = todo.clone(); - while !todo.is_empty() { - println!("todo: {} seen: {}", todo.len(), seen.len()); - - todo = todo - .par_iter() - .flat_map(|&parent| { - if parent == INDEX_NULL { - return FxHashSet::default(); - } + { + let span = info_span!("mark", indicatif.pb_show = tracing::field::Empty).entered(); + span.pb_set_message("marking"); + span.pb_set_style(&tvix_tracing::PB_PROGRESS_STYLE); + + while !todo.is_empty() { + span.pb_set_length(seen.len() as u64); + span.pb_set_position(seen.len().saturating_sub(todo.len()) as u64); + + todo = todo + .par_iter() + .flat_map(|&parent| { + if parent == INDEX_NULL { + return FxHashSet::default(); + } + + ri_array[parent as usize] + .iter() + .cloned() + .filter(|child| !seen.contains(child)) + .collect::>() + }) + .collect(); + + for &index in &todo { + seen.insert(index); + } + } - ri_array[parent as usize] - .iter() - .cloned() - .filter(|child| !seen.contains(child)) - .collect::>() - }) - .collect(); + span.pb_set_length(seen.len() as u64); + span.pb_set_position(seen.len() as u64); - for &index in &todo { - seen.insert(index); + if seen.remove(&INDEX_NULL) { + warn!("WARNING: missing edges"); } } - println!("done: {} paths", seen.len()); + let seen = { + let span = info_span!("gather_live", indicatif.pb_show = tracing::field::Empty).entered(); + span.pb_set_message("gathering live set"); - if seen.remove(&INDEX_NULL) { - println!("WARNING: missing edges"); - } + let mut seen: Vec = seen.into_iter().collect(); + seen.par_sort(); + seen + }; - eprint!("… gathering live set\r"); - let mut seen: Vec = seen.into_iter().collect(); - seen.par_sort(); - eprintln!("{DONE}"); + { + let span = info_span!("write_output", indicatif.pb_show = tracing::field::Empty).entered(); + span.pb_set_message("writing output"); + span.pb_start(); - eprint!("… writing output\r"); - ParquetWriter::new(File::create("live_idxs.parquet")?).finish(&mut df! { - "live_idx" => seen, - }?)?; - eprintln!("{DONE}"); + ParquetWriter::new(File::create("live_idxs.parquet")?).finish(&mut df! { + "live_idx" => seen, + }?)?; + } Ok(()) } -- cgit 1.4.1