diff options
| -rw-r--r-- | tvix/Cargo.lock | 57 | ||||
| -rw-r--r-- | tvix/Cargo.nix | 218 | ||||
| -rw-r--r-- | tvix/docs/src/TODO.md | 7 | ||||
| -rw-r--r-- | tvix/store/Cargo.toml | 3 | ||||
| -rw-r--r-- | tvix/store/src/pathinfoservice/nix_http.rs | 6 | ||||
| -rw-r--r-- | tvix/tracing/Cargo.toml | 8 | ||||
| -rw-r--r-- | tvix/tracing/default.nix | 2 | ||||
| -rw-r--r-- | tvix/tracing/src/lib.rs | 1 | ||||
| -rw-r--r-- | tvix/tracing/src/propagate/mod.rs | 4 | ||||
| -rw-r--r-- | tvix/tracing/src/propagate/reqwest.rs | 13 |
10 files changed, 302 insertions, 17 deletions
diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock index e758b359760d..731731e72a1b 100644 --- a/tvix/Cargo.lock +++ b/tvix/Cargo.lock @@ -1432,8 +1432,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -2033,6 +2035,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] +name = "mime_guess" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +dependencies = [ + "mime", + "unicase", +] + +[[package]] name = "minimal-lexical" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2961,6 +2973,7 @@ dependencies = [ "js-sys", "log", "mime", + "mime_guess", "once_cell", "percent-encoding", "pin-project-lite", @@ -2984,6 +2997,39 @@ dependencies = [ ] [[package]] +name = "reqwest-middleware" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a735987236a8e238bf0296c7e351b999c188ccc11477f311b82b55c93984216" +dependencies = [ + "anyhow", + "async-trait", + "http", + "reqwest", + "serde", + "task-local-extensions", + "thiserror", +] + +[[package]] +name = "reqwest-tracing" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "190838e54153d7a7e2ea98851304b3ce92daeabf14c54d32b01b84a3e636f683" +dependencies = [ + "anyhow", + "async-trait", + "getrandom", + "matchit", + "opentelemetry", + "reqwest", + "reqwest-middleware", + "task-local-extensions", + "tracing", + "tracing-opentelemetry", +] + +[[package]] name = "ring" version = "0.17.7" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3643,6 +3689,15 @@ dependencies = [ ] [[package]] +name = "task-local-extensions" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba323866e5d033818e3240feeb9f7db2c4296674e4d9e16b97b7bf8f490434e8" +dependencies = [ + "pin-utils", +] + +[[package]] name = "tempfile" version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4450,6 +4505,7 @@ dependencies = [ "prost", "prost-build", "reqwest", + "reqwest-middleware", "rstest", "rstest_reuse", "serde", @@ -4489,6 +4545,7 @@ dependencies = [ "opentelemetry-http", "opentelemetry-otlp", "opentelemetry_sdk", + "reqwest-tracing", "thiserror", "tokio", "tonic", diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix index 25a9a4e6b34e..4d6e1d94628f 100644 --- a/tvix/Cargo.nix +++ b/tvix/Cargo.nix @@ -4326,6 +4326,12 @@ rec { packageId = "cfg-if"; } { + name = "js-sys"; + packageId = "js-sys"; + optional = true; + target = { target, features }: ((("wasm32" == target."arch" or null) || ("wasm64" == target."arch" or null)) && ("unknown" == target."os" or null)); + } + { name = "libc"; packageId = "libc"; usesDefaultFeatures = false; @@ -4337,6 +4343,13 @@ rec { usesDefaultFeatures = false; target = { target, features }: ("wasi" == target."os" or null); } + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: ((("wasm32" == target."arch" or null) || ("wasm64" == target."arch" or null)) && ("unknown" == target."os" or null)); + } ]; features = { "compiler_builtins" = [ "dep:compiler_builtins" ]; @@ -4346,7 +4359,7 @@ rec { "rustc-dep-of-std" = [ "compiler_builtins" "core" "libc/rustc-dep-of-std" "wasi/rustc-dep-of-std" ]; "wasm-bindgen" = [ "dep:wasm-bindgen" ]; }; - resolvedDefaultFeatures = [ "std" ]; + resolvedDefaultFeatures = [ "js" "js-sys" "std" "wasm-bindgen" ]; }; "gimli" = rec { crateName = "gimli"; @@ -6084,6 +6097,34 @@ rec { ]; }; + "mime_guess" = rec { + crateName = "mime_guess"; + version = "2.0.4"; + edition = "2015"; + sha256 = "1vs28rxnbfwil6f48hh58lfcx90klcvg68gxdc60spwa4cy2d4j1"; + authors = [ + "Austin Bonander <austin.bonander@gmail.com>" + ]; + dependencies = [ + { + name = "mime"; + packageId = "mime"; + } + { + name = "unicase"; + packageId = "unicase"; + } + ]; + buildDependencies = [ + { + name = "unicase"; + packageId = "unicase"; + } + ]; + features = { + "default" = [ "rev-mappings" ]; + }; + }; "minimal-lexical" = rec { crateName = "minimal-lexical"; version = "0.2.1"; @@ -8992,6 +9033,12 @@ rec { target = { target, features }: (!("wasm32" == target."arch" or null)); } { + name = "mime_guess"; + packageId = "mime_guess"; + optional = true; + usesDefaultFeatures = false; + } + { name = "once_cell"; packageId = "once_cell"; target = { target, features }: (!("wasm32" == target."arch" or null)); @@ -9178,7 +9225,140 @@ rec { "wasm-streams" = [ "dep:wasm-streams" ]; "webpki-roots" = [ "dep:webpki-roots" ]; }; - resolvedDefaultFeatures = [ "__rustls" "__tls" "hyper-rustls" "json" "rustls" "rustls-native-certs" "rustls-pemfile" "rustls-tls-native-roots" "serde_json" "stream" "tokio-rustls" "tokio-util" "wasm-streams" ]; + resolvedDefaultFeatures = [ "__rustls" "__tls" "hyper-rustls" "json" "mime_guess" "multipart" "rustls" "rustls-native-certs" "rustls-pemfile" "rustls-tls-native-roots" "serde_json" "stream" "tokio-rustls" "tokio-util" "wasm-streams" ]; + }; + "reqwest-middleware" = rec { + crateName = "reqwest-middleware"; + version = "0.2.5"; + edition = "2018"; + sha256 = "05j2k29mrdc23cqpyiqirj61i74r3cspwv19y25j73ka4f3mjwss"; + authors = [ + "Rodrigo Gryzinski <rodrigo.gryzinski@truelayer.com>" + ]; + dependencies = [ + { + name = "anyhow"; + packageId = "anyhow"; + } + { + name = "async-trait"; + packageId = "async-trait"; + } + { + name = "http"; + packageId = "http"; + } + { + name = "reqwest"; + packageId = "reqwest"; + usesDefaultFeatures = false; + features = [ "json" "multipart" ]; + } + { + name = "serde"; + packageId = "serde"; + } + { + name = "task-local-extensions"; + packageId = "task-local-extensions"; + } + { + name = "thiserror"; + packageId = "thiserror"; + } + ]; + + }; + "reqwest-tracing" = rec { + crateName = "reqwest-tracing"; + version = "0.4.8"; + edition = "2018"; + sha256 = "10zn6vka710vn0r4vi8lpzmdm4nfnc2171cqxbiagmsk87jkh20r"; + authors = [ + "Rodrigo Gryzinski <rodrigo.gryzinski@truelayer.com>" + ]; + dependencies = [ + { + name = "anyhow"; + packageId = "anyhow"; + } + { + name = "async-trait"; + packageId = "async-trait"; + } + { + name = "getrandom"; + packageId = "getrandom"; + target = { target, features }: ("wasm32" == target."arch" or null); + features = [ "js" ]; + } + { + name = "matchit"; + packageId = "matchit"; + } + { + name = "opentelemetry"; + packageId = "opentelemetry"; + rename = "opentelemetry_0_22_pkg"; + optional = true; + } + { + name = "reqwest"; + packageId = "reqwest"; + usesDefaultFeatures = false; + } + { + name = "reqwest-middleware"; + packageId = "reqwest-middleware"; + } + { + name = "task-local-extensions"; + packageId = "task-local-extensions"; + } + { + name = "tracing"; + packageId = "tracing"; + } + { + name = "tracing-opentelemetry"; + packageId = "tracing-opentelemetry"; + rename = "tracing-opentelemetry_0_23_pkg"; + optional = true; + } + ]; + features = { + "opentelemetry_0_13" = [ "opentelemetry_0_13_pkg" "tracing-opentelemetry_0_12_pkg" ]; + "opentelemetry_0_13_pkg" = [ "dep:opentelemetry_0_13_pkg" ]; + "opentelemetry_0_14" = [ "opentelemetry_0_14_pkg" "tracing-opentelemetry_0_13_pkg" ]; + "opentelemetry_0_14_pkg" = [ "dep:opentelemetry_0_14_pkg" ]; + "opentelemetry_0_15" = [ "opentelemetry_0_15_pkg" "tracing-opentelemetry_0_14_pkg" ]; + "opentelemetry_0_15_pkg" = [ "dep:opentelemetry_0_15_pkg" ]; + "opentelemetry_0_16" = [ "opentelemetry_0_16_pkg" "tracing-opentelemetry_0_16_pkg" ]; + "opentelemetry_0_16_pkg" = [ "dep:opentelemetry_0_16_pkg" ]; + "opentelemetry_0_17" = [ "opentelemetry_0_17_pkg" "tracing-opentelemetry_0_17_pkg" ]; + "opentelemetry_0_17_pkg" = [ "dep:opentelemetry_0_17_pkg" ]; + "opentelemetry_0_18" = [ "opentelemetry_0_18_pkg" "tracing-opentelemetry_0_18_pkg" ]; + "opentelemetry_0_18_pkg" = [ "dep:opentelemetry_0_18_pkg" ]; + "opentelemetry_0_19" = [ "opentelemetry_0_19_pkg" "tracing-opentelemetry_0_19_pkg" ]; + "opentelemetry_0_19_pkg" = [ "dep:opentelemetry_0_19_pkg" ]; + "opentelemetry_0_20" = [ "opentelemetry_0_20_pkg" "tracing-opentelemetry_0_20_pkg" ]; + "opentelemetry_0_20_pkg" = [ "dep:opentelemetry_0_20_pkg" ]; + "opentelemetry_0_21" = [ "opentelemetry_0_21_pkg" "tracing-opentelemetry_0_22_pkg" ]; + "opentelemetry_0_21_pkg" = [ "dep:opentelemetry_0_21_pkg" ]; + "opentelemetry_0_22" = [ "opentelemetry_0_22_pkg" "tracing-opentelemetry_0_23_pkg" ]; + "opentelemetry_0_22_pkg" = [ "dep:opentelemetry_0_22_pkg" ]; + "tracing-opentelemetry_0_12_pkg" = [ "dep:tracing-opentelemetry_0_12_pkg" ]; + "tracing-opentelemetry_0_13_pkg" = [ "dep:tracing-opentelemetry_0_13_pkg" ]; + "tracing-opentelemetry_0_14_pkg" = [ "dep:tracing-opentelemetry_0_14_pkg" ]; + "tracing-opentelemetry_0_16_pkg" = [ "dep:tracing-opentelemetry_0_16_pkg" ]; + "tracing-opentelemetry_0_17_pkg" = [ "dep:tracing-opentelemetry_0_17_pkg" ]; + "tracing-opentelemetry_0_18_pkg" = [ "dep:tracing-opentelemetry_0_18_pkg" ]; + "tracing-opentelemetry_0_19_pkg" = [ "dep:tracing-opentelemetry_0_19_pkg" ]; + "tracing-opentelemetry_0_20_pkg" = [ "dep:tracing-opentelemetry_0_20_pkg" ]; + "tracing-opentelemetry_0_22_pkg" = [ "dep:tracing-opentelemetry_0_22_pkg" ]; + "tracing-opentelemetry_0_23_pkg" = [ "dep:tracing-opentelemetry_0_23_pkg" ]; + }; + resolvedDefaultFeatures = [ "opentelemetry_0_22" "opentelemetry_0_22_pkg" "tracing-opentelemetry_0_23_pkg" ]; }; "ring" = rec { crateName = "ring"; @@ -11164,6 +11344,23 @@ rec { features = { }; resolvedDefaultFeatures = [ "default" ]; }; + "task-local-extensions" = rec { + crateName = "task-local-extensions"; + version = "0.1.4"; + edition = "2018"; + sha256 = "1s1l0i4qzgxpjxmy3ng4fik2ki5jgngypzj06a782cyhwmk3hcms"; + authors = [ + "Conrad Ludgate <conrad.ludgate@truelayer.com>" + "Rodrigo Gryzinski <rodrigo.gryzinski@truelayer.com>" + ]; + dependencies = [ + { + name = "pin-utils"; + packageId = "pin-utils"; + } + ]; + + }; "tempfile" = rec { crateName = "tempfile"; version = "3.9.0"; @@ -14185,6 +14382,10 @@ rec { features = [ "rustls-tls-native-roots" "stream" ]; } { + name = "reqwest-middleware"; + packageId = "reqwest-middleware"; + } + { name = "serde"; packageId = "serde"; features = [ "derive" ]; @@ -14267,7 +14468,7 @@ rec { { name = "tvix-tracing"; packageId = "tvix-tracing"; - features = [ "tonic" ]; + features = [ "tonic" "reqwest" ]; } { name = "url"; @@ -14362,6 +14563,12 @@ rec { features = [ "rt-tokio" ]; } { + name = "reqwest-tracing"; + packageId = "reqwest-tracing"; + optional = true; + usesDefaultFeatures = false; + } + { name = "thiserror"; packageId = "thiserror"; } @@ -14402,11 +14609,12 @@ rec { } ]; features = { - "otlp" = [ "dep:tracing-opentelemetry" "dep:opentelemetry" "dep:opentelemetry-otlp" "dep:opentelemetry_sdk" "dep:opentelemetry-http" ]; + "otlp" = [ "dep:tracing-opentelemetry" "dep:opentelemetry" "dep:opentelemetry-otlp" "dep:opentelemetry_sdk" "dep:opentelemetry-http" "reqwest-tracing?/opentelemetry_0_22" ]; + "reqwest" = [ "dep:reqwest-tracing" ]; "tonic" = [ "dep:tonic" "dep:http" ]; "tracy" = [ "dep:tracing-tracy" ]; }; - resolvedDefaultFeatures = [ "default" "otlp" "tonic" "tracy" ]; + resolvedDefaultFeatures = [ "default" "otlp" "reqwest" "tonic" "tracy" ]; }; "typenum" = rec { crateName = "typenum"; diff --git a/tvix/docs/src/TODO.md b/tvix/docs/src/TODO.md index 127fb6f4d0c1..52fa0a9e7480 100644 --- a/tvix/docs/src/TODO.md +++ b/tvix/docs/src/TODO.md @@ -224,7 +224,6 @@ logs etc, but this is something requiring a lot of designing. - Maybe drop `--log-level` entirely, and only use `RUST_LOG` env exclusively? `debug`,`trace` level across all crates is a bit useless, and `RUST_LOG` can be much more granular… - - Trace propagation for HTTP clients too, using - https://www.w3.org/TR/trace-context/ or https://www.w3.org/TR/baggage/, - whichever makes more sense. - Candidates: nix+http(s) protocol, object_store crates. + - Trace propagation for object_store once they support a way to register a + middleware, so we can use that to register a tracing middleware. + https://github.com/apache/arrow-rs/issues/5990 diff --git a/tvix/store/Cargo.toml b/tvix/store/Cargo.toml index e8fbaf02c594..8063a0e874c3 100644 --- a/tvix/store/Cargo.toml +++ b/tvix/store/Cargo.toml @@ -36,9 +36,10 @@ tvix-castore = { path = "../castore" } url = "2.4.0" walkdir = "2.4.0" reqwest = { version = "0.11.22", features = ["rustls-tls-native-roots", "stream"], default-features = false } +reqwest-middleware = "0.2.5" lru = "0.12.3" parking_lot = "0.12.2" -tvix-tracing = { path = "../tracing", features = ["tonic"] } +tvix-tracing = { path = "../tracing", features = ["tonic", "reqwest"] } tracing = "0.1.40" tracing-indicatif = "0.3.6" diff --git a/tvix/store/src/pathinfoservice/nix_http.rs b/tvix/store/src/pathinfoservice/nix_http.rs index 1dd7da4831b4..57fe37f44e88 100644 --- a/tvix/store/src/pathinfoservice/nix_http.rs +++ b/tvix/store/src/pathinfoservice/nix_http.rs @@ -31,7 +31,7 @@ use tvix_castore::{ /// TODO: what about reading from nix-cache-info? pub struct NixHTTPPathInfoService<BS, DS> { base_url: url::Url, - http_client: reqwest::Client, + http_client: reqwest_middleware::ClientWithMiddleware, blob_service: BS, directory_service: DS, @@ -45,7 +45,9 @@ impl<BS, DS> NixHTTPPathInfoService<BS, DS> { pub fn new(base_url: url::Url, blob_service: BS, directory_service: DS) -> Self { Self { base_url, - http_client: reqwest::Client::new(), + http_client: reqwest_middleware::ClientBuilder::new(reqwest::Client::new()) + .with(tvix_tracing::propagate::reqwest::tracing_middleware()) + .build(), blob_service, directory_service, diff --git a/tvix/tracing/Cargo.toml b/tvix/tracing/Cargo.toml index bc9a8c3c7792..80892bf7a09f 100644 --- a/tvix/tracing/Cargo.toml +++ b/tvix/tracing/Cargo.toml @@ -22,6 +22,8 @@ opentelemetry-http = { version = "0.11.0", optional = true } tonic = { version = "0.11.0", optional = true } http = { version = "0.2.11", optional = true } +reqwest-tracing = { version = "0.4.8", default-features = false, optional = true } + [features] default = [] otlp = [ @@ -29,7 +31,8 @@ otlp = [ "dep:opentelemetry", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk", - "dep:opentelemetry-http" + "dep:opentelemetry-http", + "reqwest-tracing?/opentelemetry_0_22", ] tracy = [ "dep:tracing-tracy" @@ -38,6 +41,9 @@ tonic = [ "dep:tonic", "dep:http", ] +reqwest = [ + "dep:reqwest-tracing", +] [lints] workspace = true diff --git a/tvix/tracing/default.nix b/tvix/tracing/default.nix index dd7dc200f2c6..a4ac9de1c8bc 100644 --- a/tvix/tracing/default.nix +++ b/tvix/tracing/default.nix @@ -6,6 +6,6 @@ meta.ci.targets = lib.filter (x: lib.hasPrefix "with-features" x || x == "no-features") (lib.attrNames passthru); passthru = depot.tvix.utils.mkFeaturePowerset { inherit (old) crateName; - features = [ "otlp" "tracy" ]; + features = [ "otlp" "tracy" "tonic" "reqwest" ]; }; }) diff --git a/tvix/tracing/src/lib.rs b/tvix/tracing/src/lib.rs index b914408ff4c7..35fdcda5a88a 100644 --- a/tvix/tracing/src/lib.rs +++ b/tvix/tracing/src/lib.rs @@ -17,7 +17,6 @@ use opentelemetry_sdk::{ #[cfg(feature = "tracy")] use tracing_tracy::TracyLayer; -#[cfg(feature = "tonic")] // TODO or http pub mod propagate; lazy_static! { diff --git a/tvix/tracing/src/propagate/mod.rs b/tvix/tracing/src/propagate/mod.rs index 42c532e9d8cf..9a7e4332b637 100644 --- a/tvix/tracing/src/propagate/mod.rs +++ b/tvix/tracing/src/propagate/mod.rs @@ -1,8 +1,8 @@ #[cfg(feature = "tonic")] pub mod tonic; -// TODO: Helper library for reqwest. We could use -// https://github.com/TrueLayer/reqwest-middleware/tree/main/reqwest-tracing to realise this +#[cfg(feature = "reqwest")] +pub mod reqwest; // TODO: Helper library for axum or another http server, see // https://github.com/hseeberger/hello-tracing-rs/blob/main/hello-tracing-common/src/otel/http.rs diff --git a/tvix/tracing/src/propagate/reqwest.rs b/tvix/tracing/src/propagate/reqwest.rs new file mode 100644 index 000000000000..e2afb3e948c7 --- /dev/null +++ b/tvix/tracing/src/propagate/reqwest.rs @@ -0,0 +1,13 @@ +use reqwest_tracing::{SpanBackendWithUrl, TracingMiddleware}; + +/// Returns a new tracing middleware which can be used with reqwest_middleware. +/// It will then write the `traceparent` in the header on the request and additionally records the +/// `url` into `http.url`. +/// +/// If otlp feature is disabled, this will not insert a `traceparent` into the header. It will +/// basically function as a noop. +/// +/// `traceparent` => https://www.w3.org/TR/trace-context/#trace-context-http-headers-format +pub fn tracing_middleware() -> TracingMiddleware<SpanBackendWithUrl> { + TracingMiddleware::<SpanBackendWithUrl>::new() +} |