feat(tvix/tracing): introduce common tvix-tracing crate r/8242

Introduce a new common crate that contains tracing boilerplate which then
can be used in the cli, tvix-store and tvix-build crates.
It has otlp as an optional feature, which is currently only used by
tvix-store.

Change-Id: I41468ac4d9c65174515d721513b96fea463d6ed2
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11758
Tested-by: BuildkiteCI
Reviewed-by: flokli <flokli@flokli.de>
Autosubmit: Simon Hauser <simon.hauser@helsinki-systems.de>

15 files changed, 271 insertions, 203 deletions

diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock
index 584290084bde..d7955cbc1e89 100644
--- a/tvix/Cargo.lock
+++ b/tvix/Cargo.lock
@@ -4127,8 +4127,8 @@ dependencies = [
  "tonic-build",
  "tonic-reflection",
  "tracing",
- "tracing-subscriber",
  "tvix-castore",
+ "tvix-tracing",
  "url",
 ]
 
@@ -4204,12 +4204,12 @@ dependencies = [
  "tikv-jemallocator",
  "tokio",
  "tracing",
- "tracing-subscriber",
  "tvix-build",
  "tvix-castore",
  "tvix-eval",
  "tvix-glue",
  "tvix-store",
+ "tvix-tracing",
  "wu-manber",
 ]
 
@@ -4326,13 +4326,9 @@ dependencies = [
  "count-write",
  "data-encoding",
  "futures",
- "indicatif",
  "lazy_static",
  "lru",
  "nix-compat",
- "opentelemetry",
- "opentelemetry-otlp",
- "opentelemetry_sdk",
  "parking_lot 0.12.2",
  "pin-project-lite",
  "prost",
@@ -4359,14 +4355,28 @@ dependencies = [
  "tower",
  "tracing",
  "tracing-indicatif",
- "tracing-opentelemetry",
- "tracing-subscriber",
  "tvix-castore",
+ "tvix-tracing",
  "url",
  "walkdir",
 ]
 
 [[package]]
+name = "tvix-tracing"
+version = "0.1.0"
+dependencies = [
+ "indicatif",
+ "lazy_static",
+ "opentelemetry",
+ "opentelemetry-otlp",
+ "opentelemetry_sdk",
+ "tracing",
+ "tracing-indicatif",
+ "tracing-opentelemetry",
+ "tracing-subscriber",
+]
+
+[[package]]
 name = "typenum"
 version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix
index 1f92aed2830d..da67d34c311d 100644
--- a/tvix/Cargo.nix
+++ b/tvix/Cargo.nix
@@ -123,6 +123,16 @@ rec {
       # File a bug if you depend on any for non-debug work!
       debug = internal.debugCrate { inherit packageId; };
     };
+    "tvix-tracing" = rec {
+      packageId = "tvix-tracing";
+      build = internal.buildRustCrateWithFeatures {
+        packageId = "tvix-tracing";
+      };
+
+      # Debug support which might change between releases.
+      # File a bug if you depend on any for non-debug work!
+      debug = internal.debugCrate { inherit packageId; };
+    };
   };
 
   # A derivation that joins the outputs of all workspace members together.
@@ -12954,14 +12964,14 @@ rec {
             packageId = "tracing";
           }
           {
-            name = "tracing-subscriber";
-            packageId = "tracing-subscriber";
-          }
-          {
             name = "tvix-castore";
             packageId = "tvix-castore";
           }
           {
+            name = "tvix-tracing";
+            packageId = "tvix-tracing";
+          }
+          {
             name = "url";
             packageId = "url";
           }
@@ -13298,11 +13308,6 @@ rec {
           {
             name = "tracing";
             packageId = "tracing";
-            features = [ "max_level_trace" "release_max_level_info" ];
-          }
-          {
-            name = "tracing-subscriber";
-            packageId = "tracing-subscriber";
           }
           {
             name = "tvix-build";
@@ -13326,6 +13331,10 @@ rec {
             usesDefaultFeatures = false;
           }
           {
+            name = "tvix-tracing";
+            packageId = "tvix-tracing";
+          }
+          {
             name = "wu-manber";
             packageId = "wu-manber";
           }
@@ -13795,10 +13804,6 @@ rec {
             packageId = "futures";
           }
           {
-            name = "indicatif";
-            packageId = "indicatif";
-          }
-          {
             name = "lazy_static";
             packageId = "lazy_static";
           }
@@ -13812,22 +13817,6 @@ rec {
             features = [ "async" ];
           }
           {
-            name = "opentelemetry";
-            packageId = "opentelemetry";
-            optional = true;
-          }
-          {
-            name = "opentelemetry-otlp";
-            packageId = "opentelemetry-otlp";
-            optional = true;
-          }
-          {
-            name = "opentelemetry_sdk";
-            packageId = "opentelemetry_sdk";
-            optional = true;
-            features = [ "rt-tokio" ];
-          }
-          {
             name = "parking_lot";
             packageId = "parking_lot 0.12.2";
           }
@@ -13917,19 +13906,14 @@ rec {
             packageId = "tracing-indicatif";
           }
           {
-            name = "tracing-opentelemetry";
-            packageId = "tracing-opentelemetry";
-          }
-          {
-            name = "tracing-subscriber";
-            packageId = "tracing-subscriber";
-            features = [ "env-filter" ];
-          }
-          {
             name = "tvix-castore";
             packageId = "tvix-castore";
           }
           {
+            name = "tvix-tracing";
+            packageId = "tvix-tracing";
+          }
+          {
             name = "url";
             packageId = "url";
           }
@@ -13974,12 +13958,72 @@ rec {
           "cloud" = [ "dep:bigtable_rs" "tvix-castore/cloud" ];
           "default" = [ "cloud" "fuse" "otlp" "tonic-reflection" ];
           "fuse" = [ "tvix-castore/fuse" ];
-          "otlp" = [ "dep:opentelemetry" "dep:opentelemetry-otlp" "dep:opentelemetry_sdk" ];
+          "otlp" = [ "tvix-tracing/otlp" ];
           "tonic-reflection" = [ "dep:tonic-reflection" "tvix-castore/tonic-reflection" ];
           "virtiofs" = [ "tvix-castore/virtiofs" ];
         };
         resolvedDefaultFeatures = [ "cloud" "default" "fuse" "integration" "otlp" "tonic-reflection" "virtiofs" ];
       };
+      "tvix-tracing" = rec {
+        crateName = "tvix-tracing";
+        version = "0.1.0";
+        edition = "2021";
+        # We can't filter paths with references in Nix 2.4
+        # See https://github.com/NixOS/nix/issues/5410
+        src =
+          if ((lib.versionOlder builtins.nixVersion "2.4pre20211007") || (lib.versionOlder "2.5" builtins.nixVersion))
+          then lib.cleanSourceWith { filter = sourceFilter; src = ./tracing; }
+          else ./tracing;
+        dependencies = [
+          {
+            name = "indicatif";
+            packageId = "indicatif";
+          }
+          {
+            name = "lazy_static";
+            packageId = "lazy_static";
+          }
+          {
+            name = "opentelemetry";
+            packageId = "opentelemetry";
+            optional = true;
+          }
+          {
+            name = "opentelemetry-otlp";
+            packageId = "opentelemetry-otlp";
+            optional = true;
+          }
+          {
+            name = "opentelemetry_sdk";
+            packageId = "opentelemetry_sdk";
+            optional = true;
+            features = [ "rt-tokio" ];
+          }
+          {
+            name = "tracing";
+            packageId = "tracing";
+            features = [ "max_level_trace" "release_max_level_info" ];
+          }
+          {
+            name = "tracing-indicatif";
+            packageId = "tracing-indicatif";
+          }
+          {
+            name = "tracing-opentelemetry";
+            packageId = "tracing-opentelemetry";
+            optional = true;
+          }
+          {
+            name = "tracing-subscriber";
+            packageId = "tracing-subscriber";
+            features = [ "env-filter" ];
+          }
+        ];
+        features = {
+          "otlp" = [ "dep:tracing-opentelemetry" "dep:opentelemetry" "dep:opentelemetry-otlp" "dep:opentelemetry_sdk" ];
+        };
+        resolvedDefaultFeatures = [ "default" "otlp" ];
+      };
       "typenum" = rec {
         crateName = "typenum";
         version = "1.17.0";
diff --git a/tvix/Cargo.toml b/tvix/Cargo.toml
index 847d9aceeca0..ed5c6d0d8b5c 100644
--- a/tvix/Cargo.toml
+++ b/tvix/Cargo.toml
@@ -28,6 +28,7 @@ members = [
   "nix-compat",
   "serde",
   "store",
+  "tracing",
 ]
 
 [workspace.lints.clippy]
diff --git a/tvix/build/Cargo.toml b/tvix/build/Cargo.toml
index c8123c2617f6..6faa3ad7abdb 100644
--- a/tvix/build/Cargo.toml
+++ b/tvix/build/Cargo.toml
@@ -13,8 +13,8 @@ tokio = { version = "1.32.0" }
 tokio-listener = { version = "0.4.1", features = [ "tonic011" ] }
 tonic = { version = "0.11.0", features = ["tls", "tls-roots"] }
 tvix-castore = { path = "../castore" }
-tracing = "0.1.37"
-tracing-subscriber = "0.3.16"
+tvix-tracing = { path = "../tracing" }
+tracing = "0.1.40"
 url = "2.4.0"
 
 [dependencies.tonic-reflection]
diff --git a/tvix/build/src/bin/tvix-build.rs b/tvix/build/src/bin/tvix-build.rs
index 07d7e30dfda5..07085a433cfa 100644
--- a/tvix/build/src/bin/tvix-build.rs
+++ b/tvix/build/src/bin/tvix-build.rs
@@ -7,7 +7,6 @@ use tokio_listener::SystemOptions;
 use tokio_listener::UserOptions;
 use tonic::{self, transport::Server};
 use tracing::{info, Level};
-use tracing_subscriber::prelude::*;
 use tvix_build::{
     buildservice,
     proto::{build_service_server::BuildServiceServer, GRPCBuildServiceWrapper},
@@ -23,8 +22,12 @@ use tvix_castore::proto::FILE_DESCRIPTOR_SET as CASTORE_FILE_DESCRIPTOR_SET;
 #[derive(Parser)]
 #[command(author, version, about, long_about = None)]
 struct Cli {
-    #[arg(long)]
-    log_level: Option<Level>,
+    /// A global log level to use when printing logs.
+    /// It's also possible to set `RUST_LOG` according to
+    /// `tracing_subscriber::filter::EnvFilter`, which will always have
+    /// priority.
+    #[arg(long, default_value_t=Level::INFO)]
+    log_level: Level,
 
     #[command(subcommand)]
     command: Commands,
@@ -51,16 +54,7 @@ enum Commands {
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let cli = Cli::parse();
 
-    // configure log settings
-    let level = cli.log_level.unwrap_or(Level::INFO);
-
-    tracing_subscriber::registry()
-        .with(
-            tracing_subscriber::fmt::Layer::new()
-                .with_writer(std::io::stderr.with_max_level(level))
-                .pretty(),
-        )
-        .init();
+    tvix_tracing::init(cli.log_level)?;
 
     match cli.command {
         Commands::Daemon {
diff --git a/tvix/cli/Cargo.toml b/tvix/cli/Cargo.toml
index d3c05172e456..d91da42864ad 100644
--- a/tvix/cli/Cargo.toml
+++ b/tvix/cli/Cargo.toml
@@ -14,6 +14,7 @@ tvix-castore = { path = "../castore" }
 tvix-store = { path = "../store", default-features = false, features = []}
 tvix-eval = { path = "../eval" }
 tvix-glue = { path = "../glue" }
+tvix-tracing = { path = "../tracing" }
 bytes = "1.4.0"
 clap = { version = "4.0", features = ["derive", "env"] }
 dirs = "4.0.0"
@@ -21,8 +22,7 @@ rustyline = "10.0.0"
 rnix = "0.11.0"
 thiserror = "1.0.38"
 tokio = "1.28.0"
-tracing = { version = "0.1.37", features = ["max_level_trace", "release_max_level_info"] }
-tracing-subscriber = "0.3.16"
+tracing = "0.1.40"
 
 [dependencies.wu-manber]
 git = "https://github.com/tvlfyi/wu-manber.git"
diff --git a/tvix/cli/src/main.rs b/tvix/cli/src/main.rs
index 66ec5a99dba6..bb0b9b768b02 100644
--- a/tvix/cli/src/main.rs
+++ b/tvix/cli/src/main.rs
@@ -5,9 +5,6 @@ use repl::Repl;
 use std::rc::Rc;
 use std::{fs, path::PathBuf};
 use tracing::Level;
-use tracing_subscriber::fmt::writer::MakeWriterExt;
-use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
-use tracing_subscriber::{EnvFilter, Layer};
 use tvix_build::buildservice;
 use tvix_eval::builtins::impure_builtins;
 use tvix_eval::observer::{DisassemblingObserver, TracingObserver};
@@ -27,8 +24,12 @@ static GLOBAL: Jemalloc = Jemalloc;
 
 #[derive(Parser, Clone)]
 struct Args {
-    #[arg(long)]
-    log_level: Option<Level>,
+    /// A global log level to use when printing logs.
+    /// It's also possible to set `RUST_LOG` according to
+    /// `tracing_subscriber::filter::EnvFilter`, which will always have
+    /// priority.
+    #[arg(long, default_value_t=Level::INFO)]
+    log_level: Level,
 
     /// Path to a script to evaluate
     script: Option<PathBuf>,
@@ -270,24 +271,7 @@ fn lint(code: &str, path: Option<PathBuf>, args: &Args) -> bool {
 fn main() {
     let args = Args::parse();
 
-    // configure log settings
-    let level = args.log_level.unwrap_or(Level::INFO);
-
-    let subscriber = tracing_subscriber::registry().with(
-        tracing_subscriber::fmt::Layer::new()
-            .with_writer(std::io::stderr.with_max_level(level))
-            .compact()
-            .with_filter(
-                EnvFilter::builder()
-                    .with_default_directive(level.into())
-                    .from_env()
-                    .expect("invalid RUST_LOG"),
-            ),
-    );
-    subscriber
-        .try_init()
-        .expect("unable to set up tracing subscriber");
-
+    tvix_tracing::init(args.log_level).expect("unable to set up tracing subscriber");
     let tokio_runtime = tokio::runtime::Runtime::new().expect("failed to setup tokio runtime");
 
     let io_handle = init_io_handle(&tokio_runtime, &args);
diff --git a/tvix/default.nix b/tvix/default.nix
index edcae8dd0795..e1b8781ba603 100644
--- a/tvix/default.nix
+++ b/tvix/default.nix
@@ -128,6 +128,10 @@ let
         src = filterRustCrateSrc { root = prev.src.origSrc; };
       };
 
+      tvix-tracing = prev: {
+        src = filterRustCrateSrc { root = prev.src.origSrc; };
+      };
+
       nix-compat = prev: {
         src = filterRustCrateSrc rec {
           root = prev.src.origSrc;
diff --git a/tvix/docs/src/TODO.md b/tvix/docs/src/TODO.md
index 66d90aaa9494..97f5d8b7d67d 100644
--- a/tvix/docs/src/TODO.md
+++ b/tvix/docs/src/TODO.md
@@ -198,14 +198,9 @@ logs etc, but this is something requiring a lot of designing.
 - Some work ongoing on the worker operation parsing (griff, picnoir)
 
 ### O11Y
- - `[tracing-]indicatif` for progress/log reporting (cl/11747)
- - Currently there's a lot of boilerplate in the `tvix-store` CLI entrypoint,
-   and half of the boilerplate copied over to `tvix-cli`.
-   Setup of the tracing things should be unified into the `tvix-tracing` crate,
-   maybe including some of the CLI parameters (@simon).
-   Or maybe drop `--log-level` entirely, and only use `RUST_LOG` env
-   exclusively? `debug`,`trace` level across all crates is a bit useless, and
-   `RUST_LOG` can be much more granular…
+ - Maybe drop `--log-level` entirely, and only use `RUST_LOG` env exclusively?
+   `debug`,`trace` level across all crates is a bit useless, and `RUST_LOG` can
+   be much more granular…
  - The OTLP stack is quite spammy if there's no OTLP collector running on
    localhost.
    https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/
diff --git a/tvix/store/Cargo.toml b/tvix/store/Cargo.toml
index 28dad300fe20..dc6126724fe6 100644
--- a/tvix/store/Cargo.toml
+++ b/tvix/store/Cargo.toml
@@ -18,9 +18,6 @@ lazy_static = "1.4.0"
 nix-compat = { path = "../nix-compat", features = ["async"] }
 pin-project-lite = "0.2.13"
 prost = "0.12.1"
-opentelemetry = { version = "0.22.0", optional = true}
-opentelemetry-otlp = { version = "0.15.0", optional = true }
-opentelemetry_sdk = { version = "0.22.1", features = ["rt-tokio"], optional = true}
 serde = { version = "1.0.197", features = [ "derive" ] }
 serde_json = "1.0"
 serde_with = "3.7.0"
@@ -34,16 +31,14 @@ tokio-stream = { version = "0.1.14", features = ["fs"] }
 tokio-util = { version = "0.7.9", features = ["io", "io-util", "compat"] }
 tonic = { version = "0.11.0", features = ["tls", "tls-roots"] }
 tower = "0.4.13"
-tracing = "0.1.37"
-tracing-opentelemetry = "0.23.0"
-tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 tvix-castore = { path = "../castore" }
 url = "2.4.0"
 walkdir = "2.4.0"
 reqwest = { version = "0.11.22", features = ["rustls-tls-native-roots", "stream"], default-features = false }
 lru = "0.12.3"
 parking_lot = "0.12.2"
-indicatif = "0.17.8"
+tvix-tracing = { path = "../tracing" }
+tracing = "0.1.40"
 tracing-indicatif = "0.3.6"
 
 [dependencies.tonic-reflection]
@@ -74,7 +69,7 @@ cloud = [
   "tvix-castore/cloud"
 ]
 fuse = ["tvix-castore/fuse"]
-otlp = ["dep:opentelemetry", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk"]
+otlp = ["tvix-tracing/otlp"]
 tonic-reflection = ["dep:tonic-reflection", "tvix-castore/tonic-reflection"]
 virtiofs = ["tvix-castore/virtiofs"]
 # Whether to run the integration tests.
diff --git a/tvix/store/src/bin/tvix-store.rs b/tvix/store/src/bin/tvix-store.rs
index 2a2d6fe6f7d0..03c699b893cd 100644
--- a/tvix/store/src/bin/tvix-store.rs
+++ b/tvix/store/src/bin/tvix-store.rs
@@ -4,7 +4,6 @@ use clap::Subcommand;
 use futures::future::try_join_all;
 use futures::StreamExt;
 use futures::TryStreamExt;
-use indicatif::ProgressStyle;
 use nix_compat::path_info::ExportedPathInfo;
 use serde::Deserialize;
 use serde::Serialize;
@@ -14,14 +13,8 @@ use tokio_listener::Listener;
 use tokio_listener::SystemOptions;
 use tokio_listener::UserOptions;
 use tonic::transport::Server;
-use tracing::info;
-use tracing::info_span;
-use tracing::instrument;
-use tracing::Level;
-use tracing::Span;
-use tracing_indicatif::filter::IndicatifFilter;
-use tracing_indicatif::{span_ext::IndicatifSpanExt, IndicatifLayer};
-use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer};
+use tracing::{info, info_span, instrument, Level, Span};
+use tracing_indicatif::span_ext::IndicatifSpanExt as _;
 use tvix_castore::import::fs::ingest_path;
 use tvix_store::nar::NarCalculationService;
 use tvix_store::proto::NarInfo;
@@ -35,35 +28,12 @@ use tvix_store::pathinfoservice::PathInfoService;
 use tvix_store::proto::path_info_service_server::PathInfoServiceServer;
 use tvix_store::proto::GRPCPathInfoServiceWrapper;
 
-use lazy_static::lazy_static;
-
-// FUTUREWORK: move this to tracing crate
-lazy_static! {
-    pub static ref PB_PROGRESS_STYLE: ProgressStyle = ProgressStyle::with_template(
-        "{span_child_prefix}{bar:30} {wide_msg} [{elapsed_precise}]  {pos:>7}/{len:7}"
-    )
-    .expect("invalid progress template");
-    pub static ref PB_SPINNER_STYLE: ProgressStyle = ProgressStyle::with_template(
-        "{span_child_prefix}{spinner} {wide_msg} [{elapsed_precise}]  {pos:>7}/{len:7}"
-    )
-    .expect("invalid progress template");
-}
-
 #[cfg(any(feature = "fuse", feature = "virtiofs"))]
 use tvix_store::pathinfoservice::make_fs;
 
 #[cfg(feature = "fuse")]
 use tvix_castore::fs::fuse::FuseDaemon;
 
-#[cfg(feature = "otlp")]
-use opentelemetry::KeyValue;
-#[cfg(feature = "otlp")]
-use opentelemetry_sdk::{
-    resource::{ResourceDetector, SdkProvidedResourceDetector},
-    trace::BatchConfig,
-    Resource,
-};
-
 #[cfg(feature = "virtiofs")]
 use tvix_castore::fs::virtiofs::start_virtiofs_daemon;
 
@@ -83,8 +53,8 @@ struct Cli {
     /// It's also possible to set `RUST_LOG` according to
     /// `tracing_subscriber::filter::EnvFilter`, which will always have
     /// priority.
-    #[arg(long)]
-    log_level: Option<Level>,
+    #[arg(long, default_value_t=Level::INFO)]
+    log_level: Level,
 
     #[command(subcommand)]
     command: Commands,
@@ -234,75 +204,18 @@ fn default_threads() -> usize {
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let cli = Cli::parse();
 
-    // configure log settings
-    let level = cli.log_level.unwrap_or(Level::INFO);
-
-    let indicatif_layer = IndicatifLayer::new().with_progress_style(PB_SPINNER_STYLE.clone());
-
-    // Set up the tracing subscriber.
-    let subscriber = tracing_subscriber::registry()
-        .with(
-            tracing_subscriber::fmt::Layer::new()
-                .with_writer(indicatif_layer.get_stderr_writer())
-                .compact()
-                .with_filter(
-                    EnvFilter::builder()
-                        .with_default_directive(level.into())
-                        .from_env()
-                        .expect("invalid RUST_LOG"),
-                ),
-        )
-        .with(indicatif_layer.with_filter(
-            // only show progress for spans with indicatif.pb_show field being set
-            IndicatifFilter::new(false),
-        ));
-
-    // Add the otlp layer (when otlp is enabled, and it's not disabled in the CLI)
-    // then init the registry.
-    // If the feature is feature-flagged out, just init without adding the layer.
-    // It's necessary to do this separately, as every with() call chains the
-    // layer into the type of the registry.
     #[cfg(feature = "otlp")]
     {
-        let subscriber = if cli.otlp {
-            let tracer = opentelemetry_otlp::new_pipeline()
-                .tracing()
-                .with_exporter(opentelemetry_otlp::new_exporter().tonic())
-                .with_batch_config(BatchConfig::default())
-                .with_trace_config(opentelemetry_sdk::trace::config().with_resource({
-                    // use SdkProvidedResourceDetector.detect to detect resources,
-                    // but replace the default service name with our default.
-                    // https://github.com/open-telemetry/opentelemetry-rust/issues/1298
-                    let resources =
-                        SdkProvidedResourceDetector.detect(std::time::Duration::from_secs(0));
-                    // SdkProvidedResourceDetector currently always sets
-                    // `service.name`, but we don't like its default.
-                    if resources.get("service.name".into()).unwrap() == "unknown_service".into() {
-                        resources.merge(&Resource::new([KeyValue::new(
-                            "service.name",
-                            "tvix.store",
-                        )]))
-                    } else {
-                        resources
-                    }
-                }))
-                .install_batch(opentelemetry_sdk::runtime::Tokio)?;
-
-            // Create a tracing layer with the configured tracer
-            let layer = tracing_opentelemetry::layer().with_tracer(tracer);
-
-            subscriber.with(Some(layer))
+        if cli.otlp {
+            tvix_tracing::init_with_otlp(cli.log_level, "tvix.store")?;
         } else {
-            subscriber.with(None)
-        };
-
-        subscriber.try_init()?;
+            tvix_tracing::init(cli.log_level)?;
+        }
     }
 
-    // Init the registry (when otlp is not enabled)
     #[cfg(not(feature = "otlp"))]
     {
-        subscriber.try_init()?;
+        tvix_tracing::init(cli.log_level)?;
     }
 
     match cli.command {
@@ -383,7 +296,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
             let root_span = {
                 let s = Span::current();
-                s.pb_set_style(&PB_PROGRESS_STYLE);
+                s.pb_set_style(&tvix_tracing::PB_PROGRESS_STYLE);
                 s.pb_set_message("Importing paths");
                 s.pb_set_length(paths.len() as u64);
                 s.pb_start();
@@ -458,7 +371,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 "indicatif.pb_show" = tracing::field::Empty
             );
             lookups_span.pb_set_length(reference_graph.closure.len() as u64);
-            lookups_span.pb_set_style(&PB_PROGRESS_STYLE);
+            lookups_span.pb_set_style(&tvix_tracing::PB_PROGRESS_STYLE);
             lookups_span.pb_start();
 
             // From our reference graph, lookup all pathinfos that might exist.
diff --git a/tvix/store/src/pathinfoservice/sled.rs b/tvix/store/src/pathinfoservice/sled.rs
index eb3cf2ff1b88..96ade181694c 100644
--- a/tvix/store/src/pathinfoservice/sled.rs
+++ b/tvix/store/src/pathinfoservice/sled.rs
@@ -6,8 +6,7 @@ use nix_compat::nixbase32;
 use prost::Message;
 use std::path::Path;
 use tonic::async_trait;
-use tracing::instrument;
-use tracing::warn;
+use tracing::{instrument, warn};
 use tvix_castore::Error;
 
 /// SledPathInfoService stores PathInfo in a [sled](https://github.com/spacejam/sled).
diff --git a/tvix/tracing/Cargo.toml b/tvix/tracing/Cargo.toml
new file mode 100644
index 000000000000..eea6ca17711f
--- /dev/null
+++ b/tvix/tracing/Cargo.toml
@@ -0,0 +1,28 @@
+[package]
+name = "tvix-tracing"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+lazy_static = "1.4.0"
+tracing = { version = "0.1.40", features = ["max_level_trace", "release_max_level_info"] }
+tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
+indicatif = "0.17.8"
+tracing-indicatif = "0.3.6"
+
+tracing-opentelemetry = { version = "0.23.0", optional = true }
+opentelemetry = { version = "0.22.0", optional = true }
+opentelemetry-otlp = { version = "0.15.0", optional = true }
+opentelemetry_sdk = { version = "0.22.1", features = ["rt-tokio"], optional = true }
+
+[features]
+default = []
+otlp = [
+  "dep:tracing-opentelemetry",
+  "dep:opentelemetry",
+  "dep:opentelemetry-otlp",
+  "dep:opentelemetry_sdk"
+]
+
+[lints]
+workspace = true
diff --git a/tvix/tracing/default.nix b/tvix/tracing/default.nix
new file mode 100644
index 000000000000..a4fe3a5d90c3
--- /dev/null
+++ b/tvix/tracing/default.nix
@@ -0,0 +1,11 @@
+{ depot, lib, ... }:
+
+(depot.tvix.crates.workspaceMembers.tvix-tracing.build.override {
+  runTests = true;
+}).overrideAttrs (old: rec {
+  meta.ci.targets = lib.filter (x: lib.hasPrefix "with-features" x || x == "no-features") (lib.attrNames passthru);
+  passthru = depot.tvix.utils.mkFeaturePowerset {
+    inherit (old) crateName;
+    features = [ "otlp" ];
+  };
+})
diff --git a/tvix/tracing/src/lib.rs b/tvix/tracing/src/lib.rs
new file mode 100644
index 000000000000..8ad90835338e
--- /dev/null
+++ b/tvix/tracing/src/lib.rs
@@ -0,0 +1,90 @@
+use indicatif::ProgressStyle;
+use lazy_static::lazy_static;
+use tracing::Level;
+use tracing_indicatif::{filter::IndicatifFilter, IndicatifLayer};
+use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer};
+
+#[cfg(feature = "otlp")]
+use opentelemetry::KeyValue;
+#[cfg(feature = "otlp")]
+use opentelemetry_sdk::{
+    resource::{ResourceDetector, SdkProvidedResourceDetector},
+    trace::BatchConfig,
+    Resource,
+};
+
+lazy_static! {
+    pub static ref PB_PROGRESS_STYLE: ProgressStyle = ProgressStyle::with_template(
+        "{span_child_prefix}{bar:30} {wide_msg} [{elapsed_precise}]  {pos:>7}/{len:7}"
+    )
+    .expect("invalid progress template");
+    pub static ref PB_SPINNER_STYLE: ProgressStyle = ProgressStyle::with_template(
+        "{span_child_prefix}{spinner} {wide_msg} [{elapsed_precise}]  {pos:>7}/{len:7}"
+    )
+    .expect("invalid progress template");
+}
+
+// using a macro_rule here because of the complex return type
+macro_rules! init_base_subscriber {
+    ($level: expr) => {{
+        let indicatif_layer = IndicatifLayer::new().with_progress_style(PB_SPINNER_STYLE.clone());
+
+        // Set up the tracing subscriber.
+        tracing_subscriber::registry()
+            .with(
+                tracing_subscriber::fmt::Layer::new()
+                    .with_writer(indicatif_layer.get_stderr_writer())
+                    .compact()
+                    .with_filter(
+                        EnvFilter::builder()
+                            .with_default_directive($level.into())
+                            .from_env()
+                            .expect("invalid RUST_LOG"),
+                    ),
+            )
+            .with(indicatif_layer.with_filter(
+                // only show progress for spans with indicatif.pb_show field being set
+                IndicatifFilter::new(false),
+            ))
+    }};
+}
+
+pub fn init(level: Level) -> Result<(), tracing_subscriber::util::TryInitError> {
+    init_base_subscriber!(level).try_init()
+}
+
+#[cfg(feature = "otlp")]
+pub fn init_with_otlp(
+    level: Level,
+    service_name: &'static str,
+) -> Result<(), tracing_subscriber::util::TryInitError> {
+    let subscriber = init_base_subscriber!(level);
+
+    let tracer = opentelemetry_otlp::new_pipeline()
+        .tracing()
+        .with_exporter(opentelemetry_otlp::new_exporter().tonic())
+        .with_batch_config(BatchConfig::default())
+        .with_trace_config(opentelemetry_sdk::trace::config().with_resource({
+            // use SdkProvidedResourceDetector.detect to detect resources,
+            // but replace the default service name with our default.
+            // https://github.com/open-telemetry/opentelemetry-rust/issues/1298
+            let resources = SdkProvidedResourceDetector.detect(std::time::Duration::from_secs(0));
+            // SdkProvidedResourceDetector currently always sets
+            // `service.name`, but we don't like its default.
+            if resources.get("service.name".into()).unwrap() == "unknown_service".into() {
+                resources.merge(&Resource::new([KeyValue::new(
+                    "service.name",
+                    service_name,
+                )]))
+            } else {
+                resources
+            }
+        }))
+        .install_batch(opentelemetry_sdk::runtime::Tokio)
+        .expect("Failed to install tokio runtime");
+
+    // Create a tracing layer with the configured tracer
+    let layer = tracing_opentelemetry::layer().with_tracer(tracer);
+
+    subscriber.with(Some(layer)).try_init()
+}