refactor(tvix): move tvix glue code into glue crate r/6936

There's various bits and pieces in tvix-cli that use both the store and
evaluator, as well as nix-compat. For example, builtins.derivation, as
well as the reference scanning implementation.

This "glue code" currently isn't accessible from anywhere else, but it'd
be very useful if it were.

Move it out into a `glue` crate, and make `tvix-cli` a consumer of it.

All the KnownPaths setup and passing around, as well as NIX_PATH
handling is also something that should probably be moved into the glue
crate as well, but that's something left for a future CL.

Change-Id: I080ed3d1825ab23790666486840f301f00856277
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9908
Autosubmit: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
Reviewed-by: raitobezarius <tvl@lahfa.xyz>

17 files changed, 154 insertions, 22 deletions

diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock
index 4748c3abce41..9dd66f83fe35 100644
--- a/tvix/Cargo.lock
+++ b/tvix/Cargo.lock
@@ -2899,6 +2899,7 @@ dependencies = [
  "tracing",
  "tvix-castore",
  "tvix-eval",
+ "tvix-glue",
  "tvix-store",
  "wu-manber",
 ]
@@ -2946,6 +2947,22 @@ dependencies = [
 ]
 
 [[package]]
+name = "tvix-glue"
+version = "0.1.0"
+dependencies = [
+ "bytes",
+ "nix-compat",
+ "test-case",
+ "thiserror",
+ "tokio",
+ "tracing",
+ "tvix-castore",
+ "tvix-eval",
+ "tvix-store",
+ "wu-manber",
+]
+
+[[package]]
 name = "tvix-serde"
 version = "0.1.0"
 dependencies = [
diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix
index dab411ebb863..ad3eac33b665 100644
--- a/tvix/Cargo.nix
+++ b/tvix/Cargo.nix
@@ -93,6 +93,16 @@ rec {
       # File a bug if you depend on any for non-debug work!
       debug = internal.debugCrate { inherit packageId; };
     };
+    "tvix-glue" = rec {
+      packageId = "tvix-glue";
+      build = internal.buildRustCrateWithFeatures {
+        packageId = "tvix-glue";
+      };
+
+      # Debug support which might change between releases.
+      # File a bug if you depend on any for non-debug work!
+      debug = internal.debugCrate { inherit packageId; };
+    };
     "tvix-serde" = rec {
       packageId = "tvix-serde";
       build = internal.buildRustCrateWithFeatures {
@@ -8731,6 +8741,10 @@ rec {
             packageId = "tvix-eval";
           }
           {
+            name = "tvix-glue";
+            packageId = "tvix-glue";
+          }
+          {
             name = "tvix-store";
             packageId = "tvix-store";
             usesDefaultFeatures = false;
@@ -8921,6 +8935,63 @@ rec {
         ];
 
       };
+      "tvix-glue" = rec {
+        crateName = "tvix-glue";
+        version = "0.1.0";
+        edition = "2021";
+        # We can't filter paths with references in Nix 2.4
+        # See https://github.com/NixOS/nix/issues/5410
+        src =
+          if ((lib.versionOlder builtins.nixVersion "2.4pre20211007") || (lib.versionOlder "2.5" builtins.nixVersion))
+          then lib.cleanSourceWith { filter = sourceFilter; src = ./glue; }
+          else ./glue;
+        dependencies = [
+          {
+            name = "bytes";
+            packageId = "bytes";
+          }
+          {
+            name = "nix-compat";
+            packageId = "nix-compat";
+          }
+          {
+            name = "thiserror";
+            packageId = "thiserror";
+          }
+          {
+            name = "tokio";
+            packageId = "tokio";
+          }
+          {
+            name = "tracing";
+            packageId = "tracing";
+          }
+          {
+            name = "tvix-castore";
+            packageId = "tvix-castore";
+          }
+          {
+            name = "tvix-eval";
+            packageId = "tvix-eval";
+          }
+          {
+            name = "tvix-store";
+            packageId = "tvix-store";
+            usesDefaultFeatures = false;
+          }
+          {
+            name = "wu-manber";
+            packageId = "wu-manber";
+          }
+        ];
+        devDependencies = [
+          {
+            name = "test-case";
+            packageId = "test-case";
+          }
+        ];
+
+      };
       "tvix-serde" = rec {
         crateName = "tvix-serde";
         version = "0.1.0";
diff --git a/tvix/Cargo.toml b/tvix/Cargo.toml
index 645aac362738..23486a921b9d 100644
--- a/tvix/Cargo.toml
+++ b/tvix/Cargo.toml
@@ -23,6 +23,7 @@ members = [
   "cli",
   "eval",
   "eval/builtin-macros",
+  "glue",
   "nix_cli",
   "nix-compat",
   "serde",
diff --git a/tvix/cli/Cargo.toml b/tvix/cli/Cargo.toml
index 8bdd26ff459b..7a27c148f45d 100644
--- a/tvix/cli/Cargo.toml
+++ b/tvix/cli/Cargo.toml
@@ -12,6 +12,7 @@ nix-compat = { path = "../nix-compat" }
 tvix-castore = { path = "../castore" }
 tvix-store = { path = "../store", default-features = false, features = []}
 tvix-eval = { path = "../eval" }
+tvix-glue = { path = "../glue" }
 bytes = "1.4.0"
 clap = { version = "4.0", features = ["derive", "env"] }
 dirs = "4.0.0"
diff --git a/tvix/cli/src/main.rs b/tvix/cli/src/main.rs
index ebcfe4b800b8..1ffa2be256ac 100644
--- a/tvix/cli/src/main.rs
+++ b/tvix/cli/src/main.rs
@@ -1,24 +1,18 @@
-mod derivation;
-mod errors;
-mod known_paths;
-mod refscan;
-mod tvix_io;
-mod tvix_store_io;
-
 use std::cell::RefCell;
 use std::rc::Rc;
 use std::sync::Arc;
 use std::{fs, path::PathBuf};
+use tvix_glue::add_derivation_builtins;
+use tvix_glue::known_paths::KnownPaths;
 
 use clap::Parser;
-use known_paths::KnownPaths;
 use rustyline::{error::ReadlineError, Editor};
 use tvix_castore::blobservice::MemoryBlobService;
 use tvix_castore::directoryservice::MemoryDirectoryService;
 use tvix_eval::observer::{DisassemblingObserver, TracingObserver};
 use tvix_eval::Value;
+use tvix_glue::tvix_store_io::TvixStoreIO;
 use tvix_store::pathinfoservice::MemoryPathInfoService;
-use tvix_store_io::TvixStoreIO;
 
 #[derive(Parser)]
 struct Args {
@@ -69,7 +63,6 @@ struct Args {
 /// evaluation succeeded.
 fn interpret(code: &str, path: Option<PathBuf>, args: &Args, explain: bool) -> bool {
     let mut eval = tvix_eval::Evaluation::new_impure(code, path);
-    let known_paths: Rc<RefCell<KnownPaths>> = Default::default();
 
     eval.strict = args.strict;
 
@@ -80,9 +73,11 @@ fn interpret(code: &str, path: Option<PathBuf>, args: &Args, explain: bool) -> b
         directory_service.clone(),
     ));
 
-    let tokio_runtime = tokio::runtime::Runtime::new().unwrap();
+    let known_paths: Rc<RefCell<KnownPaths>> = Default::default();
+    add_derivation_builtins(&mut eval, known_paths.clone());
 
-    eval.io_handle = Box::new(tvix_io::TvixIO::new(
+    let tokio_runtime = tokio::runtime::Runtime::new().unwrap();
+    eval.io_handle = Box::new(tvix_glue::tvix_io::TvixIO::new(
         known_paths.clone(),
         TvixStoreIO::new(
             blob_service,
@@ -100,13 +95,6 @@ fn interpret(code: &str, path: Option<PathBuf>, args: &Args, explain: bool) -> b
         .map(|p| format!("nix=/__corepkgs__:{}", p))
         .or_else(|| Some("nix=/__corepkgs__".to_string()));
 
-    eval.builtins
-        .extend(derivation::derivation_builtins(known_paths));
-
-    // Add the actual `builtins.derivation` from compiled Nix code
-    eval.src_builtins
-        .push(("derivation", include_str!("derivation.nix")));
-
     let source_map = eval.source_map();
     let result = {
         let mut compiler_observer =
diff --git a/tvix/glue/Cargo.toml b/tvix/glue/Cargo.toml
new file mode 100644
index 000000000000..4ebfda870399
--- /dev/null
+++ b/tvix/glue/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+name = "tvix-glue"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+nix-compat = { path = "../nix-compat" }
+tvix-eval = { path = "../eval" }
+tvix-castore = { path = "../castore" }
+tvix-store = { path = "../store", default-features = false, features = []}
+bytes = "1.4.0"
+tracing = "0.1.37"
+tokio = "1.28.0"
+thiserror = "1.0.38"
+
+[dependencies.wu-manber]
+git = "https://github.com/tvlfyi/wu-manber.git"
+
+[dev-dependencies]
+test-case = "2.2.2"
diff --git a/tvix/glue/default.nix b/tvix/glue/default.nix
new file mode 100644
index 000000000000..975b0a5a7007
--- /dev/null
+++ b/tvix/glue/default.nix
@@ -0,0 +1,5 @@
+{ depot, ... }:
+
+(depot.tvix.crates.workspaceMembers.tvix-glue.build.override {
+  runTests = true;
+})
diff --git a/tvix/cli/src/.skip-subtree b/tvix/glue/src/.skip-subtree
index a16a2afe1f1e..a16a2afe1f1e 100644
--- a/tvix/cli/src/.skip-subtree
+++ b/tvix/glue/src/.skip-subtree
diff --git a/tvix/cli/src/derivation.nix b/tvix/glue/src/derivation.nix
index 9355cc3a96f0..9355cc3a96f0 100644
--- a/tvix/cli/src/derivation.nix
+++ b/tvix/glue/src/derivation.nix
diff --git a/tvix/cli/src/derivation.rs b/tvix/glue/src/derivation.rs
index 86a271a3966d..86a271a3966d 100644
--- a/tvix/cli/src/derivation.rs
+++ b/tvix/glue/src/derivation.rs
diff --git a/tvix/cli/src/errors.rs b/tvix/glue/src/errors.rs
index 5cbddcbba811..5cbddcbba811 100644
--- a/tvix/cli/src/errors.rs
+++ b/tvix/glue/src/errors.rs
diff --git a/tvix/cli/src/fetchurl.nix b/tvix/glue/src/fetchurl.nix
index 3f182a5a319b..3f182a5a319b 100644
--- a/tvix/cli/src/fetchurl.nix
+++ b/tvix/glue/src/fetchurl.nix
diff --git a/tvix/cli/src/known_paths.rs b/tvix/glue/src/known_paths.rs
index 07373ef0da7a..07373ef0da7a 100644
--- a/tvix/cli/src/known_paths.rs
+++ b/tvix/glue/src/known_paths.rs
diff --git a/tvix/glue/src/lib.rs b/tvix/glue/src/lib.rs
new file mode 100644
index 000000000000..acb81d31445f
--- /dev/null
+++ b/tvix/glue/src/lib.rs
@@ -0,0 +1,28 @@
+use std::{cell::RefCell, rc::Rc};
+
+use known_paths::KnownPaths;
+
+pub mod derivation;
+pub mod errors;
+pub mod known_paths;
+pub mod refscan;
+pub mod tvix_io;
+pub mod tvix_store_io;
+
+/// Adds derivation-related builtins to the passed [tvix_eval::Evaluation].
+///
+/// These are `derivation` and `derivationStrict`.
+///
+/// As they need to interact with `known_paths`, we also need to pass in
+/// `known_paths`.
+pub fn add_derivation_builtins(
+    eval: &mut tvix_eval::Evaluation,
+    known_paths: Rc<RefCell<KnownPaths>>,
+) {
+    eval.builtins
+        .extend(derivation::derivation_builtins(known_paths));
+
+    // Add the actual `builtins.derivation` from compiled Nix code
+    eval.src_builtins
+        .push(("derivation", include_str!("derivation.nix")));
+}
diff --git a/tvix/cli/src/refscan.rs b/tvix/glue/src/refscan.rs
index 0e0bb6c77828..0e0bb6c77828 100644
--- a/tvix/cli/src/refscan.rs
+++ b/tvix/glue/src/refscan.rs
diff --git a/tvix/cli/src/tvix_io.rs b/tvix/glue/src/tvix_io.rs
index 74f91138bff8..caadbeb5e663 100644
--- a/tvix/cli/src/tvix_io.rs
+++ b/tvix/glue/src/tvix_io.rs
@@ -8,15 +8,16 @@
 //! otherwise fundamental features like nixpkgs bootstrapping and hash
 //! calculation will not work.
 
-use crate::KnownPaths;
 use std::cell::RefCell;
 use std::io;
 use std::path::{Path, PathBuf};
 use std::rc::Rc;
 use tvix_eval::{EvalIO, FileType};
 
+use crate::known_paths::KnownPaths;
+
 // TODO: Merge this together with TvixStoreIO?
-pub(crate) struct TvixIO<T: EvalIO> {
+pub struct TvixIO<T: EvalIO> {
     /// Ingested paths must be reported to this known paths tracker
     /// for accurate build reference scanning.
     known_paths: Rc<RefCell<KnownPaths>>,
@@ -26,7 +27,7 @@ pub(crate) struct TvixIO<T: EvalIO> {
 }
 
 impl<T: EvalIO> TvixIO<T> {
-    pub(crate) fn new(known_paths: Rc<RefCell<KnownPaths>>, actual: T) -> Self {
+    pub fn new(known_paths: Rc<RefCell<KnownPaths>>, actual: T) -> Self {
         Self {
             known_paths,
             actual,
diff --git a/tvix/cli/src/tvix_store_io.rs b/tvix/glue/src/tvix_store_io.rs
index 9be896ffc40f..9be896ffc40f 100644
--- a/tvix/cli/src/tvix_store_io.rs
+++ b/tvix/glue/src/tvix_store_io.rs