refactor(tvix): move tvix glue code into glue crate r/6936

There's various bits and pieces in tvix-cli that use both the store and
evaluator, as well as nix-compat. For example, builtins.derivation, as
well as the reference scanning implementation.

This "glue code" currently isn't accessible from anywhere else, but it'd
be very useful if it were.

Move it out into a `glue` crate, and make `tvix-cli` a consumer of it.

All the KnownPaths setup and passing around, as well as NIX_PATH
handling is also something that should probably be moved into the glue
crate as well, but that's something left for a future CL.

Change-Id: I080ed3d1825ab23790666486840f301f00856277
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9908
Autosubmit: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
Reviewed-by: raitobezarius <tvl@lahfa.xyz>

17 files changed, 154 insertions, 22 deletions

diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock
index 4748c3abce..9dd66f83fe 100644
--- a/tvix/Cargo.lock
+++ b/tvix/Cargo.lock
@@ -2899,6 +2899,7 @@ dependencies = [
  "tracing",
  "tvix-castore",
  "tvix-eval",
+ "tvix-glue",
  "tvix-store",
  "wu-manber",
 ]
@@ -2946,6 +2947,22 @@ dependencies = [
 ]
 
 [[package]]
+name = "tvix-glue"
+version = "0.1.0"
+dependencies = [
+ "bytes",
+ "nix-compat",
+ "test-case",
+ "thiserror",
+ "tokio",
+ "tracing",
+ "tvix-castore",
+ "tvix-eval",
+ "tvix-store",
+ "wu-manber",
+]
+
+[[package]]
 name = "tvix-serde"
 version = "0.1.0"
 dependencies = [
diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix
index dab411ebb8..ad3eac33b6 100644
--- a/tvix/Cargo.nix
+++ b/tvix/Cargo.nix
@@ -93,6 +93,16 @@ rec {
       # File a bug if you depend on any for non-debug work!
       debug = internal.debugCrate { inherit packageId; };
     };
+    "tvix-glue" = rec {
+      packageId = "tvix-glue";
+      build = internal.buildRustCrateWithFeatures {
+        packageId = "tvix-glue";
+      };
+
+      # Debug support which might change between releases.
+      # File a bug if you depend on any for non-debug work!
+      debug = internal.debugCrate { inherit packageId; };
+    };
     "tvix-serde" = rec {
       packageId = "tvix-serde";
       build = internal.buildRustCrateWithFeatures {
@@ -8731,6 +8741,10 @@ rec {
             packageId = "tvix-eval";
           }
           {
+            name = "tvix-glue";
+            packageId = "tvix-glue";
+          }
+          {
             name = "tvix-store";
             packageId = "tvix-store";
             usesDefaultFeatures = false;
@@ -8921,6 +8935,63 @@ rec {
         ];
 
       };
+      "tvix-glue" = rec {
+        crateName = "tvix-glue";
+        version = "0.1.0";
+        edition = "2021";
+        # We can't filter paths with references in Nix 2.4
+        # See https://github.com/NixOS/nix/issues/5410
+        src =
+          if ((lib.versionOlder builtins.nixVersion "2.4pre20211007") || (lib.versionOlder "2.5" builtins.nixVersion))
+          then lib.cleanSourceWith { filter = sourceFilter; src = ./glue; }
+          else ./glue;
+        dependencies = [
+          {
+            name = "bytes";
+            packageId = "bytes";
+          }
+          {
+            name = "nix-compat";
+            packageId = "nix-compat";
+          }
+          {
+            name = "thiserror";
+            packageId = "thiserror";
+          }
+          {
+            name = "tokio";
+            packageId = "tokio";
+          }
+          {
+            name = "tracing";
+            packageId = "tracing";
+          }
+          {
+            name = "tvix-castore";
+            packageId = "tvix-castore";
+          }
+          {
+            name = "tvix-eval";
+            packageId = "tvix-eval";
+          }
+          {
+            name = "tvix-store";
+            packageId = "tvix-store";
+            usesDefaultFeatures = false;
+          }
+          {
+            name = "wu-manber";
+            packageId = "wu-manber";
+          }
+        ];
+        devDependencies = [
+          {
+            name = "test-case";
+            packageId = "test-case";
+          }
+        ];
+
+      };
       "tvix-serde" = rec {
         crateName = "tvix-serde";
         version = "0.1.0";
diff --git a/tvix/Cargo.toml b/tvix/Cargo.toml
index 645aac3627..23486a921b 100644
--- a/tvix/Cargo.toml
+++ b/tvix/Cargo.toml
@@ -23,6 +23,7 @@ members = [
   "cli",
   "eval",
   "eval/builtin-macros",
+  "glue",
   "nix_cli",
   "nix-compat",
   "serde",
diff --git a/tvix/cli/Cargo.toml b/tvix/cli/Cargo.toml
index 8bdd26ff45..7a27c148f4 100644
--- a/tvix/cli/Cargo.toml
+++ b/tvix/cli/Cargo.toml
@@ -12,6 +12,7 @@ nix-compat = { path = "../nix-compat" }
 tvix-castore = { path = "../castore" }
 tvix-store = { path = "../store", default-features = false, features = []}
 tvix-eval = { path = "../eval" }
+tvix-glue = { path = "../glue" }
 bytes = "1.4.0"
 clap = { version = "4.0", features = ["derive", "env"] }
 dirs = "4.0.0"
diff --git a/tvix/cli/src/main.rs b/tvix/cli/src/main.rs
index ebcfe4b800..1ffa2be256 100644
--- a/tvix/cli/src/main.rs
+++ b/tvix/cli/src/main.rs
@@ -1,24 +1,18 @@
-mod derivation;
-mod errors;
-mod known_paths;
-mod refscan;
-mod tvix_io;
-mod tvix_store_io;
-
 use std::cell::RefCell;
 use std::rc::Rc;
 use std::sync::Arc;
 use std::{fs, path::PathBuf};
+use tvix_glue::add_derivation_builtins;
+use tvix_glue::known_paths::KnownPaths;
 
 use clap::Parser;
-use known_paths::KnownPaths;
 use rustyline::{error::ReadlineError, Editor};
 use tvix_castore::blobservice::MemoryBlobService;
 use tvix_castore::directoryservice::MemoryDirectoryService;
 use tvix_eval::observer::{DisassemblingObserver, TracingObserver};
 use tvix_eval::Value;
+use tvix_glue::tvix_store_io::TvixStoreIO;
 use tvix_store::pathinfoservice::MemoryPathInfoService;
-use tvix_store_io::TvixStoreIO;
 
 #[derive(Parser)]
 struct Args {
@@ -69,7 +63,6 @@ struct Args {
 /// evaluation succeeded.
 fn interpret(code: &str, path: Option<PathBuf>, args: &Args, explain: bool) -> bool {
     let mut eval = tvix_eval::Evaluation::new_impure(code, path);
-    let known_paths: Rc<RefCell<KnownPaths>> = Default::default();
 
     eval.strict = args.strict;
 
@@ -80,9 +73,11 @@ fn interpret(code: &str, path: Option<PathBuf>, args: &Args, explain: bool) -> b
         directory_service.clone(),
     ));
 
-    let tokio_runtime = tokio::runtime::Runtime::new().unwrap();
+    let known_paths: Rc<RefCell<KnownPaths>> = Default::default();
+    add_derivation_builtins(&mut eval, known_paths.clone());
 
-    eval.io_handle = Box::new(tvix_io::TvixIO::new(
+    let tokio_runtime = tokio::runtime::Runtime::new().unwrap();
+    eval.io_handle = Box::new(tvix_glue::tvix_io::TvixIO::new(
         known_paths.clone(),
         TvixStoreIO::new(
             blob_service,
@@ -100,13 +95,6 @@ fn interpret(code: &str, path: Option<PathBuf>, args: &Args, explain: bool) -> b
         .map(|p| format!("nix=/__corepkgs__:{}", p))
         .or_else(|| Some("nix=/__corepkgs__".to_string()));
 
-    eval.builtins
-        .extend(derivation::derivation_builtins(known_paths));
-
-    // Add the actual `builtins.derivation` from compiled Nix code
-    eval.src_builtins
-        .push(("derivation", include_str!("derivation.nix")));
-
     let source_map = eval.source_map();
     let result = {
         let mut compiler_observer =
diff --git a/tvix/glue/Cargo.toml b/tvix/glue/Cargo.toml
new file mode 100644
index 0000000000..4ebfda8703
--- /dev/null
+++ b/tvix/glue/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+name = "tvix-glue"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+nix-compat = { path = "../nix-compat" }
+tvix-eval = { path = "../eval" }
+tvix-castore = { path = "../castore" }
+tvix-store = { path = "../store", default-features = false, features = []}
+bytes = "1.4.0"
+tracing = "0.1.37"
+tokio = "1.28.0"
+thiserror = "1.0.38"
+
+[dependencies.wu-manber]
+git = "https://github.com/tvlfyi/wu-manber.git"
+
+[dev-dependencies]
+test-case = "2.2.2"
diff --git a/tvix/glue/default.nix b/tvix/glue/default.nix
new file mode 100644
index 0000000000..975b0a5a70
--- /dev/null
+++ b/tvix/glue/default.nix
@@ -0,0 +1,5 @@
+{ depot, ... }:
+
+(depot.tvix.crates.workspaceMembers.tvix-glue.build.override {
+  runTests = true;
+})
diff --git a/tvix/cli/src/.skip-subtree b/tvix/glue/src/.skip-subtree
index a16a2afe1f..a16a2afe1f 100644
--- a/tvix/cli/src/.skip-subtree
+++ b/tvix/glue/src/.skip-subtree
diff --git a/tvix/cli/src/derivation.nix b/tvix/glue/src/derivation.nix
index 9355cc3a96..9355cc3a96 100644
--- a/tvix/cli/src/derivation.nix
+++ b/tvix/glue/src/derivation.nix
diff --git a/tvix/cli/src/derivation.rs b/tvix/glue/src/derivation.rs
index 86a271a396..86a271a396 100644
--- a/tvix/cli/src/derivation.rs
+++ b/tvix/glue/src/derivation.rs
diff --git a/tvix/cli/src/errors.rs b/tvix/glue/src/errors.rs
index 5cbddcbba8..5cbddcbba8 100644
--- a/tvix/cli/src/errors.rs
+++ b/tvix/glue/src/errors.rs
diff --git a/tvix/cli/src/fetchurl.nix b/tvix/glue/src/fetchurl.nix
index 3f182a5a31..3f182a5a31 100644
--- a/tvix/cli/src/fetchurl.nix
+++ b/tvix/glue/src/fetchurl.nix
diff --git a/tvix/cli/src/known_paths.rs b/tvix/glue/src/known_paths.rs
index 07373ef0da..07373ef0da 100644
--- a/tvix/cli/src/known_paths.rs
+++ b/tvix/glue/src/known_paths.rs
diff --git a/tvix/glue/src/lib.rs b/tvix/glue/src/lib.rs
new file mode 100644
index 0000000000..acb81d3144
--- /dev/null
+++ b/tvix/glue/src/lib.rs
@@ -0,0 +1,28 @@
+use std::{cell::RefCell, rc::Rc};
+
+use known_paths::KnownPaths;
+
+pub mod derivation;
+pub mod errors;
+pub mod known_paths;
+pub mod refscan;
+pub mod tvix_io;
+pub mod tvix_store_io;
+
+/// Adds derivation-related builtins to the passed [tvix_eval::Evaluation].
+///
+/// These are `derivation` and `derivationStrict`.
+///
+/// As they need to interact with `known_paths`, we also need to pass in
+/// `known_paths`.
+pub fn add_derivation_builtins(
+    eval: &mut tvix_eval::Evaluation,
+    known_paths: Rc<RefCell<KnownPaths>>,
+) {
+    eval.builtins
+        .extend(derivation::derivation_builtins(known_paths));
+
+    // Add the actual `builtins.derivation` from compiled Nix code
+    eval.src_builtins
+        .push(("derivation", include_str!("derivation.nix")));
+}
diff --git a/tvix/cli/src/refscan.rs b/tvix/glue/src/refscan.rs
index 0e0bb6c778..0e0bb6c778 100644
--- a/tvix/cli/src/refscan.rs
+++ b/tvix/glue/src/refscan.rs
diff --git a/tvix/cli/src/tvix_io.rs b/tvix/glue/src/tvix_io.rs
index 74f91138bf..caadbeb5e6 100644
--- a/tvix/cli/src/tvix_io.rs
+++ b/tvix/glue/src/tvix_io.rs
@@ -8,15 +8,16 @@
 //! otherwise fundamental features like nixpkgs bootstrapping and hash
 //! calculation will not work.
 
-use crate::KnownPaths;
 use std::cell::RefCell;
 use std::io;
 use std::path::{Path, PathBuf};
 use std::rc::Rc;
 use tvix_eval::{EvalIO, FileType};
 
+use crate::known_paths::KnownPaths;
+
 // TODO: Merge this together with TvixStoreIO?
-pub(crate) struct TvixIO<T: EvalIO> {
+pub struct TvixIO<T: EvalIO> {
     /// Ingested paths must be reported to this known paths tracker
     /// for accurate build reference scanning.
     known_paths: Rc<RefCell<KnownPaths>>,
@@ -26,7 +27,7 @@ pub(crate) struct TvixIO<T: EvalIO> {
 }
 
 impl<T: EvalIO> TvixIO<T> {
-    pub(crate) fn new(known_paths: Rc<RefCell<KnownPaths>>, actual: T) -> Self {
+    pub fn new(known_paths: Rc<RefCell<KnownPaths>>, actual: T) -> Self {
         Self {
             known_paths,
             actual,
diff --git a/tvix/cli/src/tvix_store_io.rs b/tvix/glue/src/tvix_store_io.rs
index 9be896ffc4..9be896ffc4 100644
--- a/tvix/cli/src/tvix_store_io.rs
+++ b/tvix/glue/src/tvix_store_io.rs