about summary refs log tree commit diff
diff options
context:
space:
mode:
-rw-r--r--corp/tvixbolt/Cargo.lock78
-rw-r--r--tvix/Cargo.lock80
-rw-r--r--tvix/Cargo.nix207
-rw-r--r--tvix/eval/Cargo.toml1
-rw-r--r--tvix/eval/src/value/mod.rs2
-rw-r--r--tvix/eval/src/vm.rs2
-rw-r--r--tvix/eval/src/vm/generators.rs538
7 files changed, 897 insertions, 11 deletions
diff --git a/corp/tvixbolt/Cargo.lock b/corp/tvixbolt/Cargo.lock
index 479e4d49e585..11ff5336167d 100644
--- a/corp/tvixbolt/Cargo.lock
+++ b/corp/tvixbolt/Cargo.lock
@@ -142,6 +142,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608"
 
 [[package]]
+name = "genawaiter"
+version = "0.99.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c86bd0361bcbde39b13475e6e36cb24c329964aa2611be285289d1e4b751c1a0"
+dependencies = [
+ "genawaiter-macro",
+ "genawaiter-proc-macro",
+ "proc-macro-hack",
+]
+
+[[package]]
+name = "genawaiter-macro"
+version = "0.99.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b32dfe1fdfc0bbde1f22a5da25355514b5e450c33a6af6770884c8750aedfbc"
+
+[[package]]
+name = "genawaiter-proc-macro"
+version = "0.99.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "784f84eebc366e15251c4a8c3acee82a6a6f427949776ecb88377362a9621738"
+dependencies = [
+ "proc-macro-error 0.4.12",
+ "proc-macro-hack",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
 name = "getrandom"
 version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -456,14 +486,40 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
 
 [[package]]
 name = "proc-macro-error"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "18f33027081eba0a6d8aba6d1b1c3a3be58cbb12106341c2d5759fcd9b5277e7"
+dependencies = [
+ "proc-macro-error-attr 0.4.12",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro-error"
 version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
 dependencies = [
- "proc-macro-error-attr",
+ "proc-macro-error-attr 1.0.4",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro-error-attr"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a5b4b77fdb63c1eca72173d68d24501c54ab1269409f6b672c85deb18af69de"
+dependencies = [
  "proc-macro2",
  "quote",
  "syn",
+ "syn-mid",
  "version_check",
 ]
 
@@ -479,6 +535,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "proc-macro-hack"
+version = "0.5.20+deprecated"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
+
+[[package]]
 name = "proc-macro2"
 version = "1.0.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -694,6 +756,17 @@ dependencies = [
 ]
 
 [[package]]
+name = "syn-mid"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baa8e7560a164edb1621a55d18a0c59abf49d360f47aa7b821061dd7eea7fac9"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
 name = "tabwriter"
 version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -778,6 +851,7 @@ dependencies = [
  "codemap",
  "codemap-diagnostic",
  "dirs",
+ "genawaiter",
  "imbl",
  "lazy_static",
  "lexical-core",
@@ -982,7 +1056,7 @@ checksum = "5fab79082b556d768d6e21811869c761893f0450e1d550a67892b9bce303b7bb"
 dependencies = [
  "boolinator",
  "lazy_static",
- "proc-macro-error",
+ "proc-macro-error 1.0.4",
  "proc-macro2",
  "quote",
  "syn",
diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock
index fa3ac2269b5b..913d88b9fc12 100644
--- a/tvix/Cargo.lock
+++ b/tvix/Cargo.lock
@@ -364,7 +364,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8"
 dependencies = [
  "heck",
- "proc-macro-error",
+ "proc-macro-error 1.0.4",
  "proc-macro2 1.0.50",
  "quote 1.0.23",
  "syn 1.0.107",
@@ -820,6 +820,36 @@ dependencies = [
 ]
 
 [[package]]
+name = "genawaiter"
+version = "0.99.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c86bd0361bcbde39b13475e6e36cb24c329964aa2611be285289d1e4b751c1a0"
+dependencies = [
+ "genawaiter-macro",
+ "genawaiter-proc-macro",
+ "proc-macro-hack",
+]
+
+[[package]]
+name = "genawaiter-macro"
+version = "0.99.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b32dfe1fdfc0bbde1f22a5da25355514b5e450c33a6af6770884c8750aedfbc"
+
+[[package]]
+name = "genawaiter-proc-macro"
+version = "0.99.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "784f84eebc366e15251c4a8c3acee82a6a6f427949776ecb88377362a9621738"
+dependencies = [
+ "proc-macro-error 0.4.12",
+ "proc-macro-hack",
+ "proc-macro2 1.0.50",
+ "quote 1.0.23",
+ "syn 1.0.107",
+]
+
+[[package]]
 name = "generic-array"
 version = "0.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1546,14 +1576,40 @@ dependencies = [
 
 [[package]]
 name = "proc-macro-error"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "18f33027081eba0a6d8aba6d1b1c3a3be58cbb12106341c2d5759fcd9b5277e7"
+dependencies = [
+ "proc-macro-error-attr 0.4.12",
+ "proc-macro2 1.0.50",
+ "quote 1.0.23",
+ "syn 1.0.107",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro-error"
 version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
 dependencies = [
- "proc-macro-error-attr",
+ "proc-macro-error-attr 1.0.4",
+ "proc-macro2 1.0.50",
+ "quote 1.0.23",
+ "syn 1.0.107",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro-error-attr"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a5b4b77fdb63c1eca72173d68d24501c54ab1269409f6b672c85deb18af69de"
+dependencies = [
  "proc-macro2 1.0.50",
  "quote 1.0.23",
  "syn 1.0.107",
+ "syn-mid",
  "version_check",
 ]
 
@@ -1569,6 +1625,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "proc-macro-hack"
+version = "0.5.20+deprecated"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
+
+[[package]]
 name = "proc-macro2"
 version = "0.4.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2184,6 +2246,17 @@ dependencies = [
 ]
 
 [[package]]
+name = "syn-mid"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baa8e7560a164edb1621a55d18a0c59abf49d360f47aa7b821061dd7eea7fac9"
+dependencies = [
+ "proc-macro2 1.0.50",
+ "quote 1.0.23",
+ "syn 1.0.107",
+]
+
+[[package]]
 name = "sync_wrapper"
 version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2247,7 +2320,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e45b7bf6e19353ddd832745c8fcf77a17a93171df7151187f26623f2b75b5b26"
 dependencies = [
  "cfg-if",
- "proc-macro-error",
+ "proc-macro-error 1.0.4",
  "proc-macro2 1.0.50",
  "quote 1.0.23",
  "syn 1.0.107",
@@ -2649,6 +2722,7 @@ dependencies = [
  "codemap-diagnostic",
  "criterion",
  "dirs",
+ "genawaiter",
  "imbl",
  "itertools",
  "lazy_static",
diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix
index 9f172e1b9efb..126edecdff17 100644
--- a/tvix/Cargo.nix
+++ b/tvix/Cargo.nix
@@ -1118,7 +1118,7 @@ rec {
           }
           {
             name = "proc-macro-error";
-            packageId = "proc-macro-error";
+            packageId = "proc-macro-error 1.0.4";
           }
           {
             name = "proc-macro2";
@@ -2308,6 +2308,85 @@ rec {
         ];
 
       };
+      "genawaiter" = rec {
+        crateName = "genawaiter";
+        version = "0.99.1";
+        edition = "2018";
+        sha256 = "1861a6vy9lc9a8lbw496m9j9jcjcn9nf7rkm6jqkkpnb3cvd0sy8";
+        authors = [
+          "John Simon <john@whatisaph.one>"
+        ];
+        dependencies = [
+          {
+            name = "genawaiter-macro";
+            packageId = "genawaiter-macro";
+          }
+          {
+            name = "genawaiter-proc-macro";
+            packageId = "genawaiter-proc-macro";
+            optional = true;
+          }
+          {
+            name = "proc-macro-hack";
+            packageId = "proc-macro-hack";
+            optional = true;
+          }
+        ];
+        features = {
+          "default" = [ "proc_macro" ];
+          "futures-core" = [ "dep:futures-core" ];
+          "futures03" = [ "futures-core" ];
+          "genawaiter-proc-macro" = [ "dep:genawaiter-proc-macro" ];
+          "proc-macro-hack" = [ "dep:proc-macro-hack" ];
+          "proc_macro" = [ "genawaiter-proc-macro" "proc-macro-hack" "genawaiter-macro/proc_macro" ];
+        };
+        resolvedDefaultFeatures = [ "default" "genawaiter-proc-macro" "proc-macro-hack" "proc_macro" ];
+      };
+      "genawaiter-macro" = rec {
+        crateName = "genawaiter-macro";
+        version = "0.99.1";
+        edition = "2018";
+        sha256 = "1g6zmr88fk48f1ksz9ik1i2mwjsiam9s4p9aybhvs2zwzphxychb";
+        authors = [
+          "Devin R <devin.ragotzy@gmail.com>"
+        ];
+        features = { };
+        resolvedDefaultFeatures = [ "proc_macro" ];
+      };
+      "genawaiter-proc-macro" = rec {
+        crateName = "genawaiter-proc-macro";
+        version = "0.99.1";
+        edition = "2018";
+        sha256 = "0f0pcaln4wrpi35nwxs9g516ysiax373m32a3hjiavinpkp88kvq";
+        procMacro = true;
+        authors = [
+          "Devin R <devin.ragotzy@gmail.com>"
+        ];
+        dependencies = [
+          {
+            name = "proc-macro-error";
+            packageId = "proc-macro-error 0.4.12";
+          }
+          {
+            name = "proc-macro-hack";
+            packageId = "proc-macro-hack";
+          }
+          {
+            name = "proc-macro2";
+            packageId = "proc-macro2 1.0.50";
+          }
+          {
+            name = "quote";
+            packageId = "quote 1.0.23";
+          }
+          {
+            name = "syn";
+            packageId = "syn 1.0.107";
+            features = [ "visit-mut" "full" ];
+          }
+        ];
+        features = { };
+      };
       "generic-array 0.12.4" = rec {
         crateName = "generic-array";
         version = "0.12.4";
@@ -4357,7 +4436,43 @@ rec {
           "verbatim" = [ "syn/parsing" ];
         };
       };
-      "proc-macro-error" = rec {
+      "proc-macro-error 0.4.12" = rec {
+        crateName = "proc-macro-error";
+        version = "0.4.12";
+        edition = "2018";
+        sha256 = "1rvpaadwv7vmsp142qqh2axqrr9v78f1nvdsi9nhmfhy10kk1wqq";
+        authors = [
+          "CreepySkeleton <creepy-skeleton@yandex.ru>"
+        ];
+        dependencies = [
+          {
+            name = "proc-macro-error-attr";
+            packageId = "proc-macro-error-attr 0.4.12";
+          }
+          {
+            name = "proc-macro2";
+            packageId = "proc-macro2 1.0.50";
+          }
+          {
+            name = "quote";
+            packageId = "quote 1.0.23";
+          }
+          {
+            name = "syn";
+            packageId = "syn 1.0.107";
+            usesDefaultFeatures = false;
+            features = [ "derive" "parsing" "proc-macro" "printing" ];
+          }
+        ];
+        buildDependencies = [
+          {
+            name = "version_check";
+            packageId = "version_check";
+          }
+        ];
+
+      };
+      "proc-macro-error 1.0.4" = rec {
         crateName = "proc-macro-error";
         version = "1.0.4";
         edition = "2018";
@@ -4368,7 +4483,7 @@ rec {
         dependencies = [
           {
             name = "proc-macro-error-attr";
-            packageId = "proc-macro-error-attr";
+            packageId = "proc-macro-error-attr 1.0.4";
           }
           {
             name = "proc-macro2";
@@ -4398,7 +4513,44 @@ rec {
         };
         resolvedDefaultFeatures = [ "default" "syn" "syn-error" ];
       };
-      "proc-macro-error-attr" = rec {
+      "proc-macro-error-attr 0.4.12" = rec {
+        crateName = "proc-macro-error-attr";
+        version = "0.4.12";
+        edition = "2018";
+        sha256 = "1pk9mwcfnpf8favgc2cl4sqlmi818p96hg8pfb51wg5nzmvlnnwa";
+        procMacro = true;
+        authors = [
+          "CreepySkeleton <creepy-skeleton@yandex.ru>"
+        ];
+        dependencies = [
+          {
+            name = "proc-macro2";
+            packageId = "proc-macro2 1.0.50";
+          }
+          {
+            name = "quote";
+            packageId = "quote 1.0.23";
+          }
+          {
+            name = "syn";
+            packageId = "syn 1.0.107";
+            usesDefaultFeatures = false;
+            features = [ "derive" "parsing" "proc-macro" "printing" ];
+          }
+          {
+            name = "syn-mid";
+            packageId = "syn-mid";
+          }
+        ];
+        buildDependencies = [
+          {
+            name = "version_check";
+            packageId = "version_check";
+          }
+        ];
+
+      };
+      "proc-macro-error-attr 1.0.4" = rec {
         crateName = "proc-macro-error-attr";
         version = "1.0.4";
         edition = "2018";
@@ -4425,6 +4577,17 @@ rec {
         ];
 
       };
+      "proc-macro-hack" = rec {
+        crateName = "proc-macro-hack";
+        version = "0.5.20+deprecated";
+        edition = "2018";
+        sha256 = "0s402hmcs3k9nd6rlp07zkr1lz7yimkmcwcbgnly2zr44wamwdyw";
+        procMacro = true;
+        authors = [
+          "David Tolnay <dtolnay@gmail.com>"
+        ];
+
+      };
       "proc-macro2 0.4.30" = rec {
         crateName = "proc-macro2";
         version = "0.4.30";
@@ -6247,6 +6410,36 @@ rec {
         };
         resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "full" "parsing" "printing" "proc-macro" "quote" "visit" "visit-mut" ];
       };
+      "syn-mid" = rec {
+        crateName = "syn-mid";
+        version = "0.5.3";
+        edition = "2018";
+        sha256 = "1jgslzpdf78646wafyplc39lkgwsqnh1hpd544bdnkhn19bfga5s";
+        authors = [
+          "Taiki Endo <te316e89@gmail.com>"
+        ];
+        dependencies = [
+          {
+            name = "proc-macro2";
+            packageId = "proc-macro2 1.0.50";
+            usesDefaultFeatures = false;
+          }
+          {
+            name = "quote";
+            packageId = "quote 1.0.23";
+            usesDefaultFeatures = false;
+          }
+          {
+            name = "syn";
+            packageId = "syn 1.0.107";
+            usesDefaultFeatures = false;
+            features = [ "parsing" "printing" "derive" ];
+          }
+        ];
+        features = {
+          "clone-impls" = [ "syn/clone-impls" ];
+        };
+      };
       "sync_wrapper" = rec {
         crateName = "sync_wrapper";
         version = "0.1.2";
@@ -6401,7 +6594,7 @@ rec {
           }
           {
             name = "proc-macro-error";
-            packageId = "proc-macro-error";
+            packageId = "proc-macro-error 1.0.4";
           }
           {
             name = "proc-macro2";
@@ -7858,6 +8051,10 @@ rec {
             packageId = "dirs";
           }
           {
+            name = "genawaiter";
+            packageId = "genawaiter";
+          }
+          {
             name = "imbl";
             packageId = "imbl";
             features = [ "serde" ];
diff --git a/tvix/eval/Cargo.toml b/tvix/eval/Cargo.toml
index cf46d4bbd464..6bb68834ab93 100644
--- a/tvix/eval/Cargo.toml
+++ b/tvix/eval/Cargo.toml
@@ -14,6 +14,7 @@ builtin-macros = { path = "./builtin-macros", package = "tvix-eval-builtin-macro
 codemap = "0.1.3"
 codemap-diagnostic = "0.1.1"
 dirs = "4.0.0"
+genawaiter = "0.99.1"
 imbl = { version = "2.0", features = [ "serde" ] }
 lazy_static = "1.4.0"
 lexical-core = { version = "0.8.5", features = ["format", "parse-floats"] }
diff --git a/tvix/eval/src/value/mod.rs b/tvix/eval/src/value/mod.rs
index ede93e7dddd2..07ce83020613 100644
--- a/tvix/eval/src/value/mod.rs
+++ b/tvix/eval/src/value/mod.rs
@@ -32,7 +32,7 @@ pub use path::canon_path;
 pub use string::NixString;
 pub use thunk::Thunk;
 
-use self::thunk::ThunkSet;
+pub use self::thunk::{SharedThunkSet, ThunkSet};
 
 use lazy_static::lazy_static;
 
diff --git a/tvix/eval/src/vm.rs b/tvix/eval/src/vm.rs
index 10966aca811b..3e3052555639 100644
--- a/tvix/eval/src/vm.rs
+++ b/tvix/eval/src/vm.rs
@@ -1,6 +1,8 @@
 //! This module implements the virtual (or abstract) machine that runs
 //! Tvix bytecode.
 
+pub mod generators;
+
 use serde_json::json;
 use std::{cmp::Ordering, collections::HashMap, ops::DerefMut, path::PathBuf, rc::Rc};
 
diff --git a/tvix/eval/src/vm/generators.rs b/tvix/eval/src/vm/generators.rs
new file mode 100644
index 000000000000..3b822b086346
--- /dev/null
+++ b/tvix/eval/src/vm/generators.rs
@@ -0,0 +1,538 @@
+//! This module implements generator logic for the VM. Generators are functions
+//! used during evaluation which can suspend their execution during their
+//! control flow, and request that the VM do something.
+//!
+//! This is used to keep the VM's stack size constant even when evaluating
+//! deeply nested recursive data structures.
+//!
+//! We implement generators using the [`genawaiter`] crate.
+
+use core::pin::Pin;
+use genawaiter::rc::Co;
+pub use genawaiter::rc::Gen;
+use smol_str::SmolStr;
+use std::fmt::Display;
+use std::future::Future;
+
+use crate::value::SharedThunkSet;
+use crate::warnings::WarningKind;
+use crate::FileType;
+use crate::NixString;
+
+use super::*;
+
+/// Dummy type, before the actual implementation is in place.
+#[derive(Debug)]
+pub struct PointerEquality {}
+
+// -- Implementation of generic generator logic.
+
+/// States that a generator can be in while being driven by the VM.
+pub(crate) enum GeneratorState {
+    /// Normal execution of the generator.
+    Running,
+
+    /// Generator is awaiting the result of a forced value.
+    AwaitingValue,
+}
+
+/// Messages that can be sent from generators to the VM. In most
+/// cases, the VM will suspend the generator when receiving a message
+/// and enter some other frame to process the request.
+///
+/// Responses are returned to generators via the [`GeneratorResponse`] type.
+pub enum GeneratorRequest {
+    /// Request that the VM forces this value. This message is first sent to the
+    /// VM with the unforced value, then returned to the generator with the
+    /// forced result.
+    ForceValue(Value),
+
+    /// Request that the VM deep-forces the value.
+    DeepForceValue(Value, SharedThunkSet),
+
+    /// Request the value at the given index from the VM's with-stack, in forced
+    /// state.
+    ///
+    /// The value is returned in the `ForceValue` message.
+    WithValue(usize),
+
+    /// Request the value at the given index from the *captured* with-stack, in
+    /// forced state.
+    CapturedWithValue(usize),
+
+    /// Request that the two values be compared for Nix equality. The result is
+    /// returned in the `ForceValue` message.
+    NixEquality(Box<(Value, Value)>, PointerEquality),
+
+    /// Push the given value to the VM's stack. This is used to prepare the
+    /// stack for requesting a function call from the VM.
+    ///
+    /// The VM does not respond to this request, so the next message received is
+    /// a `NoOp`.
+    StackPush(Value),
+
+    /// Pop a value from the stack and return it to the generator.
+    StackPop,
+
+    /// Request that the VM coerces this value to a string.
+    StringCoerce(Value, CoercionKind),
+
+    /// Request that the VM calls the given value, with arguments already
+    /// prepared on the stack. Value must already be forced.
+    Call(Value),
+
+    /// Request a call frame entering the given lambda immediately. This can be
+    /// used to force thunks.
+    EnterLambda {
+        lambda: Rc<Lambda>,
+        upvalues: Rc<Upvalues>,
+        light_span: LightSpan,
+    },
+
+    /// Emit a runtime warning through the VM. Receives a NoOp-response.
+    EmitWarning(WarningKind),
+
+    /// Request a lookup in the VM's import cache, which tracks the
+    /// thunks yielded by previously imported files.
+    ImportCacheLookup(PathBuf),
+
+    /// Provide the VM with an imported value for a given path, which
+    /// it can populate its input cache with.
+    ImportCachePut(PathBuf, Value),
+
+    /// Request that the VM imports the given path through its I/O interface.
+    PathImport(PathBuf),
+
+    /// Request that the VM reads the given path to a string.
+    ReadToString(PathBuf),
+
+    /// Request that the VM checks whether the given path exists.
+    PathExists(PathBuf),
+
+    /// Request that the VM reads the given path.
+    ReadDir(PathBuf),
+
+    /// Request a reasonable span from the VM.
+    Span,
+
+    /// Request evaluation of `builtins.tryEval` from the VM. See
+    /// [`VM::catch_result`] for an explanation of how this works.
+    TryForce(Value),
+}
+
+/// Human-readable representation of a generator message, used by observers.
+impl Display for GeneratorRequest {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            GeneratorRequest::ForceValue(v) => write!(f, "force_value({})", v),
+            GeneratorRequest::DeepForceValue(v, _) => write!(f, "deep_force_value({})", v),
+            GeneratorRequest::WithValue(_) => write!(f, "with_value"),
+            GeneratorRequest::CapturedWithValue(_) => write!(f, "captured_with_value"),
+            GeneratorRequest::NixEquality(values, ptr_eq) => {
+                write!(
+                    f,
+                    "nix_eq({}, {}, PointerEquality::{:?})",
+                    values.0, values.1, ptr_eq
+                )
+            }
+            GeneratorRequest::StackPush(v) => write!(f, "stack_push({})", v),
+            GeneratorRequest::StackPop => write!(f, "stack_pop"),
+            GeneratorRequest::StringCoerce(v, kind) => match kind {
+                CoercionKind::Weak => write!(f, "weak_string_coerce({})", v),
+                CoercionKind::Strong => write!(f, "strong_string_coerce({})", v),
+                CoercionKind::ThunksOnly => todo!("remove this branch (not live)"),
+            },
+            GeneratorRequest::Call(v) => write!(f, "call({})", v),
+            GeneratorRequest::EnterLambda { lambda, .. } => {
+                write!(f, "enter_lambda({:p})", *lambda)
+            }
+            GeneratorRequest::EmitWarning(_) => write!(f, "emit_warning"),
+            GeneratorRequest::ImportCacheLookup(p) => {
+                write!(f, "import_cache_lookup({})", p.to_string_lossy())
+            }
+            GeneratorRequest::ImportCachePut(p, _) => {
+                write!(f, "import_cache_put({})", p.to_string_lossy())
+            }
+            GeneratorRequest::PathImport(p) => write!(f, "path_import({})", p.to_string_lossy()),
+            GeneratorRequest::ReadToString(p) => {
+                write!(f, "read_to_string({})", p.to_string_lossy())
+            }
+            GeneratorRequest::PathExists(p) => write!(f, "path_exists({})", p.to_string_lossy()),
+            GeneratorRequest::ReadDir(p) => write!(f, "read_dir({})", p.to_string_lossy()),
+            GeneratorRequest::Span => write!(f, "span"),
+            GeneratorRequest::TryForce(v) => write!(f, "try_force({})", v),
+        }
+    }
+}
+
+/// Responses returned to generators from the VM.
+pub enum GeneratorResponse {
+    /// Empty message. Passed to the generator as the first message,
+    /// or when return values were optional.
+    Empty,
+
+    /// Value produced by the VM and returned to the generator.
+    Value(Value),
+
+    /// Path produced by the VM in response to some IO operation.
+    Path(PathBuf),
+
+    /// VM response with the contents of a directory.
+    Directory(Vec<(SmolStr, FileType)>),
+
+    /// VM response with a span to use at the current point.
+    Span(LightSpan),
+
+    /// Message returned by the VM when a catchable error is encountered during
+    /// the evaluation of `builtins.tryEval`.
+    ForceError,
+}
+
+impl Display for GeneratorResponse {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            GeneratorResponse::Empty => write!(f, "empty"),
+            GeneratorResponse::Value(v) => write!(f, "value({})", v),
+            GeneratorResponse::Path(p) => write!(f, "path({})", p.to_string_lossy()),
+            GeneratorResponse::Directory(d) => write!(f, "dir(len = {})", d.len()),
+            GeneratorResponse::Span(_) => write!(f, "span"),
+            GeneratorResponse::ForceError => write!(f, "force_error"),
+        }
+    }
+}
+
+pub(crate) type Generator = Gen<
+    GeneratorRequest,
+    GeneratorResponse,
+    Pin<Box<dyn Future<Output = Result<Value, ErrorKind>>>>,
+>;
+
+/// Helper function to provide type annotations which are otherwise difficult to
+/// infer.
+pub fn pin_generator(
+    f: impl Future<Output = Result<Value, ErrorKind>> + 'static,
+) -> Pin<Box<dyn Future<Output = Result<Value, ErrorKind>>>> {
+    Box::pin(f)
+}
+
+pub type GenCo = Co<GeneratorRequest, GeneratorResponse>;
+
+// -- Implementation of concrete generator use-cases.
+
+/// Request that the VM place the given value on its stack.
+pub async fn request_stack_push(co: &GenCo, val: Value) {
+    match co.yield_(GeneratorRequest::StackPush(val)).await {
+        GeneratorResponse::Empty => {}
+        msg => panic!(
+            "Tvix bug: VM responded with incorrect generator message: {}",
+            msg
+        ),
+    }
+}
+
+/// Request that the VM pop a value from the stack and return it to the
+/// generator.
+pub async fn request_stack_pop(co: &GenCo) -> Value {
+    match co.yield_(GeneratorRequest::StackPop).await {
+        GeneratorResponse::Value(value) => value,
+        msg => panic!(
+            "Tvix bug: VM responded with incorrect generator message: {}",
+            msg
+        ),
+    }
+}
+
+/// Force any value and return the evaluated result from the VM.
+pub async fn request_force(co: &GenCo, val: Value) -> Value {
+    if let Value::Thunk(_) = val {
+        match co.yield_(GeneratorRequest::ForceValue(val)).await {
+            GeneratorResponse::Value(value) => value,
+            msg => panic!(
+                "Tvix bug: VM responded with incorrect generator message: {}",
+                msg
+            ),
+        }
+    } else {
+        val
+    }
+}
+
+/// Force a value, but inform the caller (by returning `None`) if a catchable
+/// error occured.
+pub(crate) async fn request_try_force(co: &GenCo, val: Value) -> Option<Value> {
+    if let Value::Thunk(_) = val {
+        match co.yield_(GeneratorRequest::TryForce(val)).await {
+            GeneratorResponse::Value(value) => Some(value),
+            GeneratorResponse::ForceError => None,
+            msg => panic!(
+                "Tvix bug: VM responded with incorrect generator message: {}",
+                msg
+            ),
+        }
+    } else {
+        Some(val)
+    }
+}
+
+/// Call the given value as a callable. The argument(s) must already be prepared
+/// on the stack.
+pub async fn request_call(co: &GenCo, val: Value) -> Value {
+    let val = request_force(co, val).await;
+    match co.yield_(GeneratorRequest::Call(val)).await {
+        GeneratorResponse::Value(value) => value,
+        msg => panic!(
+            "Tvix bug: VM responded with incorrect generator message: {}",
+            msg
+        ),
+    }
+}
+
+/// Helper function to call the given value with the provided list of arguments.
+/// This uses the StackPush and Call messages under the hood.
+pub async fn request_call_with<I>(co: &GenCo, mut callable: Value, args: I) -> Value
+where
+    I: IntoIterator<Item = Value>,
+    I::IntoIter: DoubleEndedIterator,
+{
+    let mut num_args = 0_usize;
+    for arg in args.into_iter().rev() {
+        num_args += 1;
+        request_stack_push(co, arg).await;
+    }
+
+    debug_assert!(num_args > 0, "call_with called with an empty list of args");
+
+    while num_args > 0 {
+        callable = request_call(co, callable).await;
+        num_args -= 1;
+    }
+
+    callable
+}
+
+pub async fn request_string_coerce(co: &GenCo, val: Value, kind: CoercionKind) -> NixString {
+    match val {
+        Value::String(s) => s,
+        _ => match co.yield_(GeneratorRequest::StringCoerce(val, kind)).await {
+            GeneratorResponse::Value(value) => value
+                .to_str()
+                .expect("coerce_to_string always returns a string"),
+            msg => panic!(
+                "Tvix bug: VM responded with incorrect generator message: {}",
+                msg
+            ),
+        },
+    }
+}
+
+/// Deep-force any value and return the evaluated result from the VM.
+pub async fn request_deep_force(co: &GenCo, val: Value, thunk_set: SharedThunkSet) -> Value {
+    match co
+        .yield_(GeneratorRequest::DeepForceValue(val, thunk_set))
+        .await
+    {
+        GeneratorResponse::Value(value) => value,
+        msg => panic!(
+            "Tvix bug: VM responded with incorrect generator message: {}",
+            msg
+        ),
+    }
+}
+
+/// Fetch and force a value on the with-stack from the VM.
+async fn fetch_forced_with(co: &GenCo, idx: usize) -> Value {
+    match co.yield_(GeneratorRequest::WithValue(idx)).await {
+        GeneratorResponse::Value(value) => value,
+        msg => panic!(
+            "Tvix bug: VM responded with incorrect generator message: {}",
+            msg
+        ),
+    }
+}
+
+/// Fetch and force a value on the *captured* with-stack from the VM.
+async fn fetch_captured_with(co: &GenCo, idx: usize) -> Value {
+    match co.yield_(GeneratorRequest::CapturedWithValue(idx)).await {
+        GeneratorResponse::Value(value) => value,
+        msg => panic!(
+            "Tvix bug: VM responded with incorrect generator message: {}",
+            msg
+        ),
+    }
+}
+
+/// Ask the VM to compare two values for equality.
+pub(crate) async fn check_equality(
+    co: &GenCo,
+    a: Value,
+    b: Value,
+    ptr_eq: PointerEquality,
+) -> Result<bool, ErrorKind> {
+    match co
+        .yield_(GeneratorRequest::NixEquality(Box::new((a, b)), ptr_eq))
+        .await
+    {
+        GeneratorResponse::Value(value) => value.as_bool(),
+        msg => panic!(
+            "Tvix bug: VM responded with incorrect generator message: {}",
+            msg
+        ),
+    }
+}
+
+/// Emit a runtime warning.
+pub(crate) async fn emit_warning(co: &GenCo, kind: WarningKind) {
+    match co.yield_(GeneratorRequest::EmitWarning(kind)).await {
+        GeneratorResponse::Empty => {}
+        msg => panic!(
+            "Tvix bug: VM responded with incorrect generator message: {}",
+            msg
+        ),
+    }
+}
+
+/// Request that the VM enter the given lambda.
+pub(crate) async fn request_enter_lambda(
+    co: &GenCo,
+    lambda: Rc<Lambda>,
+    upvalues: Rc<Upvalues>,
+    light_span: LightSpan,
+) -> Value {
+    let msg = GeneratorRequest::EnterLambda {
+        lambda,
+        upvalues,
+        light_span,
+    };
+
+    match co.yield_(msg).await {
+        GeneratorResponse::Value(value) => value,
+        msg => panic!(
+            "Tvix bug: VM responded with incorrect generator message: {}",
+            msg
+        ),
+    }
+}
+
+/// Request a lookup in the VM's import cache.
+pub(crate) async fn request_import_cache_lookup(co: &GenCo, path: PathBuf) -> Option<Value> {
+    match co.yield_(GeneratorRequest::ImportCacheLookup(path)).await {
+        GeneratorResponse::Value(value) => Some(value),
+        GeneratorResponse::Empty => None,
+        msg => panic!(
+            "Tvix bug: VM responded with incorrect generator message: {}",
+            msg
+        ),
+    }
+}
+
+/// Request that the VM populate its input cache for the given path.
+pub(crate) async fn request_import_cache_put(co: &GenCo, path: PathBuf, value: Value) {
+    match co
+        .yield_(GeneratorRequest::ImportCachePut(path, value))
+        .await
+    {
+        GeneratorResponse::Empty => {}
+        msg => panic!(
+            "Tvix bug: VM responded with incorrect generator message: {}",
+            msg
+        ),
+    }
+}
+
+/// Request that the VM import the given path.
+pub(crate) async fn request_path_import(co: &GenCo, path: PathBuf) -> PathBuf {
+    match co.yield_(GeneratorRequest::PathImport(path)).await {
+        GeneratorResponse::Path(path) => path,
+        msg => panic!(
+            "Tvix bug: VM responded with incorrect generator message: {}",
+            msg
+        ),
+    }
+}
+
+pub(crate) async fn request_read_to_string(co: &GenCo, path: PathBuf) -> Value {
+    match co.yield_(GeneratorRequest::ReadToString(path)).await {
+        GeneratorResponse::Value(value) => value,
+        msg => panic!(
+            "Tvix bug: VM responded with incorrect generator message: {}",
+            msg
+        ),
+    }
+}
+
+pub(crate) async fn request_path_exists(co: &GenCo, path: PathBuf) -> Value {
+    match co.yield_(GeneratorRequest::PathExists(path)).await {
+        GeneratorResponse::Value(value) => value,
+        msg => panic!(
+            "Tvix bug: VM responded with incorrect generator message: {}",
+            msg
+        ),
+    }
+}
+
+pub(crate) async fn request_read_dir(co: &GenCo, path: PathBuf) -> Vec<(SmolStr, FileType)> {
+    match co.yield_(GeneratorRequest::ReadDir(path)).await {
+        GeneratorResponse::Directory(dir) => dir,
+        msg => panic!(
+            "Tvix bug: VM responded with incorrect generator message: {}",
+            msg
+        ),
+    }
+}
+
+pub(crate) async fn request_span(co: &GenCo) -> LightSpan {
+    match co.yield_(GeneratorRequest::Span).await {
+        GeneratorResponse::Span(span) => span,
+        msg => panic!(
+            "Tvix bug: VM responded with incorrect generator message: {}",
+            msg
+        ),
+    }
+}
+
+pub(crate) async fn neo_resolve_with(
+    co: GenCo,
+    ident: String,
+    vm_with_len: usize,
+    upvalue_with_len: usize,
+) -> Result<Value, ErrorKind> {
+    for with_stack_idx in (0..vm_with_len).rev() {
+        // TODO(tazjin): is this branch still live with the current with-thunking?
+        let with = fetch_forced_with(&co, with_stack_idx).await;
+
+        match with.to_attrs()?.select(&ident) {
+            None => continue,
+            Some(val) => return Ok(val.clone()),
+        }
+    }
+
+    for upvalue_with_idx in (0..upvalue_with_len).rev() {
+        let with = fetch_captured_with(&co, upvalue_with_idx).await;
+
+        match with.to_attrs()?.select(&ident) {
+            None => continue,
+            Some(val) => return Ok(val.clone()),
+        }
+    }
+
+    Err(ErrorKind::UnknownDynamicVariable(ident))
+}
+
+/// Call the given value as if it was an attribute set containing a functor. The
+/// arguments must already be prepared on the stack when a generator frame from
+/// this function is invoked.
+///
+pub(crate) async fn call_functor(co: GenCo, value: Value) -> Result<Value, ErrorKind> {
+    let attrs = value.to_attrs()?;
+
+    match attrs.select("__functor") {
+        None => Err(ErrorKind::NotCallable("set without `__functor_` attribute")),
+        Some(functor) => {
+            // The functor receives the set itself as its first argument and
+            // needs to be called with it.
+            let functor = request_force(&co, functor.clone()).await;
+            let primed = request_call_with(&co, functor, [value]).await;
+            Ok(request_call(&co, primed).await)
+        }
+    }
+}