feat(tvix/eval): detect deferred upvalue capturing r/4665

Uses the threaded through slot offset to determine whether
initialisation of a captured local upvalue must be defered to a later
point where all values of a scope are available.

This adds a new data representation to the opcode for this situation,
but the equivalent runtime handling is not yet implemented. This is in
part because there is more compiler machinery needed to find the
resolution point.

Change-Id: Ifd0c393f76abfe6e2d91483faf0f58947ab1dedc
Reviewed-on: https://cl.tvl.fyi/c/depot/+/6329
Reviewed-by: sterni <sternenseemann@systemli.org>
Tested-by: BuildkiteCI

3 files changed, 22 insertions, 3 deletions

diff --git a/tvix/eval/src/compiler/mod.rs b/tvix/eval/src/compiler/mod.rs
index 0f22936f1cff..85a2309eb366 100644
--- a/tvix/eval/src/compiler/mod.rs
+++ b/tvix/eval/src/compiler/mod.rs
@@ -122,7 +122,7 @@ impl Compiler {
             ast::Expr::LetIn(let_in) => self.compile_let_in(let_in),
             ast::Expr::Ident(ident) => self.compile_ident(ident),
             ast::Expr::With(with) => self.compile_with(with),
-            ast::Expr::Lambda(lambda) => self.compile_lambda(lambda),
+            ast::Expr::Lambda(lambda) => self.compile_lambda(slot, lambda),
             ast::Expr::Apply(apply) => self.compile_apply(apply),
 
             // Parenthesized expressions are simply unwrapped, leaving
@@ -778,7 +778,7 @@ impl Compiler {
         self.end_scope();
     }
 
-    fn compile_lambda(&mut self, node: ast::Lambda) {
+    fn compile_lambda(&mut self, slot: Option<usize>, node: ast::Lambda) {
         // Open new lambda context in compiler, which has its own
         // scope etc.
         self.contexts.push(LambdaCtx::new());
@@ -833,9 +833,22 @@ impl Compiler {
         self.chunk().push_op(OpCode::OpClosure(closure_idx));
         for upvalue in compiled.scope.upvalues {
             match upvalue {
-                Upvalue::Stack(idx) => {
+                Upvalue::Stack(idx) if slot.is_none() => {
                     self.chunk().push_op(OpCode::DataLocalIdx(idx));
                 }
+
+                Upvalue::Stack(idx) => {
+                    // If the upvalue slot is located *after* the
+                    // closure, the upvalue resolution must be
+                    // deferred until the scope is fully initialised
+                    // and can be finalised.
+                    if slot.unwrap() < idx.0 {
+                        self.chunk().push_op(OpCode::DataDeferredLocal(idx));
+                    } else {
+                        self.chunk().push_op(OpCode::DataLocalIdx(idx));
+                    }
+                }
+
                 Upvalue::Upvalue(idx) => {
                     self.chunk().push_op(OpCode::DataUpvalueIdx(idx));
                 }
diff --git a/tvix/eval/src/opcode.rs b/tvix/eval/src/opcode.rs
index 5f1f33f2b3e5..12a76d368e4f 100644
--- a/tvix/eval/src/opcode.rs
+++ b/tvix/eval/src/opcode.rs
@@ -115,6 +115,7 @@ pub enum OpCode {
     // The VM skips over these by advancing the instruction pointer
     // according to the count.
     DataLocalIdx(StackIdx),
+    DataDeferredLocal(StackIdx),
     DataUpvalueIdx(UpvalueIdx),
     DataDynamicIdx(ConstantIdx),
     DataDynamicAncestor(UpvalueIdx),
diff --git a/tvix/eval/src/vm.rs b/tvix/eval/src/vm.rs
index 316732dba0ed..33ea81087f64 100644
--- a/tvix/eval/src/vm.rs
+++ b/tvix/eval/src/vm.rs
@@ -426,6 +426,10 @@ impl VM {
                                 closure.push_upvalue(value);
                             }
 
+                            OpCode::DataDeferredLocal(_idx) => {
+                                todo!("deferred local initialisation")
+                            }
+
                             _ => panic!("compiler error: missing closure operand"),
                         }
                     }
@@ -434,6 +438,7 @@ impl VM {
                 // Data-carrying operands should never be executed,
                 // that is a critical error in the VM.
                 OpCode::DataLocalIdx(_)
+                | OpCode::DataDeferredLocal(_)
                 | OpCode::DataUpvalueIdx(_)
                 | OpCode::DataDynamicIdx(_)
                 | OpCode::DataDynamicAncestor(_) => {