feat(tvix/eval): detect deferred upvalue capturing r/4665

Uses the threaded through slot offset to determine whether
initialisation of a captured local upvalue must be defered to a later
point where all values of a scope are available.

This adds a new data representation to the opcode for this situation,
but the equivalent runtime handling is not yet implemented. This is in
part because there is more compiler machinery needed to find the
resolution point.

Change-Id: Ifd0c393f76abfe6e2d91483faf0f58947ab1dedc
Reviewed-on: https://cl.tvl.fyi/c/depot/+/6329
Reviewed-by: sterni <sternenseemann@systemli.org>
Tested-by: BuildkiteCI

3 files changed, 22 insertions, 3 deletions

diff --git a/tvix/eval/src/compiler/mod.rs b/tvix/eval/src/compiler/mod.rs
index 0f22936f1c..85a2309eb3 100644
--- a/tvix/eval/src/compiler/mod.rs
+++ b/tvix/eval/src/compiler/mod.rs
@@ -122,7 +122,7 @@ impl Compiler {
             ast::Expr::LetIn(let_in) => self.compile_let_in(let_in),
             ast::Expr::Ident(ident) => self.compile_ident(ident),
             ast::Expr::With(with) => self.compile_with(with),
-            ast::Expr::Lambda(lambda) => self.compile_lambda(lambda),
+            ast::Expr::Lambda(lambda) => self.compile_lambda(slot, lambda),
             ast::Expr::Apply(apply) => self.compile_apply(apply),
 
             // Parenthesized expressions are simply unwrapped, leaving
@@ -778,7 +778,7 @@ impl Compiler {
         self.end_scope();
     }
 
-    fn compile_lambda(&mut self, node: ast::Lambda) {
+    fn compile_lambda(&mut self, slot: Option<usize>, node: ast::Lambda) {
         // Open new lambda context in compiler, which has its own
         // scope etc.
         self.contexts.push(LambdaCtx::new());
@@ -833,9 +833,22 @@ impl Compiler {
         self.chunk().push_op(OpCode::OpClosure(closure_idx));
         for upvalue in compiled.scope.upvalues {
             match upvalue {
-                Upvalue::Stack(idx) => {
+                Upvalue::Stack(idx) if slot.is_none() => {
                     self.chunk().push_op(OpCode::DataLocalIdx(idx));
                 }
+
+                Upvalue::Stack(idx) => {
+                    // If the upvalue slot is located *after* the
+                    // closure, the upvalue resolution must be
+                    // deferred until the scope is fully initialised
+                    // and can be finalised.
+                    if slot.unwrap() < idx.0 {
+                        self.chunk().push_op(OpCode::DataDeferredLocal(idx));
+                    } else {
+                        self.chunk().push_op(OpCode::DataLocalIdx(idx));
+                    }
+                }
+
                 Upvalue::Upvalue(idx) => {
                     self.chunk().push_op(OpCode::DataUpvalueIdx(idx));
                 }
diff --git a/tvix/eval/src/opcode.rs b/tvix/eval/src/opcode.rs
index 5f1f33f2b3..12a76d368e 100644
--- a/tvix/eval/src/opcode.rs
+++ b/tvix/eval/src/opcode.rs
@@ -115,6 +115,7 @@ pub enum OpCode {
     // The VM skips over these by advancing the instruction pointer
     // according to the count.
     DataLocalIdx(StackIdx),
+    DataDeferredLocal(StackIdx),
     DataUpvalueIdx(UpvalueIdx),
     DataDynamicIdx(ConstantIdx),
     DataDynamicAncestor(UpvalueIdx),
diff --git a/tvix/eval/src/vm.rs b/tvix/eval/src/vm.rs
index 316732dba0..33ea81087f 100644
--- a/tvix/eval/src/vm.rs
+++ b/tvix/eval/src/vm.rs
@@ -426,6 +426,10 @@ impl VM {
                                 closure.push_upvalue(value);
                             }
 
+                            OpCode::DataDeferredLocal(_idx) => {
+                                todo!("deferred local initialisation")
+                            }
+
                             _ => panic!("compiler error: missing closure operand"),
                         }
                     }
@@ -434,6 +438,7 @@ impl VM {
                 // Data-carrying operands should never be executed,
                 // that is a critical error in the VM.
                 OpCode::DataLocalIdx(_)
+                | OpCode::DataDeferredLocal(_)
                 | OpCode::DataUpvalueIdx(_)
                 | OpCode::DataDynamicIdx(_)
                 | OpCode::DataDynamicAncestor(_) => {