feat(tvix/eval): implement recursive attribute sets r/4876

Yep.

This is kind of ugly right now. The idea is that the recursive_scope
compilation function is used for recursive sets as well by emitting
the keys. The rest of the logic is pretty much identical.

There is quite a lot of code here that can be cleaned up (duplication
between attrs and let, duplication inside of the recursive scope
compilation function etc.), but I'd like to get it working first and
then make it nice.

Note that nested keys are *not* supported yet.

Change-Id: I45c7cdd5f0e1d35fd94797093904740af3a97134
Reviewed-on: https://cl.tvl.fyi/c/depot/+/6610
Reviewed-by: sterni <sternenseemann@systemli.org>
Tested-by: BuildkiteCI

2 files changed, 77 insertions, 34 deletions

diff --git a/tvix/eval/src/compiler/attrs.rs b/tvix/eval/src/compiler/attrs.rs
index b5ebfe2399..e2e932953b 100644
--- a/tvix/eval/src/compiler/attrs.rs
+++ b/tvix/eval/src/compiler/attrs.rs
@@ -215,25 +215,23 @@ impl Compiler<'_, '_> {
     /// 2. Keys can refer to nested attribute sets.
     /// 3. Attribute sets can (optionally) be recursive.
     pub(super) fn compile_attr_set(&mut self, slot: LocalIdx, node: ast::AttrSet) {
-        if node.rec_token().is_some() {
-            let span = self.span_for(&node);
-            self.emit_warning(
-                span,
-                WarningKind::NotImplemented("recursive attribute sets"),
-            );
-        }
-
         // Open a scope to track the positions of the temporaries used
         // by the `OpAttrs` instruction.
         self.scope_mut().begin_scope();
 
-        let mut count = self.compile_inherit_attrs(slot, node.inherits());
+        if node.rec_token().is_some() {
+            self.compile_recursive_scope(slot, true, &node);
+            self.push_op(OpCode::OpAttrs(Count(node.entries().count())), &node);
+        } else {
+            let mut count = self.compile_inherit_attrs(slot, node.inherits());
 
-        let dynamic_entries = self.compile_static_attr_entries(&mut count, node.attrpath_values());
+            let dynamic_entries =
+                self.compile_static_attr_entries(&mut count, node.attrpath_values());
 
-        self.compile_dynamic_attr_entries(&mut count, dynamic_entries);
+            self.compile_dynamic_attr_entries(&mut count, dynamic_entries);
 
-        self.push_op(OpCode::OpAttrs(Count(count)), &node);
+            self.push_op(OpCode::OpAttrs(Count(count)), &node);
+        }
 
         // Remove the temporary scope, but do not emit any additional
         // cleanup (OpAttrs consumes all of these locals).
diff --git a/tvix/eval/src/compiler/mod.rs b/tvix/eval/src/compiler/mod.rs
index 25ddb08068..90bc263bc8 100644
--- a/tvix/eval/src/compiler/mod.rs
+++ b/tvix/eval/src/compiler/mod.rs
@@ -19,6 +19,7 @@ mod scope;
 use path_clean::PathClean;
 use rnix::ast::{self, AstToken, HasEntry};
 use rowan::ast::{AstChildren, AstNode};
+use smol_str::SmolStr;
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 use std::rc::Rc;
@@ -520,7 +521,7 @@ impl Compiler<'_, '_> {
         self.patch_jump(else_idx); // patch jump *over* else body
     }
 
-    fn compile_recursive_scope<N>(&mut self, slot: LocalIdx, node: &N)
+    fn compile_recursive_scope<N>(&mut self, slot: LocalIdx, rec_attrs: bool, node: &N)
     where
         N: AstNode + ast::HasEntry,
     {
@@ -539,25 +540,33 @@ impl Compiler<'_, '_> {
                 // Within a `let` binding, inheriting from the outer
                 // scope is a no-op *if* the identifier can be
                 // statically resolved.
-                None if !self.scope().has_with() => {
+                None if !rec_attrs && !self.scope().has_with() => {
                     self.emit_warning(self.span_for(&inherit), WarningKind::UselessInherit);
                     continue;
                 }
 
                 None => {
                     for ident in inherit.idents() {
-                        // If the identifier resolves statically, it
-                        // has precedence over dynamic bindings, and
-                        // the inherit is useless.
-                        if matches!(
-                            self.scope_mut()
-                                .resolve_local(ident.ident_token().unwrap().text()),
-                            LocalPosition::Known(_)
-                        ) {
+                        // If the identifier resolves statically in a
+                        // `let`, it has precedence over dynamic
+                        // bindings, and the inherit is useless.
+                        if !rec_attrs
+                            && matches!(
+                                self.scope_mut()
+                                    .resolve_local(ident.ident_token().unwrap().text()),
+                                LocalPosition::Known(_)
+                            )
+                        {
                             self.emit_warning(self.span_for(&ident), WarningKind::UselessInherit);
                             continue;
                         }
 
+                        if rec_attrs {
+                            self.emit_literal_ident(&ident);
+                            let span = self.span_for(&ident);
+                            self.scope_mut().declare_phantom(span, true);
+                        }
+
                         self.compile_ident(slot, ident.clone());
                         let idx = self.declare_local(&ident, ident.ident_token().unwrap().text());
                         self.scope_mut().mark_initialised(idx);
@@ -586,8 +595,13 @@ impl Compiler<'_, '_> {
             },
         }
 
+        struct KeySlot {
+            slot: LocalIdx,
+            name: SmolStr,
+        }
+
         struct TrackedBinding {
-            key_slot: Option<LocalIdx>,
+            key_slot: Option<KeySlot>,
             value_slot: LocalIdx,
             kind: BindingKind,
         }
@@ -600,11 +614,21 @@ impl Compiler<'_, '_> {
 
         // Begin with the inherit (from)s since they all become a thunk anyway
         for (from, ident) in inherit_froms {
-            let idx = self.declare_local(&ident, ident.ident_token().unwrap().text());
+            let key_slot = if rec_attrs {
+                let span = self.span_for(&ident);
+                Some(KeySlot {
+                    slot: self.scope_mut().declare_phantom(span, false),
+                    name: SmolStr::new(ident.ident_token().unwrap().text()),
+                })
+            } else {
+                None
+            };
+
+            let value_slot = self.declare_local(&ident, ident.ident_token().unwrap().text());
 
             bindings.push(TrackedBinding {
-                key_slot: None,
-                value_slot: idx,
+                key_slot,
+                value_slot,
                 kind: BindingKind::InheritFrom {
                     ident,
                     namespace: from,
@@ -626,16 +650,26 @@ impl Compiler<'_, '_> {
                 let span = self.span_for(&entry);
                 self.emit_error(
                     span,
-                    ErrorKind::NotImplemented("nested bindings in let expressions :("),
+                    ErrorKind::NotImplemented("nested bindings in recursive scope :("),
                 );
                 continue;
             }
 
-            let idx = self.declare_local(&entry.attrpath().unwrap(), path.pop().unwrap());
+            let key_slot = if rec_attrs {
+                let span = self.span_for(&entry.attrpath().unwrap());
+                Some(KeySlot {
+                    slot: self.scope_mut().declare_phantom(span, false),
+                    name: SmolStr::new(&path[0]),
+                })
+            } else {
+                None
+            };
+
+            let value_slot = self.declare_local(&entry.attrpath().unwrap(), path.pop().unwrap());
 
             bindings.push(TrackedBinding {
-                key_slot: None,
-                value_slot: idx,
+                key_slot,
+                value_slot,
                 kind: BindingKind::Plain {
                     expr: entry.value().unwrap(),
                 },
@@ -643,9 +677,20 @@ impl Compiler<'_, '_> {
         }
 
         // Third pass to place the values in the correct stack slots.
-        let mut indices: Vec<LocalIdx> = vec![];
+        let mut value_indices: Vec<LocalIdx> = vec![];
         for binding in bindings.into_iter() {
-            indices.push(binding.value_slot);
+            value_indices.push(binding.value_slot);
+
+            if let Some(key_slot) = binding.key_slot {
+                // TODO: emit_constant should be able to take a span directly
+                let span = self.scope()[key_slot.slot].span;
+                let idx = self
+                    .chunk()
+                    .push_constant(Value::String(key_slot.name.into()));
+
+                self.chunk().push_op(OpCode::OpConstant(idx), span);
+                self.scope_mut().mark_initialised(key_slot.slot);
+            }
 
             match binding.kind {
                 // This entry is an inherit (from) expr. The value is
@@ -673,7 +718,7 @@ impl Compiler<'_, '_> {
         }
 
         // Fourth pass to emit finaliser instructions if necessary.
-        for idx in indices {
+        for idx in value_indices {
             if self.scope()[idx].needs_finaliser {
                 let stack_idx = self.scope().stack_index(idx);
                 self.push_op(OpCode::OpFinalise(stack_idx), node);
@@ -687,7 +732,7 @@ impl Compiler<'_, '_> {
     /// simply pushed on the stack and their indices noted in the
     /// entries vector.
     fn compile_let_in(&mut self, slot: LocalIdx, node: ast::LetIn) {
-        self.compile_recursive_scope(slot, &node);
+        self.compile_recursive_scope(slot, false, &node);
 
         // Deal with the body, then clean up the locals afterwards.
         self.compile(slot, node.body().unwrap());