diff options
| -rw-r--r-- | tvix/cli/src/main.rs | 15 | ||||
| -rw-r--r-- | tvix/glue/benches/eval.rs | 17 | ||||
| -rw-r--r-- | tvix/glue/src/builtins/derivation.rs | 127 | ||||
| -rw-r--r-- | tvix/glue/src/known_paths.rs | 131 | ||||
| -rw-r--r-- | tvix/glue/src/tvix_io.rs | 19 |
5 files changed, 86 insertions, 223 deletions
diff --git a/tvix/cli/src/main.rs b/tvix/cli/src/main.rs index 8d3f793f52..17632cad1a 100644 --- a/tvix/cli/src/main.rs +++ b/tvix/cli/src/main.rs @@ -91,15 +91,12 @@ fn interpret(code: &str, path: Option<PathBuf>, args: &Args, explain: bool) -> b let known_paths: Rc<RefCell<KnownPaths>> = Default::default(); add_derivation_builtins(&mut eval, known_paths.clone()); configure_nix_path(&mut eval, &args.nix_search_path); - eval.io_handle = Box::new(tvix_glue::tvix_io::TvixIO::new( - known_paths, - TvixStoreIO::new( - blob_service, - directory_service, - path_info_service.into(), // we need an Arc<_> here. - tokio_runtime.handle().clone(), - ), - )); + eval.io_handle = Box::new(tvix_glue::tvix_io::TvixIO::new(TvixStoreIO::new( + blob_service, + directory_service, + path_info_service.into(), // we need an Arc<_> here. + tokio_runtime.handle().clone(), + ))); let source_map = eval.source_map(); let result = { diff --git a/tvix/glue/benches/eval.rs b/tvix/glue/benches/eval.rs index a466a0e0fc..f5c9813c90 100644 --- a/tvix/glue/benches/eval.rs +++ b/tvix/glue/benches/eval.rs @@ -29,7 +29,7 @@ fn interpret(code: &str) { let mut eval = tvix_eval::Evaluation::new_impure(); let known_paths: Rc<RefCell<KnownPaths>> = Default::default(); - add_derivation_builtins(&mut eval, known_paths.clone()); + add_derivation_builtins(&mut eval, known_paths); configure_nix_path( &mut eval, // The benchmark requires TVIX_BENCH_NIX_PATH to be set, so barf out @@ -37,15 +37,12 @@ fn interpret(code: &str) { &Some(env::var("TVIX_BENCH_NIX_PATH").expect("TVIX_BENCH_NIX_PATH must be set")), ); - eval.io_handle = Box::new(tvix_glue::tvix_io::TvixIO::new( - known_paths.clone(), - TvixStoreIO::new( - BLOB_SERVICE.clone(), - DIRECTORY_SERVICE.clone(), - PATH_INFO_SERVICE.clone(), - TOKIO_RUNTIME.handle().clone(), - ), - )); + eval.io_handle = Box::new(tvix_glue::tvix_io::TvixIO::new(TvixStoreIO::new( + BLOB_SERVICE.clone(), + DIRECTORY_SERVICE.clone(), + PATH_INFO_SERVICE.clone(), + TOKIO_RUNTIME.handle().clone(), + ))); let result = eval.evaluate(code, None); diff --git a/tvix/glue/src/builtins/derivation.rs b/tvix/glue/src/builtins/derivation.rs index 95314f4b9d..517ea00321 100644 --- a/tvix/glue/src/builtins/derivation.rs +++ b/tvix/glue/src/builtins/derivation.rs @@ -1,6 +1,6 @@ //! Implements `builtins.derivation`, the core of what makes Nix build packages. use crate::builtins::DerivationError; -use crate::known_paths::{KnownPaths, PathKind, PathName}; +use crate::known_paths::KnownPaths; use nix_compat::derivation::{Derivation, Output}; use nix_compat::nixhash; use std::cell::RefCell; @@ -9,7 +9,8 @@ use std::rc::Rc; use tvix_eval::builtin_macros::builtins; use tvix_eval::generators::{self, emit_warning_kind, GenCo}; use tvix_eval::{ - AddContext, CatchableErrorKind, CoercionKind, ErrorKind, NixAttrs, NixList, Value, WarningKind, + AddContext, CatchableErrorKind, CoercionKind, ErrorKind, NixAttrs, NixContext, + NixContextElement, NixList, Value, WarningKind, }; // Constants used for strangely named fields in derivation inputs. @@ -46,20 +47,15 @@ async fn populate_outputs( } /// Populate the inputs of a derivation from the build references -/// found when scanning the derivation's parameters. -fn populate_inputs<I: IntoIterator<Item = PathName>>( - drv: &mut Derivation, - known_paths: &KnownPaths, - references: I, -) { - for reference in references.into_iter() { - let reference = &known_paths[&reference]; - match &reference.kind { - PathKind::Plain => { - drv.input_sources.insert(reference.path.clone()); +/// found when scanning the derivation's parameters and extracting their contexts. +fn populate_inputs(drv: &mut Derivation, full_context: NixContext) { + for element in full_context.iter() { + match element { + NixContextElement::Plain(source) => { + drv.input_sources.insert(source.clone()); } - PathKind::Output { name, derivation } => { + NixContextElement::Single { name, derivation } => { match drv.input_derivations.entry(derivation.clone()) { btree_map::Entry::Vacant(entry) => { entry.insert(BTreeSet::from([name.clone()])); @@ -71,16 +67,12 @@ fn populate_inputs<I: IntoIterator<Item = PathName>>( } } - PathKind::Derivation { output_names } => { - match drv.input_derivations.entry(reference.path.clone()) { - btree_map::Entry::Vacant(entry) => { - entry.insert(output_names.clone()); - } - - btree_map::Entry::Occupied(mut entry) => { - entry.get_mut().extend(output_names.clone().into_iter()); - } - } + NixContextElement::Derivation(_drv_path) => { + // This is a hard one, it means that + // we are depending on a drvPath of ourselves + // *or* another derivation's drvPath. + // What to do here? + panic!("please do not depend on drvPath, I have 2 hours of sleep in blood"); } } } @@ -232,6 +224,7 @@ pub(crate) mod derivation_builtins { use super::*; use nix_compat::store_path::hash_placeholder; use tvix_eval::generators::Gen; + use tvix_eval::{NixContext, NixContextElement, NixString}; #[builtin("placeholder")] async fn builtin_placeholder(co: GenCo, input: Value) -> Result<Value, ErrorKind> { @@ -299,15 +292,31 @@ pub(crate) mod derivation_builtins { Ok(Ok(None)) } + let mut input_context = NixContext::new(); + for (name, value) in input.clone().into_iter_sorted() { let value = generators::request_force(&co, value).await; if ignore_nulls && matches!(value, Value::Null) { continue; } - match strong_importing_coerce_to_string(&co, value.clone()).await? { + match generators::request_string_coerce( + &co, + value.clone(), + CoercionKind { + strong: true, + import_paths: true, + }, + ) + .await + { Err(cek) => return Ok(Value::Catchable(cek)), Ok(val_str) => { + // Learn about this derivation references + // by looking at its context. + input_context.mimic(&val_str); + + let val_str = val_str.as_str().to_string(); // handle_derivation_parameters tells us whether the // argument should be added to the environment; continue // to the next one otherwise @@ -370,21 +379,6 @@ pub(crate) mod derivation_builtins { } } - // Scan references in relevant attributes to detect any build-references. - let references = { - let state = state.borrow(); - if state.is_empty() { - // skip reference scanning, create an empty result - Default::default() - } else { - let mut refscan = state.reference_scanner(); - drv.arguments.iter().for_each(|s| refscan.scan(s)); - drv.environment.values().for_each(|s| refscan.scan(s)); - refscan.scan(&drv.builder); - refscan.finalise() - } - }; - // Each output name needs to exist in the environment, at this // point initialised as an empty string because that is the // way of Golang ;) @@ -398,8 +392,8 @@ pub(crate) mod derivation_builtins { } } + populate_inputs(&mut drv, input_context); let mut known_paths = state.borrow_mut(); - populate_inputs(&mut drv, &known_paths, references); // At this point, derivation fields are fully populated from // eval data structures. @@ -428,25 +422,35 @@ pub(crate) mod derivation_builtins { &derivation_or_fod_hash_final, ); - // mark all the new paths as known - let output_names: Vec<String> = drv.outputs.keys().map(Clone::clone).collect(); - known_paths.drv(derivation_path.to_absolute_path(), &output_names); - - for (output_name, output) in &drv.outputs { - known_paths.output( - &output.path, - output_name, - derivation_path.to_absolute_path(), - ); - } - - let mut new_attrs: Vec<(String, String)> = drv + let mut new_attrs: Vec<(String, NixString)> = drv .outputs .into_iter() - .map(|(name, output)| (name, output.path)) + .map(|(name, output)| { + ( + name.clone(), + ( + output.path, + Some( + NixContextElement::Single { + name, + derivation: derivation_path.to_absolute_path(), + } + .into(), + ), + ) + .into(), + ) + }) .collect(); - new_attrs.push(("drvPath".to_string(), derivation_path.to_absolute_path())); + new_attrs.push(( + "drvPath".to_string(), + ( + derivation_path.to_absolute_path(), + Some(NixContextElement::Derivation(derivation_path.to_absolute_path()).into()), + ) + .into(), + )); Ok(Value::Attrs(Box::new(NixAttrs::from_iter( new_attrs.into_iter(), @@ -454,12 +458,7 @@ pub(crate) mod derivation_builtins { } #[builtin("toFile")] - async fn builtin_to_file( - state: Rc<RefCell<KnownPaths>>, - co: GenCo, - name: Value, - content: Value, - ) -> Result<Value, ErrorKind> { + async fn builtin_to_file(co: GenCo, name: Value, content: Value) -> Result<Value, ErrorKind> { let name = name .to_str() .context("evaluating the `name` parameter of builtins.toFile")?; @@ -482,11 +481,11 @@ pub(crate) mod derivation_builtins { .map_err(DerivationError::InvalidDerivation)? .to_absolute_path(); - state.borrow_mut().plain(&path); + let context: NixContext = NixContextElement::Plain(path.clone()).into(); // TODO: actually persist the file in the store at that path ... - Ok(Value::String(path.into())) + Ok(Value::String(NixString::new_context_from(context, &path))) } } diff --git a/tvix/glue/src/known_paths.rs b/tvix/glue/src/known_paths.rs index a01f8007eb..b847ccada3 100644 --- a/tvix/glue/src/known_paths.rs +++ b/tvix/glue/src/known_paths.rs @@ -11,12 +11,9 @@ //! Please see //tvix/eval/docs/build-references.md for more //! information. -use crate::refscan::{ReferenceScanner, STORE_PATH_LEN}; +use crate::refscan::STORE_PATH_LEN; use nix_compat::nixhash::NixHash; -use std::{ - collections::{hash_map, BTreeSet, HashMap}, - ops::Index, -}; +use std::collections::{BTreeSet, HashMap}; #[derive(Debug, PartialEq)] pub enum PathKind { @@ -36,12 +33,6 @@ pub struct KnownPath { pub kind: PathKind, } -impl KnownPath { - fn new(path: String, kind: PathKind) -> Self { - KnownPath { path, kind } - } -} - /// Internal struct to prevent accidental leaks of the truncated path /// names. #[repr(transparent)] @@ -54,6 +45,12 @@ impl From<&str> for PathName { } } +impl From<String> for PathName { + fn from(s: String) -> Self { + s.as_str().into() + } +} + /// This instance is required to pass PathName instances as needles to /// the reference scanner. impl AsRef<[u8]> for PathName { @@ -64,125 +61,13 @@ impl AsRef<[u8]> for PathName { #[derive(Debug, Default)] pub struct KnownPaths { - /// All known paths, keyed by a truncated version of their store - /// path used for reference scanning. - paths: HashMap<PathName, KnownPath>, - /// All known derivation or FOD hashes. /// /// Keys are derivation paths, values is the NixHash. derivation_or_fod_hashes: HashMap<String, NixHash>, } -impl Index<&PathName> for KnownPaths { - type Output = KnownPath; - - fn index(&self, index: &PathName) -> &Self::Output { - &self.paths[index] - } -} - impl KnownPaths { - fn insert_path(&mut self, path: String, path_kind: PathKind) { - match self.paths.entry(path.as_str().into()) { - hash_map::Entry::Vacant(entry) => { - entry.insert(KnownPath::new(path, path_kind)); - } - - hash_map::Entry::Occupied(mut entry) => { - match (path_kind, &mut entry.get_mut().kind) { - // These variant combinations require no "merging action". - (PathKind::Plain, PathKind::Plain) => (), - - #[allow(unused_variables)] - ( - PathKind::Output { - name: name1, - derivation: drv1, - }, - PathKind::Output { - name: ref name2, - derivation: ref drv2, - }, - ) => { - #[cfg(debug_assertions)] - { - if &name1 != name2 { - panic!( - "inserted path {} with two different names: {} and {}", - path, name1, name2 - ); - } - if &drv1 != drv2 { - panic!( - "inserted path {} with two different derivations: {} and {}", - path, drv1, drv2 - ); - } - } - } - - ( - PathKind::Derivation { output_names: new }, - PathKind::Derivation { - output_names: ref mut old, - }, - ) => { - old.extend(new); - } - - _ => panic!( - "path '{}' inserted twice with different types", - entry.key().0 - ), - }; - } - }; - } - - /// Mark a plain path as known. - pub fn plain<S: ToString>(&mut self, path: S) { - self.insert_path(path.to_string(), PathKind::Plain); - } - - /// Mark a derivation as known. - pub fn drv<P: ToString, O: ToString>(&mut self, path: P, outputs: &[O]) { - self.insert_path( - path.to_string(), - PathKind::Derivation { - output_names: outputs.iter().map(ToString::to_string).collect(), - }, - ); - } - - /// Mark a derivation output path as known. - pub fn output<P: ToString, N: ToString, D: ToString>( - &mut self, - output_path: P, - name: N, - drv_path: D, - ) { - self.insert_path( - output_path.to_string(), - PathKind::Output { - name: name.to_string(), - derivation: drv_path.to_string(), - }, - ); - } - - /// Checks whether there are any known paths. If not, a reference - /// scanner can not be created. - pub fn is_empty(&self) -> bool { - self.paths.is_empty() - } - - /// Create a reference scanner from the current set of known paths. - pub fn reference_scanner(&self) -> ReferenceScanner<PathName> { - let candidates = self.paths.keys().map(Clone::clone).collect(); - ReferenceScanner::new(candidates) - } - /// Fetch the opaque "hash derivation modulo" for a given derivation path. pub fn get_hash_derivation_modulo(&self, drv_path: &str) -> NixHash { // TODO: we rely on an invariant that things *should* have diff --git a/tvix/glue/src/tvix_io.rs b/tvix/glue/src/tvix_io.rs index 52bbd7bc9c..77dcb92910 100644 --- a/tvix/glue/src/tvix_io.rs +++ b/tvix/glue/src/tvix_io.rs @@ -8,30 +8,19 @@ //! otherwise fundamental features like nixpkgs bootstrapping and hash //! calculation will not work. -use std::cell::RefCell; use std::io; use std::path::{Path, PathBuf}; -use std::rc::Rc; use tvix_eval::{EvalIO, FileType}; -use crate::known_paths::KnownPaths; - // TODO: Merge this together with TvixStoreIO? pub struct TvixIO<T: EvalIO> { - /// Ingested paths must be reported to this known paths tracker - /// for accurate build reference scanning. - known_paths: Rc<RefCell<KnownPaths>>, - // Actual underlying [EvalIO] implementation. actual: T, } impl<T: EvalIO> TvixIO<T> { - pub fn new(known_paths: Rc<RefCell<KnownPaths>>, actual: T) -> Self { - Self { - known_paths, - actual, - } + pub fn new(actual: T) -> Self { + Self { actual } } } @@ -42,10 +31,6 @@ impl<T: EvalIO> EvalIO for TvixIO<T> { fn import_path(&self, path: &Path) -> io::Result<PathBuf> { let imported_path = self.actual.import_path(path)?; - self.known_paths - .borrow_mut() - .plain(imported_path.to_string_lossy()); - Ok(imported_path) } |