diff options
Diffstat (limited to 'tvix/glue/src/builtins/derivation.rs')
| -rw-r--r-- | tvix/glue/src/builtins/derivation.rs | 552 |
1 files changed, 552 insertions, 0 deletions
diff --git a/tvix/glue/src/builtins/derivation.rs b/tvix/glue/src/builtins/derivation.rs new file mode 100644 index 000000000000..7878ce0ed9ad --- /dev/null +++ b/tvix/glue/src/builtins/derivation.rs @@ -0,0 +1,552 @@ +//! Implements `builtins.derivation`, the core of what makes Nix build packages. +use crate::builtins::DerivationError; +use crate::known_paths::KnownPaths; +use crate::tvix_store_io::TvixStoreIO; +use bstr::BString; +use nix_compat::derivation::{Derivation, Output}; +use nix_compat::nixhash; +use nix_compat::store_path::{StorePath, StorePathRef}; +use std::collections::{btree_map, BTreeSet}; +use std::rc::Rc; +use tvix_eval::builtin_macros::builtins; +use tvix_eval::generators::{self, emit_warning_kind, GenCo}; +use tvix_eval::{ + AddContext, ErrorKind, NixAttrs, NixContext, NixContextElement, Value, WarningKind, +}; + +// Constants used for strangely named fields in derivation inputs. +const STRUCTURED_ATTRS: &str = "__structuredAttrs"; +const IGNORE_NULLS: &str = "__ignoreNulls"; + +/// Populate the inputs of a derivation from the build references +/// found when scanning the derivation's parameters and extracting their contexts. +fn populate_inputs(drv: &mut Derivation, full_context: NixContext, known_paths: &KnownPaths) { + for element in full_context.iter() { + match element { + NixContextElement::Plain(source) => { + let sp = StorePathRef::from_absolute_path(source.as_bytes()) + .expect("invalid store path") + .to_owned(); + drv.input_sources.insert(sp); + } + + NixContextElement::Single { + name, + derivation: derivation_str, + } => { + // TODO: b/264 + // We assume derivations to be passed validated, so ignoring rest + // and expecting parsing is ok. + let (derivation, _rest) = + StorePath::from_absolute_path_full(derivation_str).expect("valid store path"); + + #[cfg(debug_assertions)] + assert!( + _rest.iter().next().is_none(), + "Extra path not empty for {}", + derivation_str + ); + + match drv.input_derivations.entry(derivation.clone()) { + btree_map::Entry::Vacant(entry) => { + entry.insert(BTreeSet::from([name.clone()])); + } + + btree_map::Entry::Occupied(mut entry) => { + entry.get_mut().insert(name.clone()); + } + } + } + + NixContextElement::Derivation(drv_path) => { + let (derivation, _rest) = + StorePath::from_absolute_path_full(drv_path).expect("valid store path"); + + #[cfg(debug_assertions)] + assert!( + _rest.iter().next().is_none(), + "Extra path not empty for {}", + drv_path + ); + + // We need to know all the outputs *names* of that derivation. + let output_names = known_paths + .get_drv_by_drvpath(&derivation) + .expect("no known derivation associated to that derivation path") + .outputs + .keys(); + + // FUTUREWORK(performance): ideally, we should be able to clone + // cheaply those outputs rather than duplicate them all around. + match drv.input_derivations.entry(derivation.clone()) { + btree_map::Entry::Vacant(entry) => { + entry.insert(output_names.cloned().collect()); + } + + btree_map::Entry::Occupied(mut entry) => { + entry.get_mut().extend(output_names.cloned()); + } + } + + drv.input_sources.insert(derivation); + } + } + } +} + +/// Populate the output configuration of a derivation based on the +/// parameters passed to the call, configuring a fixed-output derivation output +/// if necessary. +/// +/// This function handles all possible combinations of the +/// parameters, including invalid ones. +/// +/// Due to the support for SRI hashes, and how these are passed along to +/// builtins.derivation, outputHash and outputHashAlgo can have values which +/// need to be further modified before constructing the Derivation struct. +/// +/// If outputHashAlgo is an SRI hash, outputHashAlgo must either be an empty +/// string, or the hash algorithm as specified in the (single) SRI (entry). +/// SRI strings with multiple hash algorithms are not supported. +/// +/// In case an SRI string was used, the (single) fixed output is populated +/// with the hash algo name, and the hash digest is populated with the +/// (lowercase) hex encoding of the digest. +/// +/// These values are only rewritten for the outputs, not what's passed to env. +/// +/// The return value may optionally contain a warning. +fn handle_fixed_output( + drv: &mut Derivation, + hash_str: Option<String>, // in nix: outputHash + hash_algo_str: Option<String>, // in nix: outputHashAlgo + hash_mode_str: Option<String>, // in nix: outputHashmode +) -> Result<Option<WarningKind>, ErrorKind> { + // If outputHash is provided, ensure hash_algo_str is compatible. + // If outputHash is not provided, do nothing. + if let Some(hash_str) = hash_str { + // treat an empty algo as None + let hash_algo_str = match hash_algo_str { + Some(s) if s.is_empty() => None, + Some(s) => Some(s), + None => None, + }; + + // construct a NixHash. + let nixhash = nixhash::from_str(&hash_str, hash_algo_str.as_deref()) + .map_err(DerivationError::InvalidOutputHash)?; + let algo = nixhash.algo(); + + // construct the fixed output. + drv.outputs.insert( + "out".to_string(), + Output { + path: None, + ca_hash: match hash_mode_str.as_deref() { + None | Some("flat") => Some(nixhash::CAHash::Flat(nixhash)), + Some("recursive") => Some(nixhash::CAHash::Nar(nixhash)), + Some(other) => { + return Err(DerivationError::InvalidOutputHashMode(other.to_string()))? + } + }, + }, + ); + + // Peek at hash_str once more. + // If it was a SRI hash, but is not using the correct length, this means + // the padding was wrong. Emit a warning in that case. + let sri_prefix = format!("{}-", algo); + if let Some(rest) = hash_str.strip_prefix(&sri_prefix) { + if data_encoding::BASE64.encode_len(algo.digest_length()) != rest.len() { + return Ok(Some(WarningKind::SRIHashWrongPadding)); + } + } + } + Ok(None) +} + +#[builtins(state = "Rc<TvixStoreIO>")] +pub(crate) mod derivation_builtins { + use std::collections::BTreeMap; + + use crate::builtins::utils::{select_string, strong_importing_coerce_to_string}; + + use super::*; + use bstr::ByteSlice; + use nix_compat::store_path::hash_placeholder; + use tvix_eval::generators::Gen; + use tvix_eval::{NixContext, NixContextElement, NixString}; + + #[builtin("placeholder")] + async fn builtin_placeholder(co: GenCo, input: Value) -> Result<Value, ErrorKind> { + if input.is_catchable() { + return Ok(input); + } + + let placeholder = hash_placeholder( + input + .to_str() + .context("looking at output name in builtins.placeholder")? + .to_str()?, + ); + + Ok(placeholder.into()) + } + + /// Strictly construct a Nix derivation from the supplied arguments. + /// + /// This is considered an internal function, users usually want to + /// use the higher-level `builtins.derivation` instead. + #[builtin("derivationStrict")] + async fn builtin_derivation_strict( + state: Rc<TvixStoreIO>, + co: GenCo, + input: Value, + ) -> Result<Value, ErrorKind> { + if input.is_catchable() { + return Ok(input); + } + + let input = input.to_attrs()?; + let name = generators::request_force(&co, input.select_required("name")?.clone()).await; + + if name.is_catchable() { + return Ok(name); + } + + let name = name.to_str().context("determining derivation name")?; + if name.is_empty() { + return Err(ErrorKind::Abort("derivation has empty name".to_string())); + } + let name = name.to_str()?; + + let mut drv = Derivation::default(); + drv.outputs.insert("out".to_string(), Default::default()); + let mut input_context = NixContext::new(); + + /// Inserts a key and value into the drv.environment BTreeMap, and fails if the + /// key did already exist before. + fn insert_env( + drv: &mut Derivation, + k: &str, /* TODO: non-utf8 env keys */ + v: BString, + ) -> Result<(), DerivationError> { + if drv.environment.insert(k.into(), v).is_some() { + return Err(DerivationError::DuplicateEnvVar(k.into())); + } + Ok(()) + } + + // Check whether null attributes should be ignored or passed through. + let ignore_nulls = match input.select(IGNORE_NULLS) { + Some(b) => generators::request_force(&co, b.clone()).await.as_bool()?, + None => false, + }; + + // peek at the STRUCTURED_ATTRS argument. + // If it's set and true, provide a BTreeMap that gets populated while looking at the arguments. + // We need it to be a BTreeMap, so iteration order of keys is reproducible. + let mut structured_attrs: Option<BTreeMap<String, serde_json::Value>> = + match input.select(STRUCTURED_ATTRS) { + Some(b) => generators::request_force(&co, b.clone()) + .await + .as_bool()? + .then_some(Default::default()), + None => None, + }; + + // Look at the arguments passed to builtins.derivationStrict. + // Some set special fields in the Derivation struct, some change + // behaviour of other functionality. + for (arg_name, arg_value) in input.clone().into_iter_sorted() { + let arg_name = arg_name.to_str()?; + // force the current value. + let value = generators::request_force(&co, arg_value).await; + + // filter out nulls if ignore_nulls is set. + if ignore_nulls && matches!(value, Value::Null) { + continue; + } + + match arg_name { + // Command line arguments to the builder. + // These are only set in drv.arguments. + "args" => { + for arg in value.to_list()? { + match strong_importing_coerce_to_string(&co, arg).await { + Err(cek) => return Ok(Value::from(cek)), + Ok(s) => { + input_context.mimic(&s); + drv.arguments.push(s.to_str()?.to_owned()) + } + } + } + } + + // If outputs is set, remove the original default `out` output, + // and replace it with the list of outputs. + "outputs" => { + let outputs = value + .to_list() + .context("looking at the `outputs` parameter of the derivation")?; + + // Remove the original default `out` output. + drv.outputs.clear(); + + let mut output_names = vec![]; + + for output in outputs { + let output_name = generators::request_force(&co, output) + .await + .to_str() + .context("determining output name")?; + + input_context.mimic(&output_name); + + // Populate drv.outputs + if drv + .outputs + .insert(output_name.to_str()?.to_owned(), Default::default()) + .is_some() + { + Err(DerivationError::DuplicateOutput( + output_name.to_str_lossy().into_owned(), + ))? + } + output_names.push(output_name.to_str()?.to_owned()); + } + + match structured_attrs.as_mut() { + // add outputs to the json itself (as a list of strings) + Some(structured_attrs) => { + structured_attrs.insert(arg_name.into(), output_names.into()); + } + // add drv.environment["outputs"] as a space-separated list + None => { + insert_env(&mut drv, arg_name, output_names.join(" ").into())?; + } + } + // drv.environment[$output_name] is added after the loop, + // with whatever is in drv.outputs[$output_name]. + } + + // handle builder and system. + "builder" | "system" => { + match strong_importing_coerce_to_string(&co, value).await { + Err(cek) => return Ok(Value::from(cek)), + Ok(val_str) => { + input_context.mimic(&val_str); + + if arg_name == "builder" { + drv.builder = val_str.to_str()?.to_owned(); + } else { + drv.system = val_str.to_str()?.to_owned(); + } + + // Either populate drv.environment or structured_attrs. + if let Some(ref mut structured_attrs) = structured_attrs { + // No need to check for dups, we only iterate over every attribute name once + structured_attrs.insert( + arg_name.to_owned(), + val_str.to_str()?.to_owned().into(), + ); + } else { + insert_env(&mut drv, arg_name, val_str.as_bytes().into())?; + } + } + } + } + + // Don't add STRUCTURED_ATTRS if enabled. + STRUCTURED_ATTRS if structured_attrs.is_some() => continue, + // IGNORE_NULLS is always skipped, even if it's not set to true. + IGNORE_NULLS => continue, + + // all other args. + _ => { + // In SA case, force and add to structured attrs. + // In non-SA case, coerce to string and add to env. + if let Some(ref mut structured_attrs) = structured_attrs { + let val = generators::request_force(&co, value).await; + if val.is_catchable() { + return Ok(val); + } + + let (val_json, context) = match val.into_contextful_json(&co).await? { + Ok(v) => v, + Err(cek) => return Ok(Value::from(cek)), + }; + + input_context.extend(context.into_iter()); + + // No need to check for dups, we only iterate over every attribute name once + structured_attrs.insert(arg_name.to_owned(), val_json); + } else { + match strong_importing_coerce_to_string(&co, value).await { + Err(cek) => return Ok(Value::from(cek)), + Ok(val_str) => { + input_context.mimic(&val_str); + + insert_env(&mut drv, arg_name, val_str.as_bytes().into())?; + } + } + } + } + } + } + // end of per-argument loop + + // Configure fixed-output derivations if required. + { + let output_hash = match select_string(&co, &input, "outputHash") + .await + .context("evaluating the `outputHash` parameter")? + { + Err(cek) => return Ok(Value::from(cek)), + Ok(s) => s, + }; + let output_hash_algo = match select_string(&co, &input, "outputHashAlgo") + .await + .context("evaluating the `outputHashAlgo` parameter")? + { + Err(cek) => return Ok(Value::from(cek)), + Ok(s) => s, + }; + let output_hash_mode = match select_string(&co, &input, "outputHashMode") + .await + .context("evaluating the `outputHashMode` parameter")? + { + Err(cek) => return Ok(Value::from(cek)), + Ok(s) => s, + }; + + if let Some(warning) = + handle_fixed_output(&mut drv, output_hash, output_hash_algo, output_hash_mode)? + { + emit_warning_kind(&co, warning).await; + } + } + + // Each output name needs to exist in the environment, at this + // point initialised as an empty string, as the ATerm serialization of that is later + // used for the output path calculation (which will also update output + // paths post-calculation, both in drv.environment and drv.outputs) + for output in drv.outputs.keys() { + if drv + .environment + .insert(output.to_string(), String::new().into()) + .is_some() + { + emit_warning_kind(&co, WarningKind::ShadowedOutput(output.to_string())).await; + } + } + + if let Some(structured_attrs) = structured_attrs { + // configure __json + drv.environment.insert( + "__json".to_string(), + BString::from(serde_json::to_string(&structured_attrs)?), + ); + } + + let mut known_paths = state.as_ref().known_paths.borrow_mut(); + populate_inputs(&mut drv, input_context, &known_paths); + + // At this point, derivation fields are fully populated from + // eval data structures. + drv.validate(false) + .map_err(DerivationError::InvalidDerivation)?; + + // Calculate the hash_derivation_modulo for the current derivation.. + debug_assert!( + drv.outputs.values().all(|output| { output.path.is_none() }), + "outputs should still be unset" + ); + + // Mutate the Derivation struct and set output paths + drv.calculate_output_paths( + name, + // This one is still intermediate (so not added to known_paths), + // as the outputs are still unset. + &drv.hash_derivation_modulo(|drv_path| { + *known_paths + .get_hash_derivation_modulo(&drv_path.to_owned()) + .unwrap_or_else(|| panic!("{} not found", drv_path)) + }), + ) + .map_err(DerivationError::InvalidDerivation)?; + + let drv_path = drv + .calculate_derivation_path(name) + .map_err(DerivationError::InvalidDerivation)?; + + // Assemble the attrset to return from this builtin. + let out = Value::Attrs(Box::new(NixAttrs::from_iter( + drv.outputs + .iter() + .map(|(name, output)| { + ( + name.clone(), + NixString::new_context_from( + NixContextElement::Single { + name: name.clone(), + derivation: drv_path.to_absolute_path(), + } + .into(), + output.path.as_ref().unwrap().to_absolute_path(), + ), + ) + }) + .chain(std::iter::once(( + "drvPath".to_owned(), + NixString::new_context_from( + NixContextElement::Derivation(drv_path.to_absolute_path()).into(), + drv_path.to_absolute_path(), + ), + ))), + ))); + + // Register the Derivation in known_paths. + known_paths.add_derivation(drv_path, drv); + + Ok(out) + } + + #[builtin("toFile")] + async fn builtin_to_file(co: GenCo, name: Value, content: Value) -> Result<Value, ErrorKind> { + if name.is_catchable() { + return Ok(name); + } + + if content.is_catchable() { + return Ok(content); + } + + let name = name + .to_str() + .context("evaluating the `name` parameter of builtins.toFile")?; + let content = content + .to_contextful_str() + .context("evaluating the `content` parameter of builtins.toFile")?; + + if content.iter_derivation().count() > 0 || content.iter_single_outputs().count() > 0 { + return Err(ErrorKind::UnexpectedContext); + } + + let path = + nix_compat::store_path::build_text_path(name.to_str()?, &content, content.iter_plain()) + .map_err(|_e| { + nix_compat::derivation::DerivationError::InvalidOutputName( + name.to_str_lossy().into_owned(), + ) + }) + .map_err(DerivationError::InvalidDerivation)? + .to_absolute_path(); + + let context: NixContext = NixContextElement::Plain(path.clone()).into(); + + // TODO: actually persist the file in the store at that path ... + + Ok(Value::from(NixString::new_context_from(context, path))) + } +} |