From 0db46dacea88bc15f06d586bee7acf2f50645f0a Mon Sep 17 00:00:00 2001 From: Aspen Smith Date: Mon, 19 Feb 2024 10:17:13 -0500 Subject: feat(tvix/glue): Init fetcher builtins Initialize a new empty builtins module `fetcher_builtins`, which will contain the builtins which fetch URLs from the internet: * fetchurl * fetchGit * fetchTarball * fetchTree (maybe? this is experimental) These builtins are all implemented in CPP nix at: https://github.com/NixOS/nix/blob/2.20.2/src/libexpr/primops/fetchTree.cc These builtins are added to the evaluation context using a similar mechanism to the derivation builtins, and have been added everywhere derivation builtins were previously being added. Change-Id: I133b91cc9560f23028621414537f712e7bd8a825 Reviewed-on: https://cl.tvl.fyi/c/depot/+/10974 Autosubmit: aspen Reviewed-by: flokli Tested-by: BuildkiteCI --- tvix/cli/src/main.rs | 4 +++- tvix/glue/benches/eval.rs | 7 +++++-- tvix/glue/src/builtins/fetchers.rs | 42 ++++++++++++++++++++++++++++++++++++++ tvix/glue/src/builtins/mod.rs | 18 +++++++++++++--- tvix/glue/src/tests/mod.rs | 6 +++++- tvix/glue/src/tvix_store_io.rs | 3 ++- 6 files changed, 72 insertions(+), 8 deletions(-) create mode 100644 tvix/glue/src/builtins/fetchers.rs diff --git a/tvix/cli/src/main.rs b/tvix/cli/src/main.rs index 24b180c1f289..65f5d5363c1d 100644 --- a/tvix/cli/src/main.rs +++ b/tvix/cli/src/main.rs @@ -9,6 +9,7 @@ use tvix_build::buildservice; use tvix_eval::builtins::impure_builtins; use tvix_eval::observer::{DisassemblingObserver, TracingObserver}; use tvix_eval::{EvalIO, Value}; +use tvix_glue::builtins::add_fetcher_builtins; use tvix_glue::tvix_io::TvixIO; use tvix_glue::tvix_store_io::TvixStoreIO; use tvix_glue::{builtins::add_derivation_builtins, configure_nix_path}; @@ -128,7 +129,8 @@ fn interpret(code: &str, path: Option, args: &Args, explain: bool) -> b ); eval.strict = args.strict; eval.builtins.extend(impure_builtins()); - add_derivation_builtins(&mut eval, tvix_store_io); + add_derivation_builtins(&mut eval, Rc::clone(&tvix_store_io)); + add_fetcher_builtins(&mut eval, tvix_store_io); configure_nix_path(&mut eval, &args.nix_search_path); let source_map = eval.source_map(); diff --git a/tvix/glue/benches/eval.rs b/tvix/glue/benches/eval.rs index 4e1886c4018e..11435707aca6 100644 --- a/tvix/glue/benches/eval.rs +++ b/tvix/glue/benches/eval.rs @@ -8,7 +8,9 @@ use tvix_castore::{ }; use tvix_eval::{builtins::impure_builtins, EvalIO}; use tvix_glue::{ - builtins::add_derivation_builtins, configure_nix_path, tvix_io::TvixIO, + builtins::{add_derivation_builtins, add_fetcher_builtins}, + configure_nix_path, + tvix_io::TvixIO, tvix_store_io::TvixStoreIO, }; use tvix_store::pathinfoservice::{MemoryPathInfoService, PathInfoService}; @@ -44,7 +46,8 @@ fn interpret(code: &str) { ); eval.builtins.extend(impure_builtins()); - add_derivation_builtins(&mut eval, tvix_store_io); + add_derivation_builtins(&mut eval, Rc::clone(&tvix_store_io)); + add_fetcher_builtins(&mut eval, tvix_store_io); configure_nix_path( &mut eval, // The benchmark requires TVIX_BENCH_NIX_PATH to be set, so barf out diff --git a/tvix/glue/src/builtins/fetchers.rs b/tvix/glue/src/builtins/fetchers.rs new file mode 100644 index 000000000000..7a5d49a725d1 --- /dev/null +++ b/tvix/glue/src/builtins/fetchers.rs @@ -0,0 +1,42 @@ +//! Contains builtins that fetch paths from the Internet + +use crate::tvix_store_io::TvixStoreIO; +use std::rc::Rc; +use tvix_eval::builtin_macros::builtins; +use tvix_eval::Value; + +#[allow(unused_variables)] // for the `state` arg, for now +#[builtins(state = "Rc")] +pub(crate) mod fetcher_builtins { + use super::*; + + use tvix_eval::generators::Gen; + use tvix_eval::{generators::GenCo, ErrorKind}; + + #[builtin("fetchurl")] + async fn builtin_fetchurl( + state: Rc, + co: GenCo, + url: Value, + ) -> Result { + Err(ErrorKind::NotImplemented("fetchurl")) + } + + #[builtin("fetchTarball")] + async fn builtin_fetch_tarball( + state: Rc, + co: GenCo, + args: Value, + ) -> Result { + Err(ErrorKind::NotImplemented("fetchTarball")) + } + + #[builtin("fetchGit")] + async fn builtin_fetch_git( + state: Rc, + co: GenCo, + args: Value, + ) -> Result { + Err(ErrorKind::NotImplemented("fetchGit")) + } +} diff --git a/tvix/glue/src/builtins/mod.rs b/tvix/glue/src/builtins/mod.rs index dff6a8947c16..4a59ec9ea6e5 100644 --- a/tvix/glue/src/builtins/mod.rs +++ b/tvix/glue/src/builtins/mod.rs @@ -6,6 +6,7 @@ use crate::tvix_store_io::TvixStoreIO; mod derivation; mod derivation_error; +mod fetchers; pub use derivation_error::Error as DerivationError; @@ -17,20 +18,30 @@ pub use derivation_error::Error as DerivationError; /// `known_paths`. pub fn add_derivation_builtins(eval: &mut tvix_eval::Evaluation, io: Rc) { eval.builtins - .extend(derivation::derivation_builtins::builtins(io)); + .extend(derivation::derivation_builtins::builtins(Rc::clone(&io))); // Add the actual `builtins.derivation` from compiled Nix code eval.src_builtins .push(("derivation", include_str!("derivation.nix"))); } +/// Adds fetcher builtins to the passed [tvix_eval::Evaluation]: +/// +/// * `fetchurl` +/// * `fetchTarball` +/// * `fetchGit` +pub fn add_fetcher_builtins(eval: &mut tvix_eval::Evaluation, io: Rc) { + eval.builtins + .extend(fetchers::fetcher_builtins::builtins(Rc::clone(&io))); +} + #[cfg(test)] mod tests { use std::{rc::Rc, sync::Arc}; use crate::tvix_store_io::TvixStoreIO; - use super::add_derivation_builtins; + use super::{add_derivation_builtins, add_fetcher_builtins}; use nix_compat::store_path::hash_placeholder; use test_case::test_case; use tvix_build::buildservice::DummyBuildService; @@ -57,7 +68,8 @@ mod tests { let mut eval = tvix_eval::Evaluation::new(io.clone() as Rc, false); - add_derivation_builtins(&mut eval, io); + add_derivation_builtins(&mut eval, Rc::clone(&io)); + add_fetcher_builtins(&mut eval, io); // run the evaluation itself. eval.evaluate(str, None) diff --git a/tvix/glue/src/tests/mod.rs b/tvix/glue/src/tests/mod.rs index ddc139158049..ad6571e4d212 100644 --- a/tvix/glue/src/tests/mod.rs +++ b/tvix/glue/src/tests/mod.rs @@ -12,7 +12,10 @@ use tvix_store::pathinfoservice::{MemoryPathInfoService, PathInfoService}; use rstest::rstest; -use crate::{builtins::add_derivation_builtins, tvix_store_io::TvixStoreIO}; +use crate::{ + builtins::{add_derivation_builtins, add_fetcher_builtins}, + tvix_store_io::TvixStoreIO, +}; fn eval_test(code_path: PathBuf, expect_success: bool) { assert_eq!( @@ -51,6 +54,7 @@ fn eval_test(code_path: PathBuf, expect_success: bool) { eval.strict = true; add_derivation_builtins(&mut eval, tvix_store_io.clone()); + add_fetcher_builtins(&mut eval, tvix_store_io); let result = eval.evaluate(code, Some(code_path.clone())); let failed = match result.value { diff --git a/tvix/glue/src/tvix_store_io.rs b/tvix/glue/src/tvix_store_io.rs index a64e2d4c7bf0..025a8a4bf01c 100644 --- a/tvix/glue/src/tvix_store_io.rs +++ b/tvix/glue/src/tvix_store_io.rs @@ -475,7 +475,7 @@ mod tests { use tvix_eval::{EvalIO, EvaluationResult}; use tvix_store::pathinfoservice::MemoryPathInfoService; - use crate::builtins::add_derivation_builtins; + use crate::builtins::{add_derivation_builtins, add_fetcher_builtins}; use super::TvixStoreIO; @@ -503,6 +503,7 @@ mod tests { let mut eval = tvix_eval::Evaluation::new(io.clone() as Rc, true); add_derivation_builtins(&mut eval, io.clone()); + add_fetcher_builtins(&mut eval, io.clone()); // run the evaluation itself. eval.evaluate(str, None) -- cgit 1.4.1