about summary refs log tree commit diff
path: root/tvix/glue/src/builtins
diff options
context:
space:
mode:
Diffstat (limited to 'tvix/glue/src/builtins')
-rw-r--r--tvix/glue/src/builtins/derivation.rs41
-rw-r--r--tvix/glue/src/builtins/errors.rs30
-rw-r--r--tvix/glue/src/builtins/fetchers.rs257
-rw-r--r--tvix/glue/src/builtins/mod.rs3
-rw-r--r--tvix/glue/src/builtins/utils.rs36
5 files changed, 322 insertions, 45 deletions
diff --git a/tvix/glue/src/builtins/derivation.rs b/tvix/glue/src/builtins/derivation.rs
index 71249f1c7722..4ea9611a946c 100644
--- a/tvix/glue/src/builtins/derivation.rs
+++ b/tvix/glue/src/builtins/derivation.rs
@@ -10,8 +10,7 @@ use std::rc::Rc;
 use tvix_eval::builtin_macros::builtins;
 use tvix_eval::generators::{self, emit_warning_kind, GenCo};
 use tvix_eval::{
-    AddContext, CatchableErrorKind, CoercionKind, ErrorKind, NixAttrs, NixContext,
-    NixContextElement, Value, WarningKind,
+    AddContext, ErrorKind, NixAttrs, NixContext, NixContextElement, Value, WarningKind,
 };
 
 // Constants used for strangely named fields in derivation inputs.
@@ -144,6 +143,8 @@ fn handle_fixed_output(
 pub(crate) mod derivation_builtins {
     use std::collections::BTreeMap;
 
+    use crate::builtins::utils::{select_string, strong_importing_coerce_to_string};
+
     use super::*;
     use bstr::ByteSlice;
     use nix_compat::store_path::hash_placeholder;
@@ -197,27 +198,6 @@ pub(crate) mod derivation_builtins {
         drv.outputs.insert("out".to_string(), Default::default());
         let mut input_context = NixContext::new();
 
-        #[inline]
-        async fn strong_importing_coerce_to_string(
-            co: &GenCo,
-            val: Value,
-        ) -> Result<NixString, CatchableErrorKind> {
-            let val = generators::request_force(co, val).await;
-            match generators::request_string_coerce(
-                co,
-                val,
-                CoercionKind {
-                    strong: true,
-                    import_paths: true,
-                },
-            )
-            .await
-            {
-                Err(cek) => Err(cek),
-                Ok(val_str) => Ok(val_str),
-            }
-        }
-
         /// Inserts a key and value into the drv.environment BTreeMap, and fails if the
         /// key did already exist before.
         fn insert_env(
@@ -385,21 +365,6 @@ pub(crate) mod derivation_builtins {
 
         // Configure fixed-output derivations if required.
         {
-            async fn select_string(
-                co: &GenCo,
-                attrs: &NixAttrs,
-                key: &str,
-            ) -> Result<Result<Option<String>, CatchableErrorKind>, ErrorKind> {
-                if let Some(attr) = attrs.select(key) {
-                    match strong_importing_coerce_to_string(co, attr.clone()).await {
-                        Err(cek) => return Ok(Err(cek)),
-                        Ok(str) => return Ok(Ok(Some(str.to_str()?.to_owned()))),
-                    }
-                }
-
-                Ok(Ok(None))
-            }
-
             let output_hash = match select_string(&co, &input, "outputHash")
                 .await
                 .context("evaluating the `outputHash` parameter")?
diff --git a/tvix/glue/src/builtins/errors.rs b/tvix/glue/src/builtins/errors.rs
index b606aa6a7c48..c753a125e030 100644
--- a/tvix/glue/src/builtins/errors.rs
+++ b/tvix/glue/src/builtins/errors.rs
@@ -1,5 +1,8 @@
 //! Contains errors that can occur during evaluation of builtins in this crate
-use nix_compat::nixhash;
+use nix_compat::{
+    nixhash::{self, NixHash},
+    store_path::BuildStorePathError,
+};
 use std::rc::Rc;
 use thiserror::Error;
 
@@ -25,3 +28,28 @@ impl From<DerivationError> for tvix_eval::ErrorKind {
         tvix_eval::ErrorKind::TvixError(Rc::new(err))
     }
 }
+
+#[derive(Debug, Error)]
+pub enum FetcherError {
+    #[error("hash mismatch in file downloaded from {url}:\n  wanted: {wanted}\n     got: {got}")]
+    HashMismatch {
+        url: String,
+        wanted: NixHash,
+        got: NixHash,
+    },
+
+    #[error("Invalid hash type '{0}' for fetcher")]
+    InvalidHashType(&'static str),
+
+    #[error("Error in store path for fetcher output: {0}")]
+    StorePath(#[from] BuildStorePathError),
+
+    #[error(transparent)]
+    Http(#[from] reqwest::Error),
+}
+
+impl From<FetcherError> for tvix_eval::ErrorKind {
+    fn from(err: FetcherError) -> Self {
+        tvix_eval::ErrorKind::TvixError(Rc::new(err))
+    }
+}
diff --git a/tvix/glue/src/builtins/fetchers.rs b/tvix/glue/src/builtins/fetchers.rs
index 7a5d49a725d1..07074d93de11 100644
--- a/tvix/glue/src/builtins/fetchers.rs
+++ b/tvix/glue/src/builtins/fetchers.rs
@@ -1,9 +1,189 @@
 //! Contains builtins that fetch paths from the Internet
 
 use crate::tvix_store_io::TvixStoreIO;
+use bstr::ByteSlice;
+use nix_compat::nixhash::{self, CAHash};
+use nix_compat::store_path::{build_ca_path, StorePathRef};
 use std::rc::Rc;
 use tvix_eval::builtin_macros::builtins;
-use tvix_eval::Value;
+use tvix_eval::generators::GenCo;
+use tvix_eval::{CatchableErrorKind, ErrorKind, NixContextElement, NixString, Value};
+
+use super::utils::select_string;
+use super::{DerivationError, FetcherError};
+
+/// Attempts to mimic `nix::libutil::baseNameOf`
+fn url_basename(s: &str) -> &str {
+    if s.is_empty() {
+        return "";
+    }
+
+    let mut last = s.len() - 1;
+    if s.chars().nth(last).unwrap() == '/' && last > 0 {
+        last -= 1;
+    }
+
+    if last == 0 {
+        return "";
+    }
+
+    let pos = match s[..=last].rfind('/') {
+        Some(pos) => {
+            if pos == last - 1 {
+                0
+            } else {
+                pos
+            }
+        }
+        None => 0,
+    };
+
+    &s[(pos + 1)..=last]
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum HashMode {
+    Flat,
+    Recursive,
+}
+
+/// Struct representing the arguments passed to fetcher functions
+#[derive(Debug, PartialEq, Eq)]
+struct FetchArgs {
+    url: String,
+    name: String,
+    hash: Option<CAHash>,
+}
+
+impl FetchArgs {
+    pub fn new(
+        url: String,
+        name: Option<String>,
+        sha256: Option<String>,
+        mode: HashMode,
+    ) -> nixhash::Result<Self> {
+        Ok(Self {
+            name: name.unwrap_or_else(|| url_basename(&url).to_owned()),
+            url,
+            hash: sha256
+                .map(|h| {
+                    let hash = nixhash::from_str(&h, Some("sha256"))?;
+                    Ok(match mode {
+                        HashMode::Flat => Some(nixhash::CAHash::Flat(hash)),
+                        HashMode::Recursive => Some(nixhash::CAHash::Nar(hash)),
+                    })
+                })
+                .transpose()?
+                .flatten(),
+        })
+    }
+
+    fn store_path(&self) -> Result<Option<StorePathRef>, ErrorKind> {
+        let Some(h) = &self.hash else {
+            return Ok(None);
+        };
+        build_ca_path(&self.name, h, Vec::<String>::new(), false)
+            .map(Some)
+            .map_err(|e| FetcherError::from(e).into())
+    }
+
+    async fn extract(
+        co: &GenCo,
+        args: Value,
+        default_name: Option<&str>,
+        mode: HashMode,
+    ) -> Result<Result<Self, CatchableErrorKind>, ErrorKind> {
+        if let Ok(url) = args.to_str() {
+            return Ok(Ok(FetchArgs::new(
+                url.to_str()?.to_owned(),
+                None,
+                None,
+                mode,
+            )
+            .map_err(DerivationError::InvalidOutputHash)?));
+        }
+
+        let attrs = args.to_attrs().map_err(|_| ErrorKind::TypeError {
+            expected: "attribute set or string",
+            actual: args.type_of(),
+        })?;
+
+        let url = match select_string(co, &attrs, "url").await? {
+            Ok(s) => s.ok_or_else(|| ErrorKind::AttributeNotFound { name: "url".into() })?,
+            Err(cek) => return Ok(Err(cek)),
+        };
+        let name = match select_string(co, &attrs, "name").await? {
+            Ok(s) => s.or_else(|| default_name.map(|s| s.to_owned())),
+            Err(cek) => return Ok(Err(cek)),
+        };
+        let sha256 = match select_string(co, &attrs, "sha256").await? {
+            Ok(s) => s,
+            Err(cek) => return Ok(Err(cek)),
+        };
+
+        Ok(Ok(
+            FetchArgs::new(url, name, sha256, mode).map_err(DerivationError::InvalidOutputHash)?
+        ))
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum FetchMode {
+    Url,
+    Tarball,
+}
+
+impl From<FetchMode> for HashMode {
+    fn from(value: FetchMode) -> Self {
+        match value {
+            FetchMode::Url => HashMode::Flat,
+            FetchMode::Tarball => HashMode::Recursive,
+        }
+    }
+}
+
+impl FetchMode {
+    fn default_name(self) -> Option<&'static str> {
+        match self {
+            FetchMode::Url => None,
+            FetchMode::Tarball => Some("source"),
+        }
+    }
+}
+
+fn string_from_store_path(store_path: StorePathRef) -> NixString {
+    NixString::new_context_from(
+        NixContextElement::Plain(store_path.to_absolute_path()).into(),
+        store_path.to_absolute_path(),
+    )
+}
+
+async fn fetch(
+    state: Rc<TvixStoreIO>,
+    co: GenCo,
+    args: Value,
+    mode: FetchMode,
+) -> Result<Value, ErrorKind> {
+    let args = match FetchArgs::extract(&co, args, mode.default_name(), mode.into()).await? {
+        Ok(args) => args,
+        Err(cek) => return Ok(cek.into()),
+    };
+
+    if let Some(store_path) = args.store_path()? {
+        if state.store_path_exists(store_path).await? {
+            return Ok(string_from_store_path(store_path).into());
+        }
+    }
+
+    let hash = args.hash.as_ref().map(|h| h.hash());
+    let store_path = Rc::clone(&state).tokio_handle.block_on(state.fetch_url(
+        &args.url,
+        &args.name,
+        hash.as_deref(),
+    ))?;
+
+    Ok(string_from_store_path(store_path.as_ref()).into())
+}
 
 #[allow(unused_variables)] // for the `state` arg, for now
 #[builtins(state = "Rc<TvixStoreIO>")]
@@ -11,15 +191,14 @@ pub(crate) mod fetcher_builtins {
     use super::*;
 
     use tvix_eval::generators::Gen;
-    use tvix_eval::{generators::GenCo, ErrorKind};
 
     #[builtin("fetchurl")]
     async fn builtin_fetchurl(
         state: Rc<TvixStoreIO>,
         co: GenCo,
-        url: Value,
+        args: Value,
     ) -> Result<Value, ErrorKind> {
-        Err(ErrorKind::NotImplemented("fetchurl"))
+        fetch(state, co, args, FetchMode::Url).await
     }
 
     #[builtin("fetchTarball")]
@@ -28,7 +207,7 @@ pub(crate) mod fetcher_builtins {
         co: GenCo,
         args: Value,
     ) -> Result<Value, ErrorKind> {
-        Err(ErrorKind::NotImplemented("fetchTarball"))
+        fetch(state, co, args, FetchMode::Tarball).await
     }
 
     #[builtin("fetchGit")]
@@ -40,3 +219,71 @@ pub(crate) mod fetcher_builtins {
         Err(ErrorKind::NotImplemented("fetchGit"))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::str::FromStr;
+
+    use nix_compat::store_path::StorePath;
+
+    use super::*;
+
+    #[test]
+    fn fetchurl_store_path() {
+        let url = "https://raw.githubusercontent.com/aaptel/notmuch-extract-patch/f732a53e12a7c91a06755ebfab2007adc9b3063b/notmuch-extract-patch";
+        let sha256 = "0nawkl04sj7psw6ikzay7kydj3dhd0fkwghcsf5rzaw4bmp4kbax";
+        let args = FetchArgs::new(url.into(), None, Some(sha256.into()), HashMode::Flat).unwrap();
+
+        assert_eq!(
+            args.store_path().unwrap().unwrap().to_owned(),
+            StorePath::from_str("06qi00hylriyfm0nl827crgjvbax84mz-notmuch-extract-patch").unwrap()
+        )
+    }
+
+    #[test]
+    fn fetch_tarball_store_path() {
+        let url = "https://github.com/NixOS/nixpkgs/archive/91050ea1e57e50388fa87a3302ba12d188ef723a.tar.gz";
+        let sha256 = "1hf6cgaci1n186kkkjq106ryf8mmlq9vnwgfwh625wa8hfgdn4dm";
+        let args = FetchArgs::new(
+            url.into(),
+            Some("source".into()),
+            Some(sha256.into()),
+            HashMode::Recursive,
+        )
+        .unwrap();
+
+        assert_eq!(
+            args.store_path().unwrap().unwrap().to_owned(),
+            StorePath::from_str("7adgvk5zdfq4pwrhsm3n9lzypb12gw0g-source").unwrap()
+        )
+    }
+
+    mod url_basename {
+        use super::*;
+
+        #[test]
+        fn empty_path() {
+            assert_eq!(url_basename(""), "");
+        }
+
+        #[test]
+        fn path_on_root() {
+            assert_eq!(url_basename("/dir"), "dir");
+        }
+
+        #[test]
+        fn relative_path() {
+            assert_eq!(url_basename("dir/foo"), "foo");
+        }
+
+        #[test]
+        fn root_with_trailing_slash() {
+            assert_eq!(url_basename("/"), "");
+        }
+
+        #[test]
+        fn trailing_slash() {
+            assert_eq!(url_basename("/dir/"), "dir");
+        }
+    }
+}
diff --git a/tvix/glue/src/builtins/mod.rs b/tvix/glue/src/builtins/mod.rs
index c528bd46424b..138a52633da8 100644
--- a/tvix/glue/src/builtins/mod.rs
+++ b/tvix/glue/src/builtins/mod.rs
@@ -8,8 +8,9 @@ mod derivation;
 mod errors;
 mod fetchers;
 mod import;
+mod utils;
 
-pub use errors::DerivationError;
+pub use errors::{DerivationError, FetcherError};
 
 /// Adds derivation-related builtins to the passed [tvix_eval::Evaluation].
 ///
diff --git a/tvix/glue/src/builtins/utils.rs b/tvix/glue/src/builtins/utils.rs
new file mode 100644
index 000000000000..586169beeb69
--- /dev/null
+++ b/tvix/glue/src/builtins/utils.rs
@@ -0,0 +1,36 @@
+use bstr::ByteSlice;
+use tvix_eval::{
+    generators::{self, GenCo},
+    CatchableErrorKind, CoercionKind, ErrorKind, NixAttrs, NixString, Value,
+};
+
+pub(super) async fn strong_importing_coerce_to_string(
+    co: &GenCo,
+    val: Value,
+) -> Result<NixString, CatchableErrorKind> {
+    let val = generators::request_force(co, val).await;
+    generators::request_string_coerce(
+        co,
+        val,
+        CoercionKind {
+            strong: true,
+            import_paths: true,
+        },
+    )
+    .await
+}
+
+pub(super) async fn select_string(
+    co: &GenCo,
+    attrs: &NixAttrs,
+    key: &str,
+) -> Result<Result<Option<String>, CatchableErrorKind>, ErrorKind> {
+    if let Some(attr) = attrs.select(key) {
+        match strong_importing_coerce_to_string(co, attr.clone()).await {
+            Err(cek) => return Ok(Err(cek)),
+            Ok(str) => return Ok(Ok(Some(str.to_str()?.to_owned()))),
+        }
+    }
+
+    Ok(Ok(None))
+}