From e0ffb4d47a9fb52c8ea3b0d8b1d002606b0bc4ee Mon Sep 17 00:00:00 2001 From: Padraic-O-Mhuiris Date: Thu, 22 Feb 2024 15:33:29 +0000 Subject: feat(tvix/eval): implement `builtins.hashFile` The primary addition is the `builtin_hashFile` function which attempts value-path coercion and on success reads the file to a nix_string. Similar to the `builtins_hashString` implementation, the hash is then derived using the `hash_nix_string` function in the introduced hash module which has extracted hashing functionality initially specified in cl/11005. Change-Id: I5e3ef0317223af99488ebb79efc2fb49b4cbc1b6 Reviewed-on: https://cl.tvl.fyi/c/depot/+/11007 Tested-by: BuildkiteCI Reviewed-by: flokli --- tvix/eval/docs/builtins.md | 2 +- tvix/eval/src/builtins/hash.rs | 25 ++++++++++++++++++++++ tvix/eval/src/builtins/impure.rs | 13 ++++++++++- tvix/eval/src/builtins/mod.rs | 25 +++------------------- .../src/tests/nix_tests/eval-okay-hashfile.exp | 1 + .../src/tests/nix_tests/eval-okay-hashfile.nix | 4 ++++ .../nix_tests/notyetpassing/eval-okay-hashfile.exp | 1 - .../nix_tests/notyetpassing/eval-okay-hashfile.nix | 4 ---- 8 files changed, 46 insertions(+), 29 deletions(-) create mode 100644 tvix/eval/src/builtins/hash.rs create mode 100644 tvix/eval/src/tests/nix_tests/eval-okay-hashfile.exp create mode 100644 tvix/eval/src/tests/nix_tests/eval-okay-hashfile.nix delete mode 100644 tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashfile.exp delete mode 100644 tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashfile.nix diff --git a/tvix/eval/docs/builtins.md b/tvix/eval/docs/builtins.md index eff761c7057d..26bbd1b2d344 100644 --- a/tvix/eval/docs/builtins.md +++ b/tvix/eval/docs/builtins.md @@ -65,7 +65,7 @@ The `impl` column indicates implementation status in tvix: | getEnv | false | | false | | | hasAttr | false | | | | | hasContext | false | | | | -| hashFile | false | | false | todo | +| hashFile | false | | false | | | hashString | false | | | | | head | false | | | | | import | true | | | | diff --git a/tvix/eval/src/builtins/hash.rs b/tvix/eval/src/builtins/hash.rs new file mode 100644 index 000000000000..6d07fc9b2dc8 --- /dev/null +++ b/tvix/eval/src/builtins/hash.rs @@ -0,0 +1,25 @@ +use bstr::ByteSlice; +use data_encoding::HEXLOWER; +use md5::Md5; +use sha1::Sha1; +use sha2::{digest::Output, Digest, Sha256, Sha512}; + +use crate::ErrorKind; + +fn hash(b: &[u8]) -> Output { + let mut hasher = D::new(); + hasher.update(b); + hasher.finalize() +} + +pub fn hash_nix_string(algo: impl AsRef<[u8]>, s: impl AsRef<[u8]>) -> Result { + match algo.as_ref() { + b"md5" => Ok(HEXLOWER.encode(hash::(s.as_ref()).as_bstr())), + b"sha1" => Ok(HEXLOWER.encode(hash::(s.as_ref()).as_bstr())), + b"sha256" => Ok(HEXLOWER.encode(hash::(s.as_ref()).as_bstr())), + b"sha512" => Ok(HEXLOWER.encode(hash::(s.as_ref()).as_bstr())), + _ => Err(ErrorKind::UnknownHashType( + algo.as_ref().as_bstr().to_string(), + )), + } +} diff --git a/tvix/eval/src/builtins/impure.rs b/tvix/eval/src/builtins/impure.rs index def6ce29094c..aad55c7331e8 100644 --- a/tvix/eval/src/builtins/impure.rs +++ b/tvix/eval/src/builtins/impure.rs @@ -21,7 +21,7 @@ mod impure_builtins { use std::os::unix::ffi::OsStrExt; use super::*; - use crate::builtins::coerce_value_to_path; + use crate::builtins::{coerce_value_to_path, hash::hash_nix_string}; #[builtin("getEnv")] async fn builtin_get_env(co: GenCo, var: Value) -> Result { @@ -30,6 +30,17 @@ mod impure_builtins { .into()) } + #[builtin("hashFile")] + #[allow(non_snake_case)] + async fn builtin_hashFile(co: GenCo, algo: Value, path: Value) -> Result { + let path = match coerce_value_to_path(&co, path).await? { + Err(cek) => return Ok(Value::from(cek)), + Ok(p) => p, + }; + let s = generators::request_read_to_string(&co, path).await; + hash_nix_string(algo.to_str()?, s.to_str()?).map(Value::from) + } + #[builtin("pathExists")] async fn builtin_path_exists(co: GenCo, path: Value) -> Result { match coerce_value_to_path(&co, path).await? { diff --git a/tvix/eval/src/builtins/mod.rs b/tvix/eval/src/builtins/mod.rs index 119c0bda2dc3..0e0890f7cd28 100644 --- a/tvix/eval/src/builtins/mod.rs +++ b/tvix/eval/src/builtins/mod.rs @@ -5,14 +5,9 @@ use bstr::{ByteSlice, ByteVec}; use builtin_macros::builtins; -use data_encoding::HEXLOWER; use genawaiter::rc::Gen; use imbl::OrdMap; -use md5::Md5; use regex::Regex; -use sha1::Sha1; -use sha2::digest::Output; -use sha2::{Digest, Sha256, Sha512}; use std::cmp::{self, Ordering}; use std::collections::VecDeque; use std::collections::{BTreeMap, HashSet}; @@ -24,12 +19,14 @@ use crate::vm::generators::{self, GenCo}; use crate::warnings::WarningKind; use crate::{ self as tvix_eval, + builtins::hash::hash_nix_string, errors::{CatchableErrorKind, ErrorKind}, value::{CoercionKind, NixAttrs, NixList, NixString, Thunk, Value}, }; use self::versions::{VersionPart, VersionPartsIter}; +mod hash; mod to_xml; mod versions; @@ -692,23 +689,7 @@ mod pure_builtins { #[builtin("hashString")] #[allow(non_snake_case)] async fn builtin_hashString(co: GenCo, algo: Value, s: Value) -> Result { - fn hash(b: &[u8]) -> Output { - let mut hasher = D::new(); - hasher.update(b); - hasher.finalize() - } - - let s = s.to_str()?; - - let encoded_hash = match algo.to_str()?.as_bytes() { - b"md5" => HEXLOWER.encode(hash::(&s).as_bstr()), - b"sha1" => HEXLOWER.encode(hash::(&s).as_bstr()), - b"sha256" => HEXLOWER.encode(hash::(&s).as_bstr()), - b"sha512" => HEXLOWER.encode(hash::(&s).as_bstr()), - _ => return Err(ErrorKind::UnknownHashType(s.into())), - }; - - Ok(Value::from(encoded_hash)) + hash_nix_string(algo.to_str()?, s.to_str()?).map(Value::from) } #[builtin("head")] diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-hashfile.exp b/tvix/eval/src/tests/nix_tests/eval-okay-hashfile.exp new file mode 100644 index 000000000000..ff1e8293ef22 --- /dev/null +++ b/tvix/eval/src/tests/nix_tests/eval-okay-hashfile.exp @@ -0,0 +1 @@ +[ "d3b07384d113edec49eaa6238ad5ff00" "0f343b0931126a20f133d67c2b018a3b" "f1d2d2f924e986ac86fdf7b36c94bcdf32beec15" "60cacbf3d72e1e7834203da608037b1bf83b40e8" "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c" "5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" "0cf9180a764aba863a67b6d72f0918bc131c6772642cb2dce5a34f0a702f9470ddc2bf125c12198b1995c233c34b4afd346c54a2334c350a948a51b6e8b4e6b6" "8efb4f73c5655351c444eb109230c556d39e2c7624e9c11abc9e3fb4b9b9254218cc5085b454a9698d085cfa92198491f07a723be4574adc70617b73eb0b6461" ] diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-hashfile.nix b/tvix/eval/src/tests/nix_tests/eval-okay-hashfile.nix new file mode 100644 index 000000000000..aff5a1856814 --- /dev/null +++ b/tvix/eval/src/tests/nix_tests/eval-okay-hashfile.nix @@ -0,0 +1,4 @@ +let + paths = [ ./data ./binary-data ]; +in + builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"]) diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashfile.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashfile.exp deleted file mode 100644 index ff1e8293ef22..000000000000 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashfile.exp +++ /dev/null @@ -1 +0,0 @@ -[ "d3b07384d113edec49eaa6238ad5ff00" "0f343b0931126a20f133d67c2b018a3b" "f1d2d2f924e986ac86fdf7b36c94bcdf32beec15" "60cacbf3d72e1e7834203da608037b1bf83b40e8" "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c" "5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" "0cf9180a764aba863a67b6d72f0918bc131c6772642cb2dce5a34f0a702f9470ddc2bf125c12198b1995c233c34b4afd346c54a2334c350a948a51b6e8b4e6b6" "8efb4f73c5655351c444eb109230c556d39e2c7624e9c11abc9e3fb4b9b9254218cc5085b454a9698d085cfa92198491f07a723be4574adc70617b73eb0b6461" ] diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashfile.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashfile.nix deleted file mode 100644 index 8c9de66b7ecf..000000000000 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashfile.nix +++ /dev/null @@ -1,4 +0,0 @@ -let - paths = [ ./../data ./../binary-data ]; -in - builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"]) -- cgit 1.4.1