From 5c3065b43a61a5fa019cbbb157984fc5eb81d439 Mon Sep 17 00:00:00 2001 From: Padraic-O-Mhuiris Date: Wed, 21 Feb 2024 16:49:07 +0000 Subject: feat(tvix/eval): implement `builtins.hashString` Implements md5, sha1, sha256 and sha512 using the related crates from the RustCrypto hashes project (https://github.com/RustCrypto/hashes) Change-Id: I00730dea44ec9ef85309edc27addab0ae88814b8 Reviewed-on: https://cl.tvl.fyi/c/depot/+/11005 Tested-by: BuildkiteCI Reviewed-by: aspen --- tvix/eval/Cargo.toml | 4 +++ tvix/eval/docs/builtins.md | 2 +- tvix/eval/src/builtins/mod.rs | 32 ++++++++++++++++------ tvix/eval/src/errors.rs | 10 +++++++ .../eval/src/tests/nix_tests/eval-okay-groupBy.exp | 1 + .../eval/src/tests/nix_tests/eval-okay-groupBy.nix | 5 ++++ .../src/tests/nix_tests/eval-okay-hashstring.exp | 1 + .../src/tests/nix_tests/eval-okay-hashstring.nix | 4 +++ .../nix_tests/notyetpassing/eval-okay-groupBy.exp | 1 - .../nix_tests/notyetpassing/eval-okay-groupBy.nix | 5 ---- .../notyetpassing/eval-okay-hashstring.exp | 1 - .../notyetpassing/eval-okay-hashstring.nix | 4 --- .../tvix_tests/eval-okay-builtins-hashString.exp | 1 + .../tvix_tests/eval-okay-builtins-hashString.nix | 6 ++++ tvix/eval/src/value/string.rs | 6 ++++ 15 files changed, 62 insertions(+), 21 deletions(-) create mode 100644 tvix/eval/src/tests/nix_tests/eval-okay-groupBy.exp create mode 100644 tvix/eval/src/tests/nix_tests/eval-okay-groupBy.nix create mode 100644 tvix/eval/src/tests/nix_tests/eval-okay-hashstring.exp create mode 100644 tvix/eval/src/tests/nix_tests/eval-okay-hashstring.nix delete mode 100644 tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-groupBy.exp delete mode 100644 tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-groupBy.nix delete mode 100644 tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashstring.exp delete mode 100644 tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashstring.nix create mode 100644 tvix/eval/src/tests/tvix_tests/eval-okay-builtins-hashString.exp create mode 100644 tvix/eval/src/tests/tvix_tests/eval-okay-builtins-hashString.nix (limited to 'tvix/eval') diff --git a/tvix/eval/Cargo.toml b/tvix/eval/Cargo.toml index 4f4551a349d2..e2bd07f8279f 100644 --- a/tvix/eval/Cargo.toml +++ b/tvix/eval/Cargo.toml @@ -33,6 +33,10 @@ tabwriter = "1.2" test-strategy = { version = "0.2.1", optional = true } toml = "0.6.0" xml-rs = "0.8.4" +sha2 = "0.10.8" +sha1 = "0.10.6" +md-5 = "0.10.6" +data-encoding = "2.5.0" [dev-dependencies] criterion = "0.5" diff --git a/tvix/eval/docs/builtins.md b/tvix/eval/docs/builtins.md index 39b59437e26b..eff761c7057d 100644 --- a/tvix/eval/docs/builtins.md +++ b/tvix/eval/docs/builtins.md @@ -66,7 +66,7 @@ The `impl` column indicates implementation status in tvix: | hasAttr | false | | | | | hasContext | false | | | | | hashFile | false | | false | todo | -| hashString | false | | | todo | +| hashString | false | | | | | head | false | | | | | import | true | | | | | intersectAttrs | false | | | | diff --git a/tvix/eval/src/builtins/mod.rs b/tvix/eval/src/builtins/mod.rs index 131f2b7bb201..119c0bda2dc3 100644 --- a/tvix/eval/src/builtins/mod.rs +++ b/tvix/eval/src/builtins/mod.rs @@ -5,9 +5,14 @@ use bstr::{ByteSlice, ByteVec}; use builtin_macros::builtins; +use data_encoding::HEXLOWER; use genawaiter::rc::Gen; use imbl::OrdMap; +use md5::Md5; use regex::Regex; +use sha1::Sha1; +use sha2::digest::Output; +use sha2::{Digest, Sha256, Sha512}; use std::cmp::{self, Ordering}; use std::collections::VecDeque; use std::collections::{BTreeMap, HashSet}; @@ -686,15 +691,24 @@ mod pure_builtins { #[builtin("hashString")] #[allow(non_snake_case)] - async fn builtin_hashString( - co: GenCo, - _algo: Value, - _string: Value, - ) -> Result { - // FIXME: propagate contexts here. - Ok(Value::from(CatchableErrorKind::UnimplementedFeature( - "hashString".into(), - ))) + async fn builtin_hashString(co: GenCo, algo: Value, s: Value) -> Result { + fn hash(b: &[u8]) -> Output { + let mut hasher = D::new(); + hasher.update(b); + hasher.finalize() + } + + let s = s.to_str()?; + + let encoded_hash = match algo.to_str()?.as_bytes() { + b"md5" => HEXLOWER.encode(hash::(&s).as_bstr()), + b"sha1" => HEXLOWER.encode(hash::(&s).as_bstr()), + b"sha256" => HEXLOWER.encode(hash::(&s).as_bstr()), + b"sha512" => HEXLOWER.encode(hash::(&s).as_bstr()), + _ => return Err(ErrorKind::UnknownHashType(s.into())), + }; + + Ok(Value::from(encoded_hash)) } #[builtin("head")] diff --git a/tvix/eval/src/errors.rs b/tvix/eval/src/errors.rs index db02093b8d65..652252dadfa0 100644 --- a/tvix/eval/src/errors.rs +++ b/tvix/eval/src/errors.rs @@ -229,6 +229,10 @@ pub enum ErrorKind { /// tvix-eval when returning a result to the user, never inside of /// eval code. CatchableError(CatchableErrorKind), + + /// Invalid hash type specified, must be one of "md5", "sha1", "sha256" + /// or "sha512" + UnknownHashType(String), } impl error::Error for Error { @@ -533,6 +537,10 @@ to a missing value in the attribute set(s) included via `with`."#, ErrorKind::CatchableError(inner) => { write!(f, "{}", inner) } + + ErrorKind::UnknownHashType(hash_type) => { + write!(f, "unknown hash type '{}'", hash_type) + } } } } @@ -821,6 +829,7 @@ impl Error { | ErrorKind::TvixBug { .. } | ErrorKind::NotImplemented(_) | ErrorKind::WithContext { .. } + | ErrorKind::UnknownHashType(_) | ErrorKind::CatchableError(_) => return None, }; @@ -866,6 +875,7 @@ impl Error { ErrorKind::NotSerialisableToJson(_) => "E036", ErrorKind::UnexpectedContext => "E037", ErrorKind::Utf8 => "E038", + ErrorKind::UnknownHashType(_) => "E039", // Special error code for errors from other Tvix // components. We may want to introduce a code namespacing diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-groupBy.exp b/tvix/eval/src/tests/nix_tests/eval-okay-groupBy.exp new file mode 100644 index 000000000000..bfca5652a59b --- /dev/null +++ b/tvix/eval/src/tests/nix_tests/eval-okay-groupBy.exp @@ -0,0 +1 @@ +{ "1" = [ 9 ]; "2" = [ 8 ]; "3" = [ 13 29 ]; "4" = [ 3 4 10 11 17 18 ]; "5" = [ 0 23 26 28 ]; "6" = [ 1 12 21 27 30 ]; "7" = [ 7 22 ]; "8" = [ 14 ]; "9" = [ 19 ]; b = [ 16 25 ]; c = [ 24 ]; d = [ 2 ]; e = [ 5 6 15 31 ]; f = [ 20 ]; } diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-groupBy.nix b/tvix/eval/src/tests/nix_tests/eval-okay-groupBy.nix new file mode 100644 index 000000000000..862d89dbd670 --- /dev/null +++ b/tvix/eval/src/tests/nix_tests/eval-okay-groupBy.nix @@ -0,0 +1,5 @@ +with import ./lib.nix; + +builtins.groupBy (n: + builtins.substring 0 1 (builtins.hashString "sha256" (toString n)) +) (range 0 31) diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-hashstring.exp b/tvix/eval/src/tests/nix_tests/eval-okay-hashstring.exp new file mode 100644 index 000000000000..d720a082ddb3 --- /dev/null +++ b/tvix/eval/src/tests/nix_tests/eval-okay-hashstring.exp @@ -0,0 +1 @@ +[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ] diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-hashstring.nix b/tvix/eval/src/tests/nix_tests/eval-okay-hashstring.nix new file mode 100644 index 000000000000..b0f62b245ca8 --- /dev/null +++ b/tvix/eval/src/tests/nix_tests/eval-okay-hashstring.nix @@ -0,0 +1,4 @@ +let + strings = [ "" "text 1" "text 2" ]; +in + builtins.concatLists (map (hash: map (builtins.hashString hash) strings) ["md5" "sha1" "sha256" "sha512"]) diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-groupBy.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-groupBy.exp deleted file mode 100644 index bfca5652a59b..000000000000 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-groupBy.exp +++ /dev/null @@ -1 +0,0 @@ -{ "1" = [ 9 ]; "2" = [ 8 ]; "3" = [ 13 29 ]; "4" = [ 3 4 10 11 17 18 ]; "5" = [ 0 23 26 28 ]; "6" = [ 1 12 21 27 30 ]; "7" = [ 7 22 ]; "8" = [ 14 ]; "9" = [ 19 ]; b = [ 16 25 ]; c = [ 24 ]; d = [ 2 ]; e = [ 5 6 15 31 ]; f = [ 20 ]; } diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-groupBy.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-groupBy.nix deleted file mode 100644 index 7e0eab28b036..000000000000 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-groupBy.nix +++ /dev/null @@ -1,5 +0,0 @@ -with import ./../lib.nix; - -builtins.groupBy (n: - builtins.substring 0 1 (builtins.hashString "sha256" (toString n)) -) (range 0 31) diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashstring.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashstring.exp deleted file mode 100644 index d720a082ddb3..000000000000 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashstring.exp +++ /dev/null @@ -1 +0,0 @@ -[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ] diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashstring.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashstring.nix deleted file mode 100644 index b0f62b245ca8..000000000000 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashstring.nix +++ /dev/null @@ -1,4 +0,0 @@ -let - strings = [ "" "text 1" "text 2" ]; -in - builtins.concatLists (map (hash: map (builtins.hashString hash) strings) ["md5" "sha1" "sha256" "sha512"]) diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-hashString.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-hashString.exp new file mode 100644 index 000000000000..e00b80e561fb --- /dev/null +++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-hashString.exp @@ -0,0 +1 @@ +[ "8a0614b4eaa4cffb7515ec101847e198" "8bd218cf61321d8aa05b3602b99f90d2d8cef3d6" "80ac06d74cb6c5d14af718ce8c3c1255969a1a595b76a3cf92354a95331a879a" "0edac513b6b0454705b553deda4c9b055da0939d26d2f73548862817ebeac5378cf64ff7a752ce1a0590a736735d3bbd9e8a7f04d93617cdf514313f5ab5baa4" ] diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-hashString.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-hashString.nix new file mode 100644 index 000000000000..aed723d3670a --- /dev/null +++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-hashString.nix @@ -0,0 +1,6 @@ +[ + (builtins.hashString "md5" "tvix") + (builtins.hashString "sha1" "tvix") + (builtins.hashString "sha256" "tvix") + (builtins.hashString "sha512" "tvix") +] diff --git a/tvix/eval/src/value/string.rs b/tvix/eval/src/value/string.rs index 2f1592546b85..6ce0d190c0a0 100644 --- a/tvix/eval/src/value/string.rs +++ b/tvix/eval/src/value/string.rs @@ -524,6 +524,12 @@ impl<'a> From<&'a NixString> for &'a BStr { } } +impl From for String { + fn from(s: NixString) -> Self { + s.to_string() + } +} + impl From for BString { fn from(s: NixString) -> Self { s.as_bstr().to_owned() -- cgit 1.4.1