From eff2cc4f6844875f3684bfb17a134ccd0b5547a4 Mon Sep 17 00:00:00 2001 From: Brian Olsen Date: Fri, 1 Mar 2024 21:51:23 +0100 Subject: fix(tvix/nix-compat): Make CAHash deserialize more formats Currently CAHash only deserializes the hash in hex code while the serializer outputs a nixbase32 hash. This means that you can't currently deserialize what has been serialized. This change makes deserialize support any digest format (so hex, nixbase32 and base64) as well as flattens the deserialize code and error handling. It also implements serde methods of HashAlgo directly using Display and TryFrom implementations because otherwise these would get serialized as eg. Sha256 instead of sha256 which also broke CAHash serialize/deserialize. Change-Id: I1941a72eaec741e4956292adaaf0115b97f260ba Reviewed-on: https://cl.tvl.fyi/c/depot/+/11082 Tested-by: BuildkiteCI Reviewed-by: flokli --- tvix/nix-compat/src/derivation/output.rs | 30 ++++ tvix/nix-compat/src/nixhash/algos.rs | 21 ++- tvix/nix-compat/src/nixhash/ca_hash.rs | 246 ++++++++++++++++++++++--------- 3 files changed, 230 insertions(+), 67 deletions(-) (limited to 'tvix/nix-compat') diff --git a/tvix/nix-compat/src/derivation/output.rs b/tvix/nix-compat/src/derivation/output.rs index e2a8282c7686..266617f587f8 100644 --- a/tvix/nix-compat/src/derivation/output.rs +++ b/tvix/nix-compat/src/derivation/output.rs @@ -157,3 +157,33 @@ fn deserialize_with_error_missing_hash_fixed_output() { assert!(output.is_err()); } + +#[test] +fn serialize_deserialize() { + let json_bytes = r#" + { + "path": "/nix/store/00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432" + }"#; + let output: Output = serde_json::from_str(json_bytes).expect("must parse"); + + let s = serde_json::to_string(&output).expect("Serialize"); + let output2: Output = serde_json::from_str(&s).expect("must parse again"); + + assert_eq!(output, output2); +} + +#[test] +fn serialize_deserialize_fixed() { + let json_bytes = r#" + { + "path": "/nix/store/00bgd045z0d4icpbc2yyz4gx48ak44la-net-tools-1.60_p20170221182432", + "hash": "08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba", + "hashAlgo": "r:sha256" + }"#; + let output: Output = serde_json::from_str(json_bytes).expect("must parse"); + + let s = serde_json::to_string_pretty(&output).expect("Serialize"); + let output2: Output = serde_json::from_str(&s).expect("must parse again"); + + assert_eq!(output, output2); +} diff --git a/tvix/nix-compat/src/nixhash/algos.rs b/tvix/nix-compat/src/nixhash/algos.rs index f189eba09db6..ac8915314c83 100644 --- a/tvix/nix-compat/src/nixhash/algos.rs +++ b/tvix/nix-compat/src/nixhash/algos.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; use crate::nixhash::Error; /// This are the hash algorithms supported by cppnix. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum HashAlgo { Md5, Sha1, @@ -36,6 +36,25 @@ impl Display for HashAlgo { } } +impl Serialize for HashAlgo { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.collect_str(&self) + } +} + +impl<'de> Deserialize<'de> for HashAlgo { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s: &str = Deserialize::deserialize(deserializer)?; + HashAlgo::try_from(s).map_err(serde::de::Error::custom) + } +} + /// TODO(Raito): this could be automated via macros, I suppose. /// But this may be more expensive than just doing it by hand /// and ensuring that is kept in sync. diff --git a/tvix/nix-compat/src/nixhash/ca_hash.rs b/tvix/nix-compat/src/nixhash/ca_hash.rs index d69d19ce68f3..1c2f61dc6c65 100644 --- a/tvix/nix-compat/src/nixhash/ca_hash.rs +++ b/tvix/nix-compat/src/nixhash/ca_hash.rs @@ -1,5 +1,5 @@ use crate::nixbase32; -use crate::nixhash::{self, HashAlgo, NixHash}; +use crate::nixhash::{HashAlgo, NixHash}; use serde::de::Unexpected; use serde::ser::SerializeMap; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -7,7 +7,7 @@ use serde_json::{Map, Value}; use std::borrow::Cow; use super::algos::SUPPORTED_ALGOS; -use super::from_algo_and_digest; +use super::decode_digest; /// A Nix CAHash describes a content-addressed hash of a path. /// @@ -99,73 +99,40 @@ impl CAHash { return Ok(None); } - let digest: Vec = { - if let Some(v) = map.get("hash") { - if let Some(s) = v.as_str() { - data_encoding::HEXLOWER - .decode(s.as_bytes()) - .map_err(|e| serde::de::Error::custom(e.to_string()))? - } else { - return Err(serde::de::Error::invalid_type( - Unexpected::Other(&v.to_string()), - &"a string", - )); - } - } else { - return Err(serde::de::Error::missing_field( - "couldn't extract `hash` key but `hashAlgo` key present", - )); - } + let hash_algo_v = map.get("hashAlgo").ok_or_else(|| { + serde::de::Error::missing_field( + "couldn't extract `hashAlgo` key, but `hash` key present", + ) + })?; + let hash_algo = hash_algo_v.as_str().ok_or_else(|| { + serde::de::Error::invalid_type(Unexpected::Other(&hash_algo_v.to_string()), &"a string") + })?; + let (mode_is_nar, hash_algo) = if let Some(s) = hash_algo.strip_prefix("r:") { + (true, s) + } else { + (false, hash_algo) }; + let hash_algo = HashAlgo::try_from(hash_algo).map_err(|e| { + serde::de::Error::invalid_value( + Unexpected::Other(&e.to_string()), + &format!("one of {}", SUPPORTED_ALGOS.join(",")).as_str(), + ) + })?; - if let Some(v) = map.get("hashAlgo") { - if let Some(s) = v.as_str() { - match s.strip_prefix("r:") { - Some(rest) => Ok(Some(Self::Nar( - from_algo_and_digest( - HashAlgo::try_from(rest).map_err(|e| { - serde::de::Error::invalid_value( - Unexpected::Other(&e.to_string()), - &format!("one of {}", SUPPORTED_ALGOS.join(",")).as_str(), - ) - })?, - &digest, - ) - .map_err(|e: nixhash::Error| { - serde::de::Error::invalid_value( - Unexpected::Other(&e.to_string()), - &"a digest with right length", - ) - })?, - ))), - None => Ok(Some(Self::Flat( - from_algo_and_digest( - HashAlgo::try_from(s).map_err(|e| { - serde::de::Error::invalid_value( - Unexpected::Other(&e.to_string()), - &format!("one of {}", SUPPORTED_ALGOS.join(",")).as_str(), - ) - })?, - &digest, - ) - .map_err(|e: nixhash::Error| { - serde::de::Error::invalid_value( - Unexpected::Other(&e.to_string()), - &"a digest with right length", - ) - })?, - ))), - } - } else { - Err(serde::de::Error::invalid_type( - Unexpected::Other(&v.to_string()), - &"a string", - )) - } + let hash_v = map.get("hash").ok_or_else(|| { + serde::de::Error::missing_field( + "couldn't extract `hash` key but `hashAlgo` key present", + ) + })?; + let hash = hash_v.as_str().ok_or_else(|| { + serde::de::Error::invalid_type(Unexpected::Other(&hash_v.to_string()), &"a string") + })?; + let hash = decode_digest(hash.as_bytes(), hash_algo) + .map_err(|e| serde::de::Error::custom(e.to_string()))?; + if mode_is_nar { + Ok(Some(Self::Nar(hash))) } else { - Err(serde::de::Error::missing_field( - "couldn't extract `hashAlgo` key, but `hash` key present", - )) + Ok(Some(Self::Flat(hash))) } } } @@ -211,3 +178,150 @@ impl<'de> Deserialize<'de> for CAHash { } } } + +#[cfg(test)] +mod tests { + use crate::{derivation::CAHash, nixhash}; + + #[test] + fn serialize_flat() { + let json_bytes = r#"{ + "hash": "1fnf2m46ya7r7afkcb8ba2j0sc4a85m749sh9jz64g4hx6z3r088", + "hashAlgo": "sha256" +}"#; + let hash = CAHash::Flat( + nixhash::from_nix_str( + "sha256:08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba", + ) + .unwrap(), + ); + let serialized = serde_json::to_string_pretty(&hash).unwrap(); + assert_eq!(serialized, json_bytes); + } + + #[test] + fn serialize_nar() { + let json_bytes = r#"{ + "hash": "1fnf2m46ya7r7afkcb8ba2j0sc4a85m749sh9jz64g4hx6z3r088", + "hashAlgo": "r:sha256" +}"#; + let hash = CAHash::Nar( + nixhash::from_nix_str( + "sha256:08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba", + ) + .unwrap(), + ); + let serialized = serde_json::to_string_pretty(&hash).unwrap(); + assert_eq!(serialized, json_bytes); + } + + #[test] + fn deserialize_flat() { + let json_bytes = r#" + { + "hash": "08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba", + "hashAlgo": "sha256" + }"#; + let hash: CAHash = serde_json::from_str(json_bytes).expect("must parse"); + + assert_eq!( + hash, + CAHash::Flat( + nixhash::from_nix_str( + "sha256:08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba" + ) + .unwrap() + ) + ); + } + + #[test] + fn deserialize_hex() { + let json_bytes = r#" + { + "hash": "08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba", + "hashAlgo": "r:sha256" + }"#; + let hash: CAHash = serde_json::from_str(json_bytes).expect("must parse"); + + assert_eq!( + hash, + CAHash::Nar( + nixhash::from_nix_str( + "sha256:08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba" + ) + .unwrap() + ) + ); + } + + #[test] + fn deserialize_nixbase32() { + let json_bytes = r#" + { + "hash": "1fnf2m46ya7r7afkcb8ba2j0sc4a85m749sh9jz64g4hx6z3r088", + "hashAlgo": "r:sha256" + }"#; + let hash: CAHash = serde_json::from_str(json_bytes).expect("must parse"); + + assert_eq!( + hash, + CAHash::Nar( + nixhash::from_nix_str( + "sha256:08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba" + ) + .unwrap() + ) + ); + } + + #[test] + fn deserialize_base64() { + let json_bytes = r#" + { + "hash": "CIE8vumQPGK+TFAncmpBijANpFALLTadOvkob0gVzro=", + "hashAlgo": "r:sha256" + }"#; + let hash: CAHash = serde_json::from_str(json_bytes).expect("must parse"); + + assert_eq!( + hash, + CAHash::Nar( + nixhash::from_nix_str( + "sha256:08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba" + ) + .unwrap() + ) + ); + } + + #[test] + fn serialize_deserialize_nar() { + let json_bytes = r#" + { + "hash": "08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba", + "hashAlgo": "r:sha256" + }"#; + let hash: CAHash = serde_json::from_str(json_bytes).expect("must parse"); + + let serialized = serde_json::to_string(&hash).expect("Serialize"); + let hash2: CAHash = serde_json::from_str(&serialized).expect("must parse again"); + + assert_eq!(hash, hash2); + } + + #[test] + fn serialize_deserialize_flat() { + let json_bytes = r#" + { + "hash": "08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba", + "hashAlgo": "sha256" + }"#; + let hash: CAHash = serde_json::from_str(json_bytes).expect("must parse"); + + let serialized = serde_json::to_string(&hash).expect("Serialize"); + let hash2: CAHash = serde_json::from_str(&serialized).expect("must parse again"); + + assert_eq!(hash, hash2); + } +} -- cgit 1.4.1