From eda5d4da377913c015f6658abd5d0f2c1366db48 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Sat, 18 Nov 2023 16:33:54 +0200 Subject: feat(tvix/store): From<&nix_compat::...::NarInfo<'_>> for PathInfo This allows converting from the NarInfo falling out of the NarInfo parser (which is a bit annoying to handle due to lifetimes) to the PathInfo proto struct. The narinfo field, containing most of the data from the original NARInfo file, as well as the references (bytes) are populated. The node field is not populated, because it requires ingesting the NAR itself to describe the root node. Change-Id: I9c04dd6ad4cae556b455188a4255e34b4f6443c5 Reviewed-on: https://cl.tvl.fyi/c/depot/+/10067 Reviewed-by: raitobezarius Tested-by: BuildkiteCI Autosubmit: flokli --- tvix/store/src/proto/mod.rs | 67 +++++++++++++++++++- tvix/store/src/proto/tests/pathinfo.rs | 111 ++++++++++++++++++++++++++++++++- 2 files changed, 176 insertions(+), 2 deletions(-) (limited to 'tvix/store/src') diff --git a/tvix/store/src/proto/mod.rs b/tvix/store/src/proto/mod.rs index 232086c0f8..558d7bc283 100644 --- a/tvix/store/src/proto/mod.rs +++ b/tvix/store/src/proto/mod.rs @@ -1,7 +1,11 @@ #![allow(clippy::derive_partial_eq_without_eq, non_snake_case)] +use bytes::Bytes; use data_encoding::BASE64; // https://github.com/hyperium/tonic/issues/1056 -use nix_compat::store_path; +use nix_compat::{ + nixhash::{CAHash, NixHash}, + store_path, +}; use thiserror::Error; use tvix_castore::proto::{self as castorepb, NamedNode, ValidateNodeError}; @@ -173,3 +177,64 @@ impl PathInfo { Ok(root_nix_path) } } + +impl From<&nix_compat::narinfo::NarInfo<'_>> for NarInfo { + /// Converts from a NarInfo (returned from the NARInfo parser) to the proto- + /// level NarInfo struct. + fn from(value: &nix_compat::narinfo::NarInfo<'_>) -> Self { + let signatures = value + .signatures + .iter() + .map(|sig| nar_info::Signature { + name: sig.name().to_string(), + data: Bytes::copy_from_slice(sig.bytes()), + }) + .collect(); + + let ca = value.ca.as_ref().map(|ca_hash| nar_info::Ca { + r#type: match ca_hash { + CAHash::Flat(NixHash::Md5(_)) => nar_info::ca::Hash::FlatMd5.into(), + CAHash::Flat(NixHash::Sha1(_)) => nar_info::ca::Hash::FlatSha1.into(), + CAHash::Flat(NixHash::Sha256(_)) => nar_info::ca::Hash::FlatSha256.into(), + CAHash::Flat(NixHash::Sha512(_)) => nar_info::ca::Hash::FlatSha512.into(), + CAHash::Nar(NixHash::Md5(_)) => nar_info::ca::Hash::NarMd5.into(), + CAHash::Nar(NixHash::Sha1(_)) => nar_info::ca::Hash::NarSha1.into(), + CAHash::Nar(NixHash::Sha256(_)) => nar_info::ca::Hash::NarSha256.into(), + CAHash::Nar(NixHash::Sha512(_)) => nar_info::ca::Hash::NarSha512.into(), + CAHash::Text(_) => nar_info::ca::Hash::TextSha256.into(), + }, + digest: Bytes::copy_from_slice(ca_hash.digest().digest_as_bytes()), + }); + + NarInfo { + nar_size: value.nar_size, + nar_sha256: Bytes::copy_from_slice(&value.nar_hash), + signatures, + reference_names: value.references.iter().map(|r| r.to_string()).collect(), + deriver: value.deriver.as_ref().map(|sp| StorePath { + // The parser already errors out with an error if the .drv suffix was missing, + // so you can only miss the suffix if you're manually constructing, + // which means we can unwrap here. + name: sp.name().strip_suffix(".drv").unwrap().to_owned(), + digest: Bytes::copy_from_slice(sp.digest()), + }), + ca, + } + } +} + +impl From<&nix_compat::narinfo::NarInfo<'_>> for PathInfo { + /// Converts from a NarInfo (returned from the NARInfo parser) to a PathInfo + /// struct with the node set to None. + fn from(value: &nix_compat::narinfo::NarInfo<'_>) -> Self { + Self { + node: None, + references: value + .references + .iter() + .map(|x| Bytes::copy_from_slice(x.digest())) + .collect(), + narinfo: Some(value.into()), + } + } +} diff --git a/tvix/store/src/proto/tests/pathinfo.rs b/tvix/store/src/proto/tests/pathinfo.rs index c296512f66..31f4790b73 100644 --- a/tvix/store/src/proto/tests/pathinfo.rs +++ b/tvix/store/src/proto/tests/pathinfo.rs @@ -1,6 +1,8 @@ -use crate::proto::{PathInfo, ValidatePathInfoError}; +use crate::proto::{nar_info::Signature, NarInfo, PathInfo, ValidatePathInfoError}; use crate::tests::fixtures::*; use bytes::Bytes; +use data_encoding::BASE64; +use nix_compat::nixbase32; use nix_compat::store_path::{self, StorePath}; use std::str::FromStr; use test_case::test_case; @@ -295,3 +297,110 @@ fn validate_invalid_deriver() { e => panic!("unexpected error: {:?}", e), } } + +#[test] +fn from_nixcompat_narinfo() { + let narinfo_parsed = nix_compat::narinfo::NarInfo::parse( + r#"StorePath: /nix/store/s66mzxpvicwk07gjbjfw9izjfa797vsw-hello-2.12.1 +URL: nar/1nhgq6wcggx0plpy4991h3ginj6hipsdslv4fd4zml1n707j26yq.nar.xz +Compression: xz +FileHash: sha256:1nhgq6wcggx0plpy4991h3ginj6hipsdslv4fd4zml1n707j26yq +FileSize: 50088 +NarHash: sha256:0yzhigwjl6bws649vcs2asa4lbs8hg93hyix187gc7s7a74w5h80 +NarSize: 226488 +References: 3n58xw4373jp0ljirf06d8077j15pc4j-glibc-2.37-8 s66mzxpvicwk07gjbjfw9izjfa797vsw-hello-2.12.1 +Deriver: ib3sh3pcz10wsmavxvkdbayhqivbghlq-hello-2.12.1.drv +Sig: cache.nixos.org-1:8ijECciSFzWHwwGVOIVYdp2fOIOJAfmzGHPQVwpktfTQJF6kMPPDre7UtFw3o+VqenC5P8RikKOAAfN7CvPEAg=="#).expect("must parse"); + + assert_eq!( + PathInfo { + node: None, + references: vec![ + Bytes::copy_from_slice(&nixbase32::decode_fixed::<20>("3n58xw4373jp0ljirf06d8077j15pc4j").unwrap()), + Bytes::copy_from_slice(&nixbase32::decode_fixed::<20>("s66mzxpvicwk07gjbjfw9izjfa797vsw").unwrap()), + ], + narinfo: Some( + NarInfo { + nar_size: 226488, + nar_sha256: Bytes::copy_from_slice( + &nixbase32::decode_fixed::<32>("0yzhigwjl6bws649vcs2asa4lbs8hg93hyix187gc7s7a74w5h80".as_bytes()) + .unwrap() + ), + signatures: vec![Signature { + name: "cache.nixos.org-1".to_string(), + data: BASE64.decode("8ijECciSFzWHwwGVOIVYdp2fOIOJAfmzGHPQVwpktfTQJF6kMPPDre7UtFw3o+VqenC5P8RikKOAAfN7CvPEAg==".as_bytes()).unwrap().into(), + }], + reference_names: vec![ + "3n58xw4373jp0ljirf06d8077j15pc4j-glibc-2.37-8".to_string(), + "s66mzxpvicwk07gjbjfw9izjfa797vsw-hello-2.12.1".to_string() + ], + deriver: Some(crate::proto::StorePath { + digest: Bytes::copy_from_slice(&nixbase32::decode_fixed::<20>("ib3sh3pcz10wsmavxvkdbayhqivbghlq").unwrap()), + name: "hello-2.12.1".to_string(), + }), + ca: None, + } + ) + }, + (&narinfo_parsed).into(), + ); +} + +#[test] +fn from_nixcompat_narinfo_fod() { + let narinfo_parsed = nix_compat::narinfo::NarInfo::parse( + r#"StorePath: /nix/store/pa10z4ngm0g83kx9mssrqzz30s84vq7k-hello-2.12.1.tar.gz +URL: nar/1zjrhzhaizsrlsvdkqfl073vivmxcqnzkff4s50i0cdf541ary1r.nar.xz +Compression: xz +FileHash: sha256:1zjrhzhaizsrlsvdkqfl073vivmxcqnzkff4s50i0cdf541ary1r +FileSize: 1033524 +NarHash: sha256:1lvqpbk2k1sb39z8jfxixf7p7v8sj4z6mmpa44nnmff3w1y6h8lh +NarSize: 1033416 +References: +Deriver: dyivpmlaq2km6c11i0s6bi6mbsx0ylqf-hello-2.12.1.tar.gz.drv +Sig: cache.nixos.org-1:ywnIG629nQZQhEr6/HLDrLT/mUEp5J1LC6NmWSlJRWL/nM7oGItJQUYWGLvYGhSQvHrhIuvMpjNmBNh/WWqCDg== +CA: fixed:sha256:086vqwk2wl8zfs47sq2xpjc9k066ilmb8z6dn0q6ymwjzlm196cd"# + ).expect("must parse"); + + assert_eq!( + PathInfo { + node: None, + references: vec![], + narinfo: Some( + NarInfo { + nar_size: 1033416, + nar_sha256: Bytes::copy_from_slice( + &nixbase32::decode_fixed::<32>( + "1lvqpbk2k1sb39z8jfxixf7p7v8sj4z6mmpa44nnmff3w1y6h8lh" + ) + .unwrap() + ), + signatures: vec![Signature { + name: "cache.nixos.org-1".to_string(), + data: BASE64 + .decode("ywnIG629nQZQhEr6/HLDrLT/mUEp5J1LC6NmWSlJRWL/nM7oGItJQUYWGLvYGhSQvHrhIuvMpjNmBNh/WWqCDg==".as_bytes()) + .unwrap() + .into(), + }], + reference_names: vec![], + deriver: Some(crate::proto::StorePath { + digest: Bytes::copy_from_slice( + &nixbase32::decode_fixed::<20>("dyivpmlaq2km6c11i0s6bi6mbsx0ylqf").unwrap() + ), + name: "hello-2.12.1.tar.gz".to_string(), + }), + ca: Some(crate::proto::nar_info::Ca { + r#type: crate::proto::nar_info::ca::Hash::FlatSha256.into(), + digest: Bytes::copy_from_slice( + &nixbase32::decode_fixed::<32>( + "086vqwk2wl8zfs47sq2xpjc9k066ilmb8z6dn0q6ymwjzlm196cd" + ) + .unwrap() + ) + }), + } + ), + }, + (&narinfo_parsed).into() + ); +} -- cgit 1.4.1