From 2d2c4322d93308ddffe1647466abd91025af6a68 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Wed, 11 Oct 2023 00:22:40 +0200 Subject: feat(tvix/store/protos): add Deriver field to PathInfo This uses the newly introduced StorePath message type to add a Deriver field to the PathInfo message. Support for validation is added to both the golang and rust implementation. This includes extending unit tests. Change-Id: Ifc3eb3263fa25b9eec260db354cd74234c40af7e Reviewed-on: https://cl.tvl.fyi/c/depot/+/9647 Reviewed-by: Connor Brewster Tested-by: BuildkiteCI --- tvix/store/protos/pathinfo.go | 31 ++++++++++----- tvix/store/protos/pathinfo.pb.go | 73 ++++++++++++++++++++-------------- tvix/store/protos/pathinfo.proto | 4 ++ tvix/store/protos/pathinfo_test.go | 26 ++++++++++++ tvix/store/src/bin/tvix-store.rs | 1 + tvix/store/src/proto/mod.rs | 16 +++++++- tvix/store/src/proto/tests/pathinfo.rs | 31 +++++++++++++++ tvix/store/src/tests/fixtures.rs | 1 + 8 files changed, 144 insertions(+), 39 deletions(-) (limited to 'tvix/store') diff --git a/tvix/store/protos/pathinfo.go b/tvix/store/protos/pathinfo.go index 2c718c6245d5..9b51b5266cdc 100644 --- a/tvix/store/protos/pathinfo.go +++ b/tvix/store/protos/pathinfo.go @@ -60,42 +60,55 @@ func (p *PathInfo) Validate() (*storepath.StorePath, error) { // for all three node types, ensure the name properly parses to a store path, // and in case it refers to a digest, ensure it has the right length. + var storePath *storepath.StorePath + var err error + if node := rootNode.GetDirectory(); node != nil { if len(node.Digest) != 32 { return nil, fmt.Errorf("invalid digest size for %s, expected %d, got %d", node.Name, 32, len(node.Digest)) } - storePath, err := storepath.FromString(string(node.GetName())) + storePath, err = storepath.FromString(string(node.GetName())) if err != nil { return nil, fmt.Errorf("unable to parse %s as StorePath: %w", node.Name, err) } - return storePath, nil - } else if node := rootNode.GetFile(); node != nil { if len(node.Digest) != 32 { return nil, fmt.Errorf("invalid digest size for %s, expected %d, got %d", node.Name, 32, len(node.Digest)) } - storePath, err := storepath.FromString(string(node.GetName())) + storePath, err = storepath.FromString(string(node.GetName())) if err != nil { return nil, fmt.Errorf("unable to parse %s as StorePath: %w", node.Name, err) } - return storePath, nil - } else if node := rootNode.GetSymlink(); node != nil { - storePath, err := storepath.FromString(string(node.GetName())) + storePath, err = storepath.FromString(string(node.GetName())) if err != nil { return nil, fmt.Errorf("unable to parse %s as StorePath: %w", node.Name, err) } - return storePath, nil - } else { // this would only happen if we introduced a new type panic("unreachable") } + + // If the Deriver field is populated, ensure it parses to a StorePath. + // We can't check for it to *not* end with .drv, as the .drv files produced by + // recursive Nix end with multiple .drv suffixes, and only one is popped when + // converting to this field. + if p.Deriver != nil { + storePath := storepath.StorePath{ + Name: string(p.Deriver.GetName()), + Digest: p.Deriver.GetDigest(), + } + if err := storePath.Validate(); err != nil { + return nil, fmt.Errorf("invalid deriver field: %w", err) + } + } + + return storePath, nil } diff --git a/tvix/store/protos/pathinfo.pb.go b/tvix/store/protos/pathinfo.pb.go index d6b517296448..b8296114f602 100644 --- a/tvix/store/protos/pathinfo.pb.go +++ b/tvix/store/protos/pathinfo.pb.go @@ -39,6 +39,9 @@ type PathInfo struct { References [][]byte `protobuf:"bytes,2,rep,name=references,proto3" json:"references,omitempty"` // see below. Narinfo *NARInfo `protobuf:"bytes,3,opt,name=narinfo,proto3" json:"narinfo,omitempty"` + // The StorePath of the .drv file producing this output. + // The .drv suffix is omitted in its `name` field. + Deriver *StorePath `protobuf:"bytes,4,opt,name=deriver,proto3" json:"deriver,omitempty"` } func (x *PathInfo) Reset() { @@ -94,6 +97,13 @@ func (x *PathInfo) GetNarinfo() *NARInfo { return nil } +func (x *PathInfo) GetDeriver() *StorePath { + if x != nil { + return x.Deriver + } + return nil +} + // Represents a path in the Nix store (a direct child of STORE_DIR). // It is commonly formatted by a nixbase32-encoding the digest, and // concatenating the name, separated by a `-`. @@ -306,7 +316,7 @@ var file_tvix_store_protos_pathinfo_proto_rawDesc = []byte{ 0x74, 0x6f, 0x12, 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x1a, 0x21, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x87, 0x01, 0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, + 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xbb, 0x01, 0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x29, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x1e, 0x0a, 0x0a, @@ -314,28 +324,32 @@ var file_tvix_store_protos_pathinfo_proto_rawDesc = []byte{ 0x52, 0x0a, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x30, 0x0a, 0x07, 0x6e, 0x61, 0x72, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x41, - 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x69, 0x6e, 0x66, 0x6f, 0x22, 0x37, - 0x0a, 0x09, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, - 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0xe3, 0x01, 0x0a, 0x07, 0x4e, 0x41, 0x52, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, - 0x0a, 0x0a, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x09, 0x6e, 0x61, 0x72, 0x53, 0x68, 0x61, 0x32, 0x35, 0x36, 0x12, 0x40, 0x0a, - 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, - 0x27, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, - 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, - 0x6e, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x1a, 0x33, 0x0a, 0x09, 0x53, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, - 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x42, 0x28, 0x5a, - 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, - 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, - 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x32, + 0x0a, 0x07, 0x64, 0x65, 0x72, 0x69, 0x76, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x18, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, + 0x53, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x61, 0x74, 0x68, 0x52, 0x07, 0x64, 0x65, 0x72, 0x69, 0x76, + 0x65, 0x72, 0x22, 0x37, 0x0a, 0x09, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, + 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0xe3, 0x01, 0x0a, 0x07, + 0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x61, 0x72, 0x5f, 0x73, + 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x53, 0x69, + 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6e, 0x61, 0x72, 0x53, 0x68, 0x61, 0x32, 0x35, + 0x36, 0x12, 0x40, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, + 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, + 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x53, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x72, 0x65, + 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x1a, 0x33, 0x0a, 0x09, + 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, + 0x61, 0x42, 0x28, 0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, + 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x73, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( @@ -361,12 +375,13 @@ var file_tvix_store_protos_pathinfo_proto_goTypes = []interface{}{ var file_tvix_store_protos_pathinfo_proto_depIdxs = []int32{ 4, // 0: tvix.store.v1.PathInfo.node:type_name -> tvix.castore.v1.Node 2, // 1: tvix.store.v1.PathInfo.narinfo:type_name -> tvix.store.v1.NARInfo - 3, // 2: tvix.store.v1.NARInfo.signatures:type_name -> tvix.store.v1.NARInfo.Signature - 3, // [3:3] is the sub-list for method output_type - 3, // [3:3] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 1, // 2: tvix.store.v1.PathInfo.deriver:type_name -> tvix.store.v1.StorePath + 3, // 3: tvix.store.v1.NARInfo.signatures:type_name -> tvix.store.v1.NARInfo.Signature + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_tvix_store_protos_pathinfo_proto_init() } diff --git a/tvix/store/protos/pathinfo.proto b/tvix/store/protos/pathinfo.proto index 080e3f6b49a2..ac528628ec88 100644 --- a/tvix/store/protos/pathinfo.proto +++ b/tvix/store/protos/pathinfo.proto @@ -21,6 +21,10 @@ message PathInfo { // see below. NARInfo narinfo = 3; + + // The StorePath of the .drv file producing this output. + // The .drv suffix is omitted in its `name` field. + StorePath deriver = 4; } // Represents a path in the Nix store (a direct child of STORE_DIR). diff --git a/tvix/store/protos/pathinfo_test.go b/tvix/store/protos/pathinfo_test.go index 74af50e5691f..9a329f0010fc 100644 --- a/tvix/store/protos/pathinfo_test.go +++ b/tvix/store/protos/pathinfo_test.go @@ -120,4 +120,30 @@ func TestValidate(t *testing.T) { _, err := pi.Validate() assert.Error(t, err, "must not validate") }) + + t.Run("happy deriver", func(t *testing.T) { + pi := genPathInfoSymlink() + + // add the Deriver Field. + pi.Deriver = &storev1pb.StorePath{ + Digest: exampleStorePathDigest, + Name: "foo", + } + + _, err := pi.Validate() + assert.NoError(t, err, "must validate") + }) + + t.Run("invalid deriver", func(t *testing.T) { + pi := genPathInfoSymlink() + + // add the Deriver Field, with a broken digest + pi.Deriver = &storev1pb.StorePath{ + Digest: []byte{}, + Name: "foo2", + } + _, err := pi.Validate() + assert.Error(t, err, "must not validate") + }) + } diff --git a/tvix/store/src/bin/tvix-store.rs b/tvix/store/src/bin/tvix-store.rs index 11f19857dd6f..2f7589b07356 100644 --- a/tvix/store/src/bin/tvix-store.rs +++ b/tvix/store/src/bin/tvix-store.rs @@ -296,6 +296,7 @@ async fn main() -> Result<(), Box> { signatures: vec![], reference_names: vec![], }), + deriver: None, }; // put into [PathInfoService], and return the PathInfo that we get back diff --git a/tvix/store/src/proto/mod.rs b/tvix/store/src/proto/mod.rs index 718e24912de1..4ebd4a31908f 100644 --- a/tvix/store/src/proto/mod.rs +++ b/tvix/store/src/proto/mod.rs @@ -48,7 +48,7 @@ pub enum ValidatePathInfoError { #[error("Inconsistent Number of References: {0} (references) vs {1} (narinfo)")] InconsistentNumberOfReferences(usize, usize), - /// A string in narinfo.reference_names does not parse to a StorePath. + /// A string in narinfo.reference_names does not parse to a [store_path::StorePath]. #[error("Invalid reference_name at position {0}: {1}")] InvalidNarinfoReferenceName(usize, String), @@ -60,6 +60,10 @@ pub enum ValidatePathInfoError { [u8; store_path::DIGEST_SIZE], [u8; store_path::DIGEST_SIZE], ), + + /// The deriver field is invalid. + #[error("deriver field is invalid: {0}")] + InvalidDeriverField(store_path::Error), } /// Parses a root node name. @@ -152,6 +156,16 @@ impl PathInfo { } }; + // If the Deriver field is populated, ensure it parses to a + // [store_path::StorePath]. + // We can't check for it to *not* end with .drv, as the .drv files produced by + // recursive Nix end with multiple .drv suffixes, and only one is popped when + // converting to this field. + if let Some(deriver) = &self.deriver { + store_path::StorePath::from_name_and_digest(deriver.name.clone(), &deriver.digest) + .map_err(ValidatePathInfoError::InvalidDeriverField)?; + } + // return the root nix path Ok(root_nix_path) } diff --git a/tvix/store/src/proto/tests/pathinfo.rs b/tvix/store/src/proto/tests/pathinfo.rs index 5e1ae9c45b64..03094ed734ce 100644 --- a/tvix/store/src/proto/tests/pathinfo.rs +++ b/tvix/store/src/proto/tests/pathinfo.rs @@ -262,3 +262,34 @@ fn validate_symlink_target_null_byte_invalid() { node.validate().expect_err("must fail validation"); } + +/// Create a PathInfo with a correct deriver field and ensure it succeeds. +#[test] +fn validate_valid_deriver() { + let mut path_info = PATH_INFO_WITHOUT_NARINFO.clone(); + + // add a valid deriver + path_info.deriver = Some(crate::proto::StorePath { + name: "foo".to_string(), + digest: DUMMY_OUTPUT_HASH.clone(), + }); + + path_info.validate().expect("must validate"); +} + +/// Create a PathInfo with a broken deriver field and ensure it fails. +#[test] +fn validate_invalid_deriver() { + let mut path_info = PATH_INFO_WITHOUT_NARINFO.clone(); + + // add a broken deriver (invalid digest) + path_info.deriver = Some(crate::proto::StorePath { + name: "foo".to_string(), + digest: vec![].into(), + }); + + match path_info.validate().expect_err("must fail validation") { + ValidatePathInfoError::InvalidDeriverField(_) => {} + e => panic!("unexpected error: {:?}", e), + } +} diff --git a/tvix/store/src/tests/fixtures.rs b/tvix/store/src/tests/fixtures.rs index f0d0f35378cf..5290581688e6 100644 --- a/tvix/store/src/tests/fixtures.rs +++ b/tvix/store/src/tests/fixtures.rs @@ -109,6 +109,7 @@ lazy_static! { }), references: vec![DUMMY_OUTPUT_HASH.clone()], narinfo: None, + deriver: None, }; /// A PathInfo message with .narinfo populated. -- cgit 1.4.1