From 5f8eb4eeaaad31aedc45efee3143e6b0bbc982a4 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Tue, 10 Oct 2023 23:06:42 +0200 Subject: feat(tvix/store/protos): add StorePath message This encodes a store path a bit more concise, which is used in the Deriver field. Change-Id: Ibfb54d3b206917e51970d1d5fe94fcedb901704b Reviewed-on: https://cl.tvl.fyi/c/depot/+/9646 Reviewed-by: Connor Brewster Tested-by: BuildkiteCI --- tvix/store/protos/pathinfo.pb.go | 144 ++++++++++++++++++++++++++++++--------- tvix/store/protos/pathinfo.proto | 11 +++ tvix/store/src/proto/mod.rs | 30 ++++---- 3 files changed, 137 insertions(+), 48 deletions(-) (limited to 'tvix') diff --git a/tvix/store/protos/pathinfo.pb.go b/tvix/store/protos/pathinfo.pb.go index 1e5479ac8f75..d6b517296448 100644 --- a/tvix/store/protos/pathinfo.pb.go +++ b/tvix/store/protos/pathinfo.pb.go @@ -94,6 +94,66 @@ func (x *PathInfo) GetNarinfo() *NARInfo { return nil } +// Represents a path in the Nix store (a direct child of STORE_DIR). +// It is commonly formatted by a nixbase32-encoding the digest, and +// concatenating the name, separated by a `-`. +type StorePath struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The string after digest and `-`. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // The digest (20 bytes). + Digest []byte `protobuf:"bytes,2,opt,name=digest,proto3" json:"digest,omitempty"` +} + +func (x *StorePath) Reset() { + *x = StorePath{} + if protoimpl.UnsafeEnabled { + mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StorePath) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StorePath) ProtoMessage() {} + +func (x *StorePath) ProtoReflect() protoreflect.Message { + mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StorePath.ProtoReflect.Descriptor instead. +func (*StorePath) Descriptor() ([]byte, []int) { + return file_tvix_store_protos_pathinfo_proto_rawDescGZIP(), []int{1} +} + +func (x *StorePath) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *StorePath) GetDigest() []byte { + if x != nil { + return x.Digest + } + return nil +} + // Nix C++ uses NAR (Nix Archive) as a format to transfer store paths, // and stores metadata and signatures in NARInfo files. // Store all these attributes in a separate message. @@ -125,7 +185,7 @@ type NARInfo struct { func (x *NARInfo) Reset() { *x = NARInfo{} if protoimpl.UnsafeEnabled { - mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[1] + mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -138,7 +198,7 @@ func (x *NARInfo) String() string { func (*NARInfo) ProtoMessage() {} func (x *NARInfo) ProtoReflect() protoreflect.Message { - mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[1] + mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -151,7 +211,7 @@ func (x *NARInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use NARInfo.ProtoReflect.Descriptor instead. func (*NARInfo) Descriptor() ([]byte, []int) { - return file_tvix_store_protos_pathinfo_proto_rawDescGZIP(), []int{1} + return file_tvix_store_protos_pathinfo_proto_rawDescGZIP(), []int{2} } func (x *NARInfo) GetNarSize() uint64 { @@ -195,7 +255,7 @@ type NARInfo_Signature struct { func (x *NARInfo_Signature) Reset() { *x = NARInfo_Signature{} if protoimpl.UnsafeEnabled { - mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[2] + mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -208,7 +268,7 @@ func (x *NARInfo_Signature) String() string { func (*NARInfo_Signature) ProtoMessage() {} func (x *NARInfo_Signature) ProtoReflect() protoreflect.Message { - mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[2] + mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -221,7 +281,7 @@ func (x *NARInfo_Signature) ProtoReflect() protoreflect.Message { // Deprecated: Use NARInfo_Signature.ProtoReflect.Descriptor instead. func (*NARInfo_Signature) Descriptor() ([]byte, []int) { - return file_tvix_store_protos_pathinfo_proto_rawDescGZIP(), []int{1, 0} + return file_tvix_store_protos_pathinfo_proto_rawDescGZIP(), []int{2, 0} } func (x *NARInfo_Signature) GetName() string { @@ -254,25 +314,28 @@ var file_tvix_store_protos_pathinfo_proto_rawDesc = []byte{ 0x52, 0x0a, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x30, 0x0a, 0x07, 0x6e, 0x61, 0x72, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x41, - 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x69, 0x6e, 0x66, 0x6f, 0x22, 0xe3, - 0x01, 0x0a, 0x07, 0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x61, - 0x72, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x6e, 0x61, - 0x72, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x68, 0x61, - 0x32, 0x35, 0x36, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6e, 0x61, 0x72, 0x53, 0x68, - 0x61, 0x32, 0x35, 0x36, 0x12, 0x40, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, - 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66, 0x6f, - 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, - 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x0e, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x1a, - 0x33, 0x0a, 0x09, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, - 0x64, 0x61, 0x74, 0x61, 0x42, 0x28, 0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, - 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x69, 0x6e, 0x66, 0x6f, 0x22, 0x37, + 0x0a, 0x09, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, + 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0xe3, 0x01, 0x0a, 0x07, 0x4e, 0x41, 0x52, 0x49, + 0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, + 0x0a, 0x0a, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x09, 0x6e, 0x61, 0x72, 0x53, 0x68, 0x61, 0x32, 0x35, 0x36, 0x12, 0x40, 0x0a, + 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, + 0x27, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, + 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, + 0x6e, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x1a, 0x33, 0x0a, 0x09, 0x53, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, + 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x42, 0x28, 0x5a, + 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, + 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, + 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -287,17 +350,18 @@ func file_tvix_store_protos_pathinfo_proto_rawDescGZIP() []byte { return file_tvix_store_protos_pathinfo_proto_rawDescData } -var file_tvix_store_protos_pathinfo_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_tvix_store_protos_pathinfo_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_tvix_store_protos_pathinfo_proto_goTypes = []interface{}{ (*PathInfo)(nil), // 0: tvix.store.v1.PathInfo - (*NARInfo)(nil), // 1: tvix.store.v1.NARInfo - (*NARInfo_Signature)(nil), // 2: tvix.store.v1.NARInfo.Signature - (*protos.Node)(nil), // 3: tvix.castore.v1.Node + (*StorePath)(nil), // 1: tvix.store.v1.StorePath + (*NARInfo)(nil), // 2: tvix.store.v1.NARInfo + (*NARInfo_Signature)(nil), // 3: tvix.store.v1.NARInfo.Signature + (*protos.Node)(nil), // 4: tvix.castore.v1.Node } var file_tvix_store_protos_pathinfo_proto_depIdxs = []int32{ - 3, // 0: tvix.store.v1.PathInfo.node:type_name -> tvix.castore.v1.Node - 1, // 1: tvix.store.v1.PathInfo.narinfo:type_name -> tvix.store.v1.NARInfo - 2, // 2: tvix.store.v1.NARInfo.signatures:type_name -> tvix.store.v1.NARInfo.Signature + 4, // 0: tvix.store.v1.PathInfo.node:type_name -> tvix.castore.v1.Node + 2, // 1: tvix.store.v1.PathInfo.narinfo:type_name -> tvix.store.v1.NARInfo + 3, // 2: tvix.store.v1.NARInfo.signatures:type_name -> tvix.store.v1.NARInfo.Signature 3, // [3:3] is the sub-list for method output_type 3, // [3:3] is the sub-list for method input_type 3, // [3:3] is the sub-list for extension type_name @@ -324,7 +388,7 @@ func file_tvix_store_protos_pathinfo_proto_init() { } } file_tvix_store_protos_pathinfo_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NARInfo); i { + switch v := v.(*StorePath); i { case 0: return &v.state case 1: @@ -336,6 +400,18 @@ func file_tvix_store_protos_pathinfo_proto_init() { } } file_tvix_store_protos_pathinfo_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NARInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_tvix_store_protos_pathinfo_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*NARInfo_Signature); i { case 0: return &v.state @@ -354,7 +430,7 @@ func file_tvix_store_protos_pathinfo_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_tvix_store_protos_pathinfo_proto_rawDesc, NumEnums: 0, - NumMessages: 3, + NumMessages: 4, NumExtensions: 0, NumServices: 0, }, diff --git a/tvix/store/protos/pathinfo.proto b/tvix/store/protos/pathinfo.proto index aa98c6df9a2d..080e3f6b49a2 100644 --- a/tvix/store/protos/pathinfo.proto +++ b/tvix/store/protos/pathinfo.proto @@ -23,6 +23,17 @@ message PathInfo { NARInfo narinfo = 3; } +// Represents a path in the Nix store (a direct child of STORE_DIR). +// It is commonly formatted by a nixbase32-encoding the digest, and +// concatenating the name, separated by a `-`. +message StorePath { + // The string after digest and `-`. + string name = 1; + + // The digest (20 bytes). + bytes digest = 2; +} + // Nix C++ uses NAR (Nix Archive) as a format to transfer store paths, // and stores metadata and signatures in NARInfo files. // Store all these attributes in a separate message. diff --git a/tvix/store/src/proto/mod.rs b/tvix/store/src/proto/mod.rs index 7921adc4062e..718e24912de1 100644 --- a/tvix/store/src/proto/mod.rs +++ b/tvix/store/src/proto/mod.rs @@ -1,7 +1,7 @@ #![allow(clippy::derive_partial_eq_without_eq, non_snake_case)] use data_encoding::BASE64; // https://github.com/hyperium/tonic/issues/1056 -use nix_compat::store_path::{self, StorePath}; +use nix_compat::store_path; use thiserror::Error; use tvix_castore::proto::{self as castorepb, NamedNode, ValidateNodeError}; @@ -64,13 +64,13 @@ pub enum ValidatePathInfoError { /// Parses a root node name. /// -/// On success, this returns the parsed [StorePath]. +/// On success, this returns the parsed [store_path::StorePath]. /// On error, it returns an error generated from the supplied constructor. fn parse_node_name_root( name: &[u8], err: fn(Vec, store_path::Error) -> E, -) -> Result { - match StorePath::from_bytes(name) { +) -> Result { + match store_path::StorePath::from_bytes(name) { Ok(np) => Ok(np), Err(e) => Err(err(name.to_vec(), e)), } @@ -78,9 +78,9 @@ fn parse_node_name_root( impl PathInfo { /// validate performs some checks on the PathInfo struct, - /// Returning either a [StorePath] of the root node, or a + /// Returning either a [store_path::StorePath] of the root node, or a /// [ValidatePathInfoError]. - pub fn validate(&self) -> Result { + pub fn validate(&self) -> Result { // ensure the references have the right number of bytes. for (i, reference) in self.references.iter().enumerate() { if reference.len() != store_path::DIGEST_SIZE { @@ -111,13 +111,15 @@ impl PathInfo { // parse references in reference_names. for (i, reference_name_str) in narinfo.reference_names.iter().enumerate() { // ensure thy parse as (non-absolute) store path - let reference_names_store_path = - StorePath::from_bytes(reference_name_str.as_bytes()).map_err(|_| { - ValidatePathInfoError::InvalidNarinfoReferenceName( - i, - reference_name_str.to_owned(), - ) - })?; + let reference_names_store_path = store_path::StorePath::from_bytes( + reference_name_str.as_bytes(), + ) + .map_err(|_| { + ValidatePathInfoError::InvalidNarinfoReferenceName( + i, + reference_name_str.to_owned(), + ) + })?; // ensure their digest matches the one at self.references[i]. { @@ -137,7 +139,7 @@ impl PathInfo { } } - // Ensure there is a (root) node present, and it properly parses to a [StorePath]. + // Ensure there is a (root) node present, and it properly parses to a [store_path::StorePath]. let root_nix_path = match &self.node { None | Some(castorepb::Node { node: None }) => { Err(ValidatePathInfoError::NoNodePresent())? -- cgit 1.4.1